gm.text.Gemma3Tokenizer

class gemma.gm.text.Gemma3Tokenizer(

path: str | os.PathLike = 'gs://gemma-data/tokenizers/tokenizer_gemma3.model',

*,

custom_tokens: dict[int,

str]=<factory>,

)

Bases: gemma.gm.text._tokenizer.Tokenizer

Tokenizer for Gemma 3.

path: str | os.PathLike = 'gs://gemma-data/tokenizers/tokenizer_gemma3.model'

special_tokens

alias of gemma.gm.text._tokenizer._Gemma3SpecialTokens

FORBIDDEN_TOKENS: ClassVar[tuple[int, ...]] = (_Gemma3SpecialTokens.IMAGE_PLACEHOLDER, _Gemma3SpecialTokens.END_OF_IMAGE)

VERSION: ClassVar[int | str] = 3