gm.text.Gemma4Tokenizer

class gemma.gm.text.Gemma4Tokenizer(

path: str | os.PathLike = 'gs://gemma-data/tokenizers/tokenizer_gemma4.model',

*,

custom_tokens: dict[int,

str]=<factory>,

)

Bases: gemma.gm.text._tokenizer.Tokenizer

Tokenizer for Gemma 4.

path: str | os.PathLike = 'gs://gemma-data/tokenizers/tokenizer_gemma4.model'

special_tokens

alias of gemma.gm.text._tokenizer._Gemma4SpecialTokens

VERSION: ClassVar[int | str] = 4

FORMAT: ClassVar[dialog._src.string.str_compat.Format] = 'gemma4'