gm.text.Gemma2Tokenizer

class gemma.gm.text.Gemma2Tokenizer(

path: str | os.PathLike = 'gs://gemma-data/tokenizers/tokenizer_gemma2.model',

*,

custom_tokens: dict[int,

str]=<factory>,

)

Bases: gemma.gm.text._tokenizer.Tokenizer

Tokenizer for Gemma 2.

path: str | os.PathLike = 'gs://gemma-data/tokenizers/tokenizer_gemma2.model'

special_tokens

alias of gemma.gm.text._tokenizer._Gemma2SpecialTokens

VERSION: ClassVar[int | str] = 2