gm.text.SpecialTokens

class gemma.gm.text.SpecialTokens(

new_class_name,

/,

names,

*,

module=None,

qualname=None,

type=None,

start=1,

boundary=None,

)

Bases: enum.IntEnum

Special tokens ids.

PAD: ClassVar[int]

EOS: ClassVar[int]

BOS: ClassVar[int]

UNK: ClassVar[int]

MASK: ClassVar[int]

CUSTOM: ClassVar[int]

START_OF_TURN: ClassVar[int]

END_OF_TURN: ClassVar[int]

IMAGE_PLACEHOLDER: ClassVar[int]

START_OF_IMAGE: ClassVar[int]

END_OF_IMAGE: ClassVar[int]

AUDIO_PLACEHOLDER: ClassVar[int]

START_OF_AUDIO: ClassVar[int]

END_OF_AUDIO: ClassVar[int]

BEGIN_OF_TOOL_RESPONSE: ClassVar[int]

END_OF_TOOL_RESPONSE: ClassVar[int]