gm.data.Parquet

class gemma.gm.data.Parquet(*, _fake_refs: 'type[_FakeRefsUnset] | dict[str, _FakeRootCfg]' = <class 'kauldron.utils.config_util._FakeRefsUnset'>, batch_size: int | None = None, seed: Union[kauldron.ktyping.array_type_meta.UInt32['2'], kauldron.ktyping.array_type_meta.Fry[''], kauldron.ktyping.array_type_meta.KdPRNGKey, kauldron.ktyping.array_type_meta.ScalarInt, Sequence[int], NoneType] = _FakeRootCfg('cfg.seed'), transforms: 'tr_normalize.Transformations' = <factory>, num_epochs: 'Optional[int]' = None, batch_drop_remainder: 'bool | str | DropRemainder' = True, num_workers: 'int' = 16, read_options: 'grain.ReadOptions | None' = None, enable_profiling: 'bool' = False, per_worker_buffer_size: 'int' = 1, shard_by_process: 'bool' = True, worker_init_fn: 'Callable[[int, int], None] | None' = None, shuffle: 'bool', path: 'epath.PathLike | list[epath.PathLike]')

Bases: kauldron.data.py.base.DataSourceBase

path: str | os.PathLike | list[str | os.PathLike]

property data_source: grain._src.python.dataset.base.RandomAccessDataSource