def create_hf_dataset(
dataset_config: HFDatasetConfig, model_n_ctx: Optional[int] = None
) -> Dataset:
"""[...]
model_n_ctx (int): The max context length of the model. Used for HFDatasetConfigs. Data
sequences are packed to dataset_config.n_ctx if it is not None and is <= model_n_ctx,
otherwise to model_n_ctx.[...]
"""
assert model_n_ctx is not None
assert model_n_ctx is not None fails when model_n_ctx is None, and it's at the first line of the function.
assert model_n_ctx is not None
fails whenmodel_n_ctx
is None, and it's at the first line of the function.