MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
@dataclass
class ModelArguments:
"""
Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
"""
model_name_or_path: Optional[str] = field(
default=None,
metadata={
"help": "The model checkpoint for weights initialization. Leave None if you want to train a model from scratch."
},
)
model_type: Optional[str] = field(
default=None,
metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
)
config_name: Optional[str] = field(
default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
)
tokenizer_name: Optional[str] = field(
default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
)
cache_dir: Optional[str] = field(
default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
)
@dataclass
class DataTrainingArguments:
"""
Arguments pertaining to what data we are going to input our model for training and eval.
"""
train_data_file: Optional[str] = field(
default=None, metadata={"help": "The input training data file (a text file)."}
)
eval_data_file: Optional[str] = field(
default=None,
metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
)
line_by_line: bool = field(
default=False,
metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
)
mlm: bool = field(
default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
)
mlm_probability: float = field(
default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
)
plm_probability: float = field(
default=1 / 6,
metadata={
"help": "Ratio of length of a span of masked tokens to surrounding context length for permutation language modeling."
},
)
max_span_length: int = field(
default=5, metadata={"help": "Maximum length of a span of masked tokens for permutation language modeling."}
)
block_size: int = field(
default=-1,
metadata={
"help": "Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens)."
},
)
overwrite_cache: bool = field(
default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
)
arrow: bool = field(
default=True,
metadata={
"help": "Use Arrow-based HF NLP for optimization."
},
)
See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument."
)
if (
os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir)
and training_args.do_train
and not training_args.overwrite_output_dir
):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning("You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name"
)
tokenizer.pad_token = "<|endoftext|>"
tokenizer._pad_token = "<|endoftext|>"
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
"--mlm flag (masked language modeling)."
)
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (
get_dataset(data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir) if training_args.do_train else None
)
eval_dataset = (
get_dataset(data_args, tokenizer=tokenizer, evaluate=True, cache_dir=model_args.cache_dir)
if training_args.do_eval
else None
)
if config.model_type == "xlnet":
data_collator = DataCollatorForPermutationLanguageModeling(
tokenizer=tokenizer,
plm_probability=data_args.plm_probability,
max_span_length=data_args.max_span_length,
)
else:
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (
model_args.model_name_or_path
if model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path)
else None
)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir, "eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def _mp_fn(index):
For xla_spawn (TPUs)
main()
if name == "main":
main()
2. set torch-xla-nightly Conda & set env
3. run script from checkpoint (replace dataset, since I cannot upload 48 GB worth of arrow files)
Exception in device=TPU:3: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Exception in device=TPU:5: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Exception in device=TPU:6: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Exception in device=TPU:1: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Exception in device=TPU:0: don't know how to restore data location of torch.FloatStorage (tagged with xla:1)
Exception in device=TPU:4: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Exception in device=TPU:7: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Exception in device=TPU:2: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Traceback (most recent call last):
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 330, in _mp_start_fn
_start_fn(index, pf_cfg, fn, args)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 324, in _start_fn
fn(gindex, *args)
File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 332, in _mp_fn
main()
File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 300, in main
trainer.train(model_path=model_path)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/transformers/trainer.py", line 629, in train
torch.load(os.path.join(model_path, "optimizer.pt"), map_location=self.args.device)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 592, in load
return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 851, in _load
result = unpickler.load()
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 843, in persistent_load
load_tensor(data_type, size, key, _maybe_decode_ascii(location))
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 832, in load_tensor
loaded_storages[key] = restore_location(storage, location)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 812, in restore_location
return default_restore_location(storage, str(map_location))
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 180, in default_restore_location
location + ")")
RuntimeError: don't know how to restore data location of torch.FloatStorage (tagged with xla:0)
Traceback (most recent call last):
Traceback (most recent call last):
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 330, in _mp_start_fn
_start_fn(index, pf_cfg, fn, args)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 324, in _start_fn
fn(gindex, args)
File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 332, in _mp_fn
main()
File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 300, in main
trainer.train(model_path=model_path)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/transformers/trainer.py", line 629, in train
torch.load(os.path.join(model_path, "optimizer.pt"), map_location=self.args.device)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 330, in _mp_start_fn
_start_fn(index, pf_cfg, fn, args)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 324, in _start_fn
fn(gindex, args)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 592, in load
return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 851, in _load
result = unpickler.load()
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 843, in persistent_load
load_tensor(data_type, size, key, _maybe_decode_ascii(location))
File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 332, in _mp_fn
Environment info
transformers
version: 3.4.0Who can help
@LysandreJik @sgugger @TevenLeScao
Information
Model I am using (Bert, XLNet ...): GPT2
The problem arises when using:
The tasks I am working on is:
To reproduce
Steps to reproduce the behavior:
from datasets import list_datasets, load_dataset
from transformers import ( CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed, )
logger = logging.getLogger(name)
MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys()) MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
@dataclass class ModelArguments: """ Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. """
@dataclass class DataTrainingArguments: """ Arguments pertaining to what data we are going to input our model for training and eval. """
def get_dataset( args: DataTrainingArguments, tokenizer: PreTrainedTokenizer, evaluate: bool = False, cache_dir: Optional[str] = "./cache", ): tokenizer.pad_token = "<|endoftext|>" tokenizer._pad_token = "<|endoftext|>"
tokenizer.pad_token_id = 50256
def main():
See all possible arguments in src/transformers/training_args.py
def _mp_fn(index):
For xla_spawn (TPUs)
if name == "main": main()
XLA_USE_BF16=1 python3 examples/xla_spawn.py --num_cores 8 examples/language-modeling/run_language_modeling.py --output_dir=kogpt1 --model_type=gpt2 --do_train --train_data_file=/home/ksjcom0705_gmail_com/NEWS_ARROW --overwrite_output_dir --per_device_train_batch_size=6 --save_steps 10000 --num_train_epochs=1 --block_size 2048 --eval_steps 10000 --logging_steps=10000 --tokenizer_name /home/ksjcom0705_gmail_com/kotok --model_name_or_path=kogpt1/checkpoint-1000
Exception in device=TPU:3: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Exception in device=TPU:5: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Exception in device=TPU:6: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Exception in device=TPU:1: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Exception in device=TPU:0: don't know how to restore data location of torch.FloatStorage (tagged with xla:1) Exception in device=TPU:4: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Exception in device=TPU:7: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Exception in device=TPU:2: don't know how to restore data location of torch.FloatStorage (tagged with xla:0) Traceback (most recent call last): File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 330, in _mp_start_fn _start_fn(index, pf_cfg, fn, args) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 324, in _start_fn fn(gindex, *args) File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 332, in _mp_fn main() File "/home/ksjcom0705_gmail_com/transformers/examples/language-modeling/run_language_modeling.py", line 300, in main trainer.train(model_path=model_path) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/transformers/trainer.py", line 629, in train torch.load(os.path.join(model_path, "optimizer.pt"), map_location=self.args.device) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 592, in load return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 851, in _load result = unpickler.load() File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 843, in persistent_load load_tensor(data_type, size, key, _maybe_decode_ascii(location)) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 832, in load_tensor loaded_storages[key] = restore_location(storage, location) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 812, in restore_location return default_restore_location(storage, str(map_location)) File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch/serialization.py", line 180, in default_restore_location
Expected behavior
Run normally