Closed SrikanthChellappa closed 1 year ago
Rocketknight1
commented
1 year ago Hi, I'm sorry, we don't really support TFTrainer anymore! We recommend fine-tuning with Keras methods like model.fit() instead. Please see our notebooks on translation or summarization for examples of training TF Seq2Seq models with this approach.
SrikanthChellappa
commented
1 year ago Thanks @Rocketknight1 for the update.
SrikanthChellappa
commented
1 year ago @Rocketknight1 Can you pls assist me on how to save the model with .keras or .h5 extensions to load it again. I am able to save the model as .keras buty it throws error when i load it again. Pls see the code used as below. Kindly assist
I was able to save my model earlier using model.save('FlanT5-Chatbot_model.keras')
When i tried loading the model again as below from tensorflow.keras.models import load_model model=load_model('FlanT5-Chatbot_model.keras')
I am getting "ModuleNotFoundError" error. Error stack is given below
ModuleNotFoundError Traceback (most recent call last) File ~\AppData\Roaming\Python\Python311\site-packages\keras\src\saving\serialization_lib.py:800, in _retrieve_class_or_fn(name, registered_name, module, obj_type, full_config, custom_objects) 799 try: --> 800 mod = importlib.import_module(module) 801 except ModuleNotFoundError:
File C:\ProgramData\anaconda3\Lib\importlib__init__.py:126, in import_module(name, package) 125 level += 1 --> 126 return _bootstrap._gcd_import(name[level:], package, level)
File
File
File
File
File
File
File
File
File
File
File
ModuleNotFoundError: No module named 'transformers.models'
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last) Cell In[4], line 2 1 from tensorflow.keras.models import load_model ----> 2 model=load_model('FlanT5-Chatbot_model.keras')
File ~\AppData\Roaming\Python\Python311\site-packages\keras\src\saving\saving_api.py:254, in load_model(filepath, custom_objects, compile, safe_mode, kwargs) 249 if kwargs: 250 raise ValueError( 251 "The following argument(s) are not supported " 252 f"with the native Keras format: {list(kwargs.keys())}" 253 ) --> 254 return saving_lib.load_model( 255 filepath, 256 custom_objects=custom_objects, 257 compile=compile, 258 safe_mode=safe_mode, 259 ) 261 # Legacy case. 262 return legacy_sm_saving_lib.load_model( 263 filepath, custom_objects=custom_objects, compile=compile, kwargs 264 )
File ~\AppData\Roaming\Python\Python311\site-packages\keras\src\saving\saving_lib.py:281, in load_model(filepath, custom_objects, compile, safe_mode) 278 asset_store.close() 280 except Exception as e: --> 281 raise e 282 else: 283 return model
File ~\AppData\Roaming\Python\Python311\site-packages\keras\src\saving\saving_lib.py:246, in load_model(filepath, custom_objects, compile, safe_mode) 244 # Construct the model from the configuration file in the archive. 245 with ObjectSharingScope(): --> 246 model = deserialize_keras_object( 247 config_dict, custom_objects, safe_mode=safe_mode 248 ) 250 all_filenames = zf.namelist() 251 if _VARS_FNAME + ".h5" in all_filenames:
File ~\AppData\Roaming\Python\Python311\site-packages\keras\src\saving\serialization_lib.py:705, in deserialize_keras_object(config, custom_objects, safe_mode, **kwargs) 702 if obj is not None: 703 return obj --> 705 cls = _retrieve_class_or_fn( 706 class_name, 707 registered_name, 708 module, 709 obj_type="class", 710 full_config=config, 711 custom_objects=custom_objects, 712 ) 714 if isinstance(cls, types.FunctionType): 715 return cls
File ~\AppData\Roaming\Python\Python311\site-packages\keras\src\saving\serialization_lib.py:802, in _retrieve_class_or_fn(name, registered_name, module, obj_type, full_config, custom_objects) 800 mod = importlib.import_module(module) 801 except ModuleNotFoundError: --> 802 raise TypeError( 803 f"Could not deserialize {obj_type} '{name}' because " 804 f"its parent module {module} cannot be imported. " 805 f"Full object config: {full_config}" 806 ) 807 obj = vars(mod).get(name, None) 809 if obj is None: 810 # Special case for keras.metrics.metrics
TypeError: Could not deserialize class 'TFT5ForConditionalGeneration' because its parent module transformers.models.t5.modeling_tf_t5 cannot be imported. Full object config: {'module': 'transformers.models.t5.modeling_tf_t5', 'class_name': 'TFT5ForConditionalGeneration', 'config': {'vocab_size': 32128, 'd_model': 768, 'd_kv': 64, 'd_ff': 2048, 'num_layers': 12, 'num_decoder_layers': 12, 'num_heads': 12, 'relative_attention_num_buckets': 32, 'relative_attention_max_distance': 128, 'dropout_rate': 0.1, 'classifier_dropout': 0.0, 'layer_norm_epsilon': 1e-06, 'initializer_factor': 1.0, 'feed_forward_proj': 'gated-gelu', 'use_cache': True, 'dense_act_fn': 'gelu_new', 'is_gated_act': True, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': True, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['T5ForConditionalGeneration'], 'finetuning_task': None, 'id2label': {'0': 'LABEL_0', '1': 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': 0, 'eos_token_id': 1, 'sep_token_id': None, 'decoder_start_token_id': 0, 'task_specific_params': {'summarization': {'early_stopping': True, 'length_penalty': 2.0, 'max_length': 200, 'min_length': 30, 'no_repeat_ngram_size': 3, 'num_beams': 4, 'prefix': 'summarize: '}, 'translation_en_to_de': {'early_stopping': True, 'max_length': 300, 'num_beams': 4, 'prefix': 'translate English to German: '}, 'translation_en_to_fr': {'early_stopping': True, 'max_length': 300, 'num_beams': 4, 'prefix': 'translate English to French: '}, 'translation_en_to_ro': {'early_stopping': True, 'max_length': 300, 'num_beams': 4, 'prefix': 'translate English to Romanian: '}}, 'problem_type': None, '_name_or_path': 'google/flan-t5-base', 'transformers_version': '4.34.1', 'model_type': 't5', 'n_positions': 512, 'output_past': True}, 'registered_name': 'TFT5ForConditionalGeneration', 'compile_config': {'optimizer': {'module': 'transformers.optimization_tf', 'class_name': 'AdamWeightDecay', 'config': {'name': 'AdamWeightDecay', 'learning_rate': 1.9999999494757503e-05, 'decay': 0.0, 'beta_1': 0.8999999761581421, 'beta_2': 0.9990000128746033, 'epsilon': 1e-07, 'amsgrad': False, 'weight_decay_rate': 0.01}, 'registered_name': 'AdamWeightDecay'}, 'loss': {'module': 'builtins', 'class_name': 'function', 'config': 'dummy_loss', 'registered_name': 'function'}, 'metrics': None, 'loss_weights': None, 'weighted_metrics': None, 'run_eagerly': None, 'steps_per_execution': None, 'jit_compile': None}}
System Info
Transformers version: 4.34.1 Tensorflow version: 2.14.0 Python version: 3.11.6
My Code snippet from transformers import TFAutoModelForSeq2SeqLM, AutoTokenizer, GenerationConfig, TFTrainingArguments, TFTrainer training_args = TFTrainingArguments( output_dir=output_dir, learning_rate=1e-5, num_train_epochs=1, weight_decay=0.01, logging_steps=1, max_steps=1 )
trainer = TFTrainer( model=original_model, args=training_args, train_dataset=tokenized_datasets['train'], eval_dataset=tokenized_datasets['validation'] ) trainer.train()
Error Trace
AttributeError Traceback (most recent call last) Cell In[52], line 1 ----> 1 trainer.train()
File C:\ProgramData\anaconda3\envs\tfenv\Lib\site-packages\transformers\trainer_tf.py:479, in TFTrainer.train(self) 475 def train(self) -> None: 476 """ 477 Train method to train the model. 478 """ --> 479 train_ds = self.get_train_tfdataset() 481 if self.args.debug: 482 tf.summary.trace_on(graph=True, profiler=True)
File C:\ProgramData\anaconda3\envs\tfenv\Lib\site-packages\transformers\trainer_tf.py:159, in TFTrainer.get_train_tfdataset(self) 156 raise ValueError("Trainer: training requires a train_dataset.") 158 self.total_train_batch_size = self.args.train_batch_size * self.args.gradient_accumulation_steps --> 159 self.num_train_examples = self.train_dataset.cardinality().numpy() 161 if self.num_train_examples < 0: 162 raise ValueError("The training dataset must have an asserted cardinality")
AttributeError: 'Dataset' object has no attribute 'cardinality'
@gante @Rocketknight1 @ArthurZucker - Pls assist asap
Who can help?
No response
Information
Tasks
examplesfolder (such as GLUE/SQuAD, ...)Reproduction
from transformers import TFAutoModelForSeq2SeqLM, AutoTokenizer, GenerationConfig, TFTrainingArguments, TFTrainer training_args = TFTrainingArguments( output_dir=output_dir, learning_rate=1e-5, num_train_epochs=1, weight_decay=0.01, logging_steps=1, max_steps=1 )
trainer = TFTrainer( model=original_model, args=training_args, train_dataset=tokenized_datasets['train'], eval_dataset=tokenized_datasets['validation'] ) trainer.train()
Expected behavior
The training should happen but it throws error