zqs01
commented
1 year ago @xiabingquan Hi, all our models use fp16. I use python test_load.py dummy,
# test_load.py
from argparse import Namespace
import fairseq
from fairseq import checkpoint_utils, utils
utils.import_user_module(Namespace(user_dir='/path/to/vat_hubert/vathubert'))
models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(["/path/to/model"])
print(saved_cfg)The output logs are as follows
{'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 200, 'log_format': 'json', 'log_file': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1337, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': True, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': '/mnt/default/v-qiushizhu/vatlm_related/vat_hubert3a/vathubert', 'empty_cache_freq': 10000, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 32, 'distributed_num_procs': 8, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': 'tcp://192.168.0.60:42771', 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': True, 'ddp_backend': 'no_c10d', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_algorithm': 'LocalSGD', 'localsgd_frequency': 3, 'nprocs_per_node': 8, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': True, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False}, 'dataset': {'_name': None, 'num_workers': 6, 'skip_invalid_size_inputs_valid_test': True, 'max_tokens': 3000, 'batch_size': None, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 40, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 5, 'validate_interval_updates': 10000, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': 3000, 'batch_size_valid': None, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 600000, 'stop_time_hours': 0.0, 'clip_norm': 10.0, 'sentence_avg': False, 'update_freq': [2], 'lr': [0.002], 'stop_min_lr': -1.0, 'use_bmuf': False}, 'checkpoint': {'_name': None, 'save_dir': 'checkpoints', 'restore_file': 'checkpoint_last.pt', 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 3, 'save_interval_updates': 10000, 'keep_interval_updates': 1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': -1, 'no_save': False, 'no_epoch_checkpoints': False, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'loss', 'maximize_best_checkpoint_metric': False, 'patience': -1, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 32}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': {'_name': 'vat_hubert', 'label_rate': 25, 'skip_masked': False, 'skip_nomask': False, 'modality_dropout': 0.5, 'audio_dropout': 0.5, 'modality_fuse': 'concat', 'selection_type': 'same_seq', 'masking_type': 'input', 'mask_prob_image': 0.3, 'mask_length_image': 5, 'mask_prob_audio': 0.8, 'mask_length_audio': 10, 'extractor_mode': 'default', 'final_dim': 256, 'encoder_layerdrop': 0.05, 'dropout_input': 0.1, 'dropout_features': 0.1, 'dropout': 0.1, 'attention_dropout': 0.1, 'feature_grad_mult': 0.1, 'untie_final_proj': True, 'activation_dropout': 0.0, 'layer_norm_first': True, 'audio_feat_dim': 104, 'encoder_layers': 24, 'encoder_embed_dim': 1024, 'encoder_ffn_embed_dim': 4096, 'encoder_attention_heads': 16}, 'task': {'_name': 'vat_hubert_pretraining', 'data': '/LocalData/vatlm_related/fbankdata/fbank_lrs3_vox_tsv', 'label_dir': '/LocalData/vatlm_related/fbankdata/fbank_lrs3_vox_tsv', 'labels': ['km'], 'label_rate': 25, 'sample_rate': 25, 'max_sample_size': 500, 'min_sample_size': 5, 'pad_audio': True, 'random_crop': False, 'normalize': True, 'stack_order_audio': 4, 'image_aug': True, 'sup_data_path': '/LocalData/vatlm_related/fbankdata/fbank_tedv3_phone_concat_vox_tsv', 'sup_manifest': '/LocalData/vatlm_related/fbankdata/fbank_tedv3_phone_concat_vox_tsv', 'onlytext_manifest': '/LocalData/vatlm_related/fbankdata/cantab2_vox_tsv', 'onlyaudio_manifest': '/LocalData/vatlm_related/fbankdata/fbank_giga_vox_tsv_km', 'sample_distributions': '0.13,0.15,0.32,0.3'}, 'criterion': {'_name': 'vat_hubert', 'pred_masked_weight': 1.0, 'pred_nomask_weight': 0.0, 'loss_weights': [10], 'banlance_loss_weights': [1.0, 1.0]}, 'optimizer': {'_name': 'adam', 'adam_betas': '(0.9,0.98)', 'adam_eps': 1e-06, 'weight_decay': 0.01, 'use_old_adam': False, 'tpu': False, 'lr': [0.002]}, 'lr_scheduler': {'_name': 'polynomial_decay', 'warmup_updates': 48000, 'force_anneal': None, 'end_learning_rate': 0.0, 'power': 1.0, 'total_num_update': 600000, 'lr': [0.002]}, 'scoring': None, 'bpe': None, 'tokenizer': None, 'job_logging_cfg': {'version': 1, 'formatters': {'simple': {'format': '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'}}, 'handlers': {'console': {'class': 'logging.StreamHandler', 'formatter': 'simple', 'stream': 'ext://sys.stdout'}, 'file': {'class': 'logging.FileHandler', 'formatter': 'simple', 'filename': 'hydra_train.log'}}, 'root': {'level': 'INFO', 'handlers': ['console', 'file']}, 'disable_existing_loggers': False}}
'fp16': True, I hope this can help you.
xiabingquan
Hi, there. Recently, I'm working on extracting features using the model "VatLM Large VoxCeleb2 + LRS3 + paired audio+text+audio LRS-433h visual" From the config file https://github.com/microsoft/SpeechT5/blob/a7b72c6932e82930eb01331c2c922a0687157367/VATLM/vat_hubert/vathubert/conf/pretrain/large_vox_iter5.yaml#L4, fp16 is enabled, and it's not overwritten in the launching script https://github.com/microsoft/SpeechT5/blob/a7b72c6932e82930eb01331c2c922a0687157367/VATLM/vat_hubert/vathubert/scripts/pretrain/large_vox_pretrain_iter5.sh#L9-L30
However, when I'm loading the pretrained model, telling from the log, fp16 is disabled(The bottom line of the full traceback).
From my personal experience, the inconsistent settings of fp16 during linear probing and pre-training often leads to degraded performance, so I want to know whether fp16 is enabled or not in pretraining. Thanks!