Wav2vec2_s2s model fine_tuning error:TypeError: forward() missing 1 required positional argument: 'prev_output_tokens'

orangelulu commented 3 years ago

🐛 Bug

To Reproduce

Steps to reproduce the behavior (always include the command you ran):

Run cmd '#!/bin/bash env HYDRA_FULL_ERROR=1 \ CUDA_VISIBLE_DEVICES=0,1,2,3 \ python3 /opt/tiger/fairseq/fairseq_cli/hydra_train.py \ task.data=/mnt/bd/liulu-asr-data/mls_spanish \ model.w2v_path=/opt/tiger/fairseq/examples/wav2vec/outputs/xlsr_53_56k.pt \ dataset.train_subset='train_100h' dataset.valid_subset='valid' \ checkpoint.patience=-1 \ checkpoint.save_dir='spanish_100h_s2s' \ checkpoint.maximize_best_checkpoint_metric=False \ distributed_training.distributed_world_size=4 optimization.update_freq='[6]' \ --config-dir /opt/tiger/fairseq/examples/wav2vec/config/finetuning/ \ --config-name vox_100h \'
See error 2021-01-27 00:45:20 | INFO | fairseq.distributed_utils | distributed init (rank 1): tcp://localhost:15491 2021-01-27 00:45:20 | INFO | fairseq.distributed_utils | distributed init (rank 0): tcp://localhost:15491 2021-01-27 00:45:20 | INFO | fairseq.distributed_utils | distributed init (rank 3): tcp://localhost:15491 2021-01-27 00:45:20 | INFO | fairseq.distributed_utils | distributed init (rank 2): tcp://localhost:15491 2021-01-27 00:45:27 | INFO | fairseq.distributed_utils | initialized host n147-194-033 as rank 1 2021-01-27 00:45:27 | INFO | fairseq.distributed_utils | initialized host n147-194-033 as rank 0 2021-01-27 00:45:27 | INFO | fairseq.distributed_utils | initialized host n147-194-033 as rank 2 2021-01-27 00:45:27 | INFO | fairseq.distributed_utils | initialized host n147-194-033 as rank 3 2021-01-27 00:45:27 | INFO | fairseq_cli.train | {'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 200, 'log_format': 'json', 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': True, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': None, 'fp16_scale_tolerance': 0.0, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'user_dir': None, 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': True}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 4, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': 'tcp://localhost:15491', 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': False, 'ddp_backend': 'no_c10d', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': False, 'fast_stat_sync': False, 'broadcast_buffers': False, 'distributed_wrapper': 'DDP', 'slowmo_momentum': None, 'slowmo_algorithm': 'LocalSGD', 'localsgd_frequency': 3, 'nprocs_per_node': 4, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'tpu': False, 'distributed_num_procs': 4}, 'dataset': {'_name': None, 'num_workers': 6, 'skip_invalid_size_inputs_valid_test': True, 'max_tokens': None, 'batch_size': 32, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train_100h', 'valid_subset': 'valid', 'validate_interval': 1, 'validate_interval_updates': 0, 'validate_after_updates': 0, 'fixed_validation_seed': None, 'disable_validation': False, 'max_tokens_valid': None, 'batch_size_valid': 32, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0}, 'optimization': {'_name': None, 'max_epoch': 0, 'max_update': 80000, 'stop_time_hours': 0.0, 'clip_norm': 0.0, 'sentence_avg': True, 'update_freq': [6], 'lr': [3e-05], 'stop_min_lr': -1.0, 'use_bmuf': False}, 'checkpoint': {'_name': None, 'save_dir': 'spanish_100h_s2s', 'restore_file': 'checkpoint_last.pt', 'finetune_from_model': None, 'reset_dataloader': False, 'reset_lr_scheduler': False, 'reset_meters': False, 'reset_optimizer': False, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 0, 'keep_interval_updates': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': -1, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'wer', 'maximize_best_checkpoint_metric': False, 'patience': -1, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'model_parallel_size': 1, 'distributed_rank': 0}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 4}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': False, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': {'_name': 'wav2vec_seq2seq', 'w2v_path': '/opt/tiger/fairseq/examples/wav2vec/outputs/xlsr_53_56k.pt', 'apply_mask': True, 'mask_prob': 0.5, 'mask_channel_prob': 0.5, 'mask_channel_length': 64, 'layerdrop': 0.1, 'activation_dropout': 0.1, 'feature_grad_mult': 0.0, 'freeze_finetune_updates': 10000}, 'task': {'_name': 'audio_pretraining', 'data': '/mnt/bd/liulu-asr-data/mls_spanish', 'normalize': True, 'labels': 'ltr'}, 'criterion': {'_name': 'cross_entropy'}, 'optimizer': {'_name': 'adam', 'adam_betas': '(0.9,0.98)', 'adam_eps': 1e-08}, 'lr_scheduler': {'_name': 'tri_stage', 'phase_ratio': [0.1, 0.4, 0.5], 'final_lr_scale': 0.05}, 'scoring': None, 'bpe': None, 'tokenizer': None} 2021-01-27 00:45:27 | INFO | fairseq.data.audio.raw_audio_dataset | loaded 2408, skipped 0 samples 2021-01-27 00:45:38 | INFO | fairseq_cli.train | task: AudioPretrainingTask 2021-01-27 00:45:38 | INFO | fairseq_cli.train | model: Wav2Vec2Seq2SeqModel 2021-01-27 00:45:38 | INFO | fairseq_cli.train | criterion: CrossEntropyCriterion 2021-01-27 00:45:38 | INFO | fairseq_cli.train | num. model params: 372207744 (num. trained: 372207744) 2021-01-27 00:45:39 | INFO | fairseq.utils | CUDA enviroments for all 4 workers 2021-01-27 00:45:39 | INFO | fairseq.utils | rank 0: capabilities = 7.0 ; total memory = 31.719 GB ; name = Tesla V100-SXM2-32GB
2021-01-27 00:45:39 | INFO | fairseq.utils | rank 1: capabilities = 7.0 ; total memory = 31.719 GB ; name = Tesla V100-SXM2-32GB
2021-01-27 00:45:39 | INFO | fairseq.utils | rank 2: capabilities = 7.0 ; total memory = 31.719 GB ; name = Tesla V100-SXM2-32GB
2021-01-27 00:45:39 | INFO | fairseq.utils | rank 3: capabilities = 7.0 ; total memory = 31.719 GB ; name = Tesla V100-SXM2-32GB
2021-01-27 00:45:39 | INFO | fairseq.utils | CUDA enviroments for all 4 workers 2021-01-27 00:45:39 | INFO | fairseq_cli.train | training on 4 devices (GPUs/TPUs) 2021-01-27 00:45:39 | INFO | fairseq_cli.train | max tokens per GPU = None and batch size per GPU = 32 2021-01-27 00:45:39 | INFO | fairseq.trainer | no existing checkpoint found spanish_100h_s2s/checkpoint_last.pt 2021-01-27 00:45:39 | INFO | fairseq.trainer | loading train data for epoch 1 2021-01-27 00:45:39 | INFO | fairseq.data.audio.raw_audio_dataset | loaded 24103, skipped 0 samples 2021-01-27 00:45:40 | INFO | fairseq.trainer | begin training epoch 1 Traceback (most recent call last): File "/opt/tiger/fairseq/fairseq_cli/hydra_train.py", line 70, in cli_main() File "/opt/tiger/fairseq/fairseq_cli/hydra_train.py", line 66, in cli_main hydra_main() File "/usr/local/lib/python3.7/dist-packages/hydra/main.py", line 37, in decorated_main strict=strict, File "/usr/local/lib/python3.7/dist-packages/hydra/_internal/utils.py", line 347, in _run_hydra lambda: hydra.run( File "/usr/local/lib/python3.7/dist-packages/hydra/_internal/utils.py", line 201, in run_and_report raise ex File "/usr/local/lib/python3.7/dist-packages/hydra/_internal/utils.py", line 198, in run_and_report return func() File "/usr/local/lib/python3.7/dist-packages/hydra/_internal/utils.py", line 350, in overrides=args.overrides, File "/usr/local/lib/python3.7/dist-packages/hydra/_internal/hydra.py", line 112, in run configure_logging=with_log_configuration, File "/usr/local/lib/python3.7/dist-packages/hydra/core/utils.py", line 125, in run_job ret.return_value = task_function(task_cfg) File "/opt/tiger/fairseq/fairseq_cli/hydra_train.py", line 38, in hydra_main distributed_utils.call_main(cfg, pre_main) File "/opt/tiger/fairseq/fairseq/distributed_utils.py", line 320, in call_main cfg.distributed_training.distributed_world_size, File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 199, in spawn return start_processes(fn, args, nprocs, join, daemon, start_method='spawn') File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 157, in start_processes while not context.join(): File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 118, in join raise Exception(msg) Exception:

-- Process 0 terminated with the following error: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 19, in _wrap fn(i, *args) File "/opt/tiger/fairseq/fairseq/distributed_utils.py", line 302, in distributed_main main(cfg, kwargs) File "/opt/tiger/fairseq/fairseq_cli/train.py", line 138, in main valid_losses, should_stop = train(cfg, trainer, task, epoch_itr) File "/usr/lib/python3.7/contextlib.py", line 74, in inner return func(*args, *kwds) File "/opt/tiger/fairseq/fairseq_cli/train.py", line 235, in train log_output = trainer.train_step(samples) File "/usr/lib/python3.7/contextlib.py", line 74, in inner return func(args, kwds) File "/opt/tiger/fairseq/fairseq/trainer.py", line 536, in train_step ignore_grad=is_dummy_batch, File "/opt/tiger/fairseq/fairseq/tasks/fairseq_task.py", line 428, in train_step loss, sample_size, logging_output = criterion(model, sample) File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl result = self.forward(input, kwargs) File "/opt/tiger/fairseq/fairseq/criterions/cross_entropy.py", line 35, in forward net_output = model(sample["net_input"]) File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl result = self.forward(input, kwargs) File "/opt/tiger/fairseq/fairseq/legacy_distributed_data_parallel.py", line 83, in forward return self.module(*inputs, *kwargs) File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl result = self.forward(input, kwargs) File "/opt/tiger/fairseq/fairseq/models/wav2vec/wav2vec2_asr.py", line 249, in forward decoder_out = self.decoder(encoder_out=encoder_out, *kwargs) File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl result = self.forward(input, **kwargs) TypeError: forward() missing 1 required positional argument: 'prev_output_tokens'

Code sample

Expected behavior

Environment

fairseq Version (e.g., 1.0 or master): master
PyTorch Version (e.g., 1.0) 1.0
OS (e.g., Linux):Linux
How you installed fairseq (pip, source):pip
Build command you used (if compiling from source): pip install -e .
Python version: 3.7
CUDA/cuDNN version: 11.0
GPU models and configuration: Tesla V100 8 GPU
Any other relevant information: no

Additional context

no

orangelulu commented 3 years ago

Hello, can anybody help me with this issue, since it looks like the forward function of decoder in wav2vec_asr.py is not right, the 'prev_output_tokens' need to be given in forward process while it does't exist. Moreover in the source code I don't find the shift right operation in the s2s decoder.


def forward(self, **kwargs):
         encoder_out = self.encoder(tbc=False, **kwargs)
         decoder_out = self.decoder(encoder_out=encoder_out, **kwargs)
         return decoder_out ```

AMOHYZ commented 3 years ago

i'm facing the same problem , can anyone help with that ?

alexeib commented 3 years ago

you need to set task.autoregressive=true for seq2seq models. i'll add a check for this

zxk19981227 commented 3 years ago

how to set task.autoregressive = true?

jubick1337 commented 3 years ago

@zxk19981227 fairseq-hydra-train model.w2v_path=/path/to/pretrained_model/ task.data=/path/to/data +task.autoregressive=true --config-dir /path/to/config/dir/ --config-name something.yaml

dangvansam commented 3 years ago

Hello, can anybody help me with this issue, since it looks like the forward function of decoder in wav2vec_asr.py is not right, the 'prev_output_tokens' need to be given in forward process while it does't exist. Moreover in the source code I don't find the shift right operation in the s2s decoder.
def forward(self, **kwargs):
         encoder_out = self.encoder(tbc=False, **kwargs)
         decoder_out = self.decoder(encoder_out=encoder_out, **kwargs)
         return decoder_out ```

same prolem when run inference model wav2vec_seq2seq

Binteislam commented 2 years ago

I am a beginner and facing this issue, can anyone guide me to resolve it?

effendijohanes commented 2 years ago

hi guys, got the same problem today, has anyone found the solution for this?

theamato commented 1 year ago

Did anyone figure out how to fix this? Facing this issue now and I'm completely stuck..

dangvansam commented 1 year ago

Hello, can anybody help me with this issue, since it looks like the forward function of decoder in wav2vec_asr.py is not right, the 'prev_output_tokens' need to be given in forward process while it does't exist. Moreover in the source code I don't find the shift right operation in the s2s decoder.
def forward(self, **kwargs):
         encoder_out = self.encoder(tbc=False, **kwargs)
         decoder_out = self.decoder(encoder_out=encoder_out, **kwargs)
         return decoder_out ```
same prolem when run inference model wav2vec_seq2seq

use the previous token from the encoder in timestep t-1 as input for the decoder in timestep t:

    def get_emissions_transformer_decoder(self, models, encoder_input):
        """Run encoder and normalize emissions"""
        model = models[0]
        encoder_out = model.encoder(**encoder_input)
        batch_size = encoder_out["encoder_out"].size(1)
        assert batch_size == 1
        hyps = torch.ones([1, 1], dtype=torch.long, device=encoder_out["encoder_out"].device).fill_(self.tgt_dict.eos())
        final_logprobs_output = torch.empty([1, 1, len(self.tgt_dict)], device=encoder_out["encoder_out"].device)
        for i in range(1, encoder_out["encoder_out"].size(0)):
            decoder_out, _ = model.decoder(hyps, encoder_out)
            if hasattr(model, "get_logits"):
                emissions = model.get_logits(decoder_out) # no need to normalize emissions
            else:
                emissions = model.get_normalized_probs(decoder_out, log_probs=True)
            emissions = emissions.float().contiguous()
            last_prob = emissions[-1,:].unsqueeze(0).unsqueeze(0)
            final_logprobs_output = torch.cat([final_logprobs_output, last_prob], 1)
            tokens = last_prob.argmax(dim=-1)
            hyps = torch.cat([hyps, tokens], 1)
            if torch.eq(hyps[:, -1], self.tgt_dict.eos()).view(-1, 1):
                break
        return final_logprobs_output.float().cpu().contiguous()

facebookresearch / fairseq