[Bug] Specto Image and Audio not showing on Tensorboard

[brianadit24]

🐛 Description

I'm running training based on the recipe provided in the repository, but I'm having trouble showing the sample spectogram image generated during training and the sample audio on the tensorboard.

Log Error

E0321 12:11:56.323641 140609679509248 _internal.py:225] Error on request:
Traceback (most recent call last):
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/werkzeug/serving.py", line 324, in run_wsgi
    execute(self.server.app)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/werkzeug/serving.py", line 313, in execute
    application_iter = app(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/application.py", line 563, in __call__
    return self._app(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/application.py", line 155, in wrapper
    return wsgi_app(*args)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/security_validator.py", line 82, in __call__
    return self._application(environ, start_response_proxy)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/path_prefix.py", line 71, in __call__
    return self._application(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/experiment_id.py", line 76, in __call__
    return self._application(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/empty_path_redirect.py", line 47, in __call__
    return self._application(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/application.py", line 586, in _route_request
    return self.exact_routes[clean_path](environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/werkzeug/wrappers/request.py", line 206, in application
    resp = f(*args[:-2] + (request,))
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/plugins/image/images_plugin.py", line 165, in _serve_image_metadata
    experiment, run, tag, sample
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/plugins/image/images_plugin.py", line 197, in _image_response_for_run
    run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 302, in read_blob_sequences
    experiment_id, plugin_name, run, tag, event
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 417, in _convert_blob_sequence_event
    for idx in range(num_blobs)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 417, in <genexpr>
    for idx in range(num_blobs)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 364, in _encode_blob_key
    return six.ensure_str(encoded).rstrip("=")
AttributeError: module 'six' has no attribute 'ensure_str'

To Reproduce

Training FastPitch using this config:

{
    "model": "fast_pitch",
    "run_name": "fast_pitch_ljspeech",
    "run_description": "",
    "epochs": 1000,
    "batch_size": 32,
    "eval_batch_size": 16,
    "mixed_precision": false,
    "scheduler_after_epoch": false,
    "run_eval": true,
    "test_delay_epochs": -1,
    "print_eval": true,
    "dashboard_logger": "tensorboard",
    "print_step": 50,
    "plot_step": 100,
    "model_param_stats": false,
    "project_name": null,
    "log_model_step": null,
    "wandb_entity": null,
    "save_step": 10000,
    "checkpoint": true,
    "keep_all_best": false,
    "keep_after": 10000,
    "num_loader_workers": 8,
    "num_eval_loader_workers": 4,
    "use_noise_augment": false,
    "use_language_weighted_sampler": false,
    "output_path": "/raid/speech/msib/tts/coqui-ai/Trisa_Specto_FastPitch",
    "distributed_backend": "nccl",
    "distributed_url": "tcp://localhost:54321",
    "use_phonemes": false,
    "use_espeak_phonemes": false,
    "phoneme_language": null,
    "compute_input_seq_cache": true,
    "text_cleaner": "empty_cleaner",
    "enable_eos_bos_chars": false,
    "test_sentences_file": "",
    "phoneme_cache_path": null,
    "characters": {
        "pad": "_",
        "eos": ";",
        "bos": "^",
        "characters": "Eabcdefghijklmnoprstuwyz!',.?~ ",
        "punctuations": "!',.?~ ",
        "phonemes": "iy\u0268\u0289\u026fu\u026a\u028f\u028ae\u00f8\u0258\u0259\u0275\u0264o\u025b\u0153\u025c\u025e\u028c\u0254\u00e6\u0250a\u0276\u0251\u0252\u1d7b\u0298\u0253\u01c0\u0257\u01c3\u0284\u01c2\u0260\u01c1\u029bpbtd\u0288\u0256c\u025fk\u0261q\u0262\u0294\u0274\u014b\u0272\u0273n\u0271m\u0299r\u0280\u2c71\u027e\u027d\u0278\u03b2fv\u03b8\u00f0sz\u0283\u0292\u0282\u0290\u00e7\u029dx\u0263\u03c7\u0281\u0127\u0295h\u0266\u026c\u026e\u028b\u0279\u027bj\u0270l\u026d\u028e\u029f\u02c8\u02cc\u02d0\u02d1\u028dw\u0265\u029c\u02a2\u02a1\u0255\u0291\u027a\u0267\u02b2\u025a\u02de\u026b",
        "unique": true
    },
    "batch_group_size": 0,
    "loss_masking": null,
    "sort_by_audio_len": true,
    "min_seq_len": 13,
    "max_seq_len": 500000,
    "compute_f0": true,
    "compute_linear_spec": false,
    "add_blank": false,
    "datasets": [
        {
            "name": "ljspeech",
            "path": "/raid/speech/msib/tts/coqui-ai/Trisa_Specto_FastPitch/../Trisa_48K/",
            "meta_file_train": "metadata.csv",
            "ignored_speakers": null,
            "language": "",
            "meta_file_val": "",
            "meta_file_attn_mask": ""
        }
    ],
    "optimizer": "Adam",
    "optimizer_params": {
        "betas": [
            0.9,
            0.998
        ],
        "weight_decay": 1e-06
    },
    "lr_scheduler": "NoamLR",
    "lr_scheduler_params": {
        "warmup_steps": 4000
    },
    "test_sentences": [
        "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.",
        "Be a voice, not an echo.",
        "I'm sorry Dave. I'm afraid I can't do that.",
        "This cake is great. It's so delicious and moist.",
        "Prior to November 22, 1963."
    ],
    "base_model": "forward_tts",
    "model_args": {
        "num_chars": 34,
        "out_channels": 80,
        "hidden_channels": 384,
        "use_aligner": true,
        "use_pitch": true,
        "pitch_predictor_hidden_channels": 256,
        "pitch_predictor_kernel_size": 3,
        "pitch_predictor_dropout_p": 0.1,
        "pitch_embedding_kernel_size": 3,
        "duration_predictor_hidden_channels": 256,
        "duration_predictor_kernel_size": 3,
        "duration_predictor_dropout_p": 0.1,
        "positional_encoding": true,
        "poisitonal_encoding_use_scale": true,
        "length_scale": 1,
        "encoder_type": "fftransformer",
        "encoder_params": {
            "hidden_channels_ffn": 1024,
            "num_heads": 1,
            "num_layers": 6,
            "dropout_p": 0.1
        },
        "decoder_type": "fftransformer",
        "decoder_params": {
            "hidden_channels_ffn": 1024,
            "num_heads": 1,
            "num_layers": 6,
            "dropout_p": 0.1
        },
        "detach_duration_predictor": false,
        "max_duration": 75,
        "num_speakers": 1,
        "use_speaker_embedding": false,
        "speakers_file": null,
        "use_d_vector_file": false,
        "d_vector_dim": null,
        "d_vector_file": null
    },
    "num_speakers": 0,
    "speakers_file": null,
    "use_speaker_embedding": false,
    "use_d_vector_file": false,
    "d_vector_file": false,
    "d_vector_dim": 0,
    "lr": 0.0001,
    "grad_clip": 5.0,
    "spec_loss_type": "mse",
    "duration_loss_type": "mse",
    "use_ssim_loss": true,
    "ssim_loss_alpha": 1.0,
    "dur_loss_alpha": 1.0,
    "spec_loss_alpha": 1.0,
    "pitch_loss_alpha": 1.0,
    "aligner_loss_alpha": 1.0,
    "binary_align_loss_alpha": 1.0,
    "binary_align_loss_start_step": 20000,
    "r": 1,
    "f0_cache_path": "/raid/speech/msib/tts/coqui-ai/Trisa_Specto_FastPitch/f0_cache"
}

Expected behavior

Image and Audio can appear on tensorboard.

Environment

• 🐸TTS Version: 0.5.0
• PyTorch Version: 1.10.2
• Python version: 3.8.5
• OS: Linux
• CUDA/cuDNN version: 11.3
• GPU models and configuration: Yes
• How you installed PyTorch (conda, pip, source): pip
• Any other relevant information: No

Additional context

[erogol]

how do you run tensorboard?

[brianadit24]

how do you run tensorboard?

@erogol it's just like this:

tensorboard --logdir fast_pitch_ljspeech-March-08-2022_12+11PM-c63bb481/

[brianadit24]

how do you run tensorboard?

@erogol it's just like this:

tensorboard --logdir fast_pitch_ljspeech-March-08-2022_12+11PM-c63bb481/

@erogol but error appears after opening the Image and Audio tab on tensorboard dashboard.

Error Log:

E0321 12:11:56.323641 140609679509248 _internal.py:225] Error on request:
Traceback (most recent call last):
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/werkzeug/serving.py", line 324, in run_wsgi
    execute(self.server.app)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/werkzeug/serving.py", line 313, in execute
    application_iter = app(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/application.py", line 563, in __call__
    return self._app(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/application.py", line 155, in wrapper
    return wsgi_app(*args)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/security_validator.py", line 82, in __call__
    return self._application(environ, start_response_proxy)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/path_prefix.py", line 71, in __call__
    return self._application(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/experiment_id.py", line 76, in __call__
    return self._application(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/empty_path_redirect.py", line 47, in __call__
    return self._application(environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/application.py", line 586, in _route_request
    return self.exact_routes[clean_path](environ, start_response)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/werkzeug/wrappers/request.py", line 206, in application
    resp = f(*args[:-2] + (request,))
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/plugins/image/images_plugin.py", line 165, in _serve_image_metadata
    experiment, run, tag, sample
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/plugins/image/images_plugin.py", line 197, in _image_response_for_run
    run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 302, in read_blob_sequences
    experiment_id, plugin_name, run, tag, event
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 417, in _convert_blob_sequence_event
    for idx in range(num_blobs)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 417, in <genexpr>
    for idx in range(num_blobs)
  File "/home/brian.hermansyah/.local/lib/python3.6/site-packages/tensorboard/backend/event_processing/data_provider.py", line 364, in _encode_blob_key
    return six.ensure_str(encoded).rstrip("=")
AttributeError: module 'six' has no attribute 'ensure_str'

[erogol]

Sorry but no idea. I am able to use TB without issues and cannot really reproduce your issue.

It is maybe better to upstream it to the TB repo as it is not really relevant with 🐸TTS.

[lexkoro]

AttributeError: module 'six' has no attribute 'ensure_str'

https://stackoverflow.com/questions/67063538/module-six-has-no-attribute-ensure-text

Try updating six: python3 -m pip install --upgrade six

[erogol]

I close this one as it is not related to 🐸TTS directly.

But feel free to continue the discussion.