INFO:zerot:{'train': {'log_interval': 50, 'eval_interval': 100, 'seed': 1234, 'epochs': 20000, 'learning_rate': 0.0002, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 32, 'fp16_run': True, 'lr_decay': 0.999875, 'segment_size': 8192, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0}, 'data': {'training_files': 'filelists/train.txt.cleaned', 'validation_files': 'filelists/val.txt.cleaned', 'text_cleaners': ['japanese_phrase_cleaners'], 'max_wav_value': 32768.0, 'sampling_rate': 22050, 'filter_length': 1024, 'hop_length': 256, 'win_length': 1024, 'n_mel_channels': 80, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 0, 'cleaned_text': True}, 'model': {'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 4, 4], 'n_layers_q': 3, 'use_spectral_norm': False}, 'model_dir': './logs/zerot'}
WARNING:zerot:/content is not a git repository, therefore hash value comparison will be ignored.
DEBUG:h5py._conv:Creating converter from 7 to 5
DEBUG:h5py._conv:Creating converter from 5 to 7
DEBUG:h5py._conv:Creating converter from 7 to 5
DEBUG:h5py._conv:Creating converter from 5 to 7
INFO:numexpr.utils:NumExpr defaulting to 2 threads.
./logs/zerot/G_0.pth
INFO:zerot:Loaded checkpoint './logs/zerot/G_0.pth' (iteration 1)
./logs/zerot/D_0.pth
INFO:zerot:Loaded checkpoint './logs/zerot/D_0.pth' (iteration 1)
INFO:zerot:Train Epoch: 1 [0%]
INFO:zerot:[6.072409152984619, 6.071518421173096, 0.25956135988235474, 96.23096466064453, 2.0617756843566895, 141.94737243652344, 0, 0.0002]
INFO:zerot:Saving model and optimizer state at iteration 1 to ./logs/zerot/G_0.pth
INFO:zerot:Saving model and optimizer state at iteration 1 to ./logs/zerot/D_0.pth
Traceback (most recent call last):
File "train.py", line 290, in
main()
File "train.py", line 50, in main
mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 158, in start_processes
while not context.join():
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 119, in join
raise Exception(msg)
Exception:
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, args)
File "/content/train.py", line 117, in run
train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval])
File "/content/train.py", line 144, in train_and_evaluate
(z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths)
File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(input, kwargs)
File "/usr/local/lib/python3.7/dist-packages/torch/nn/parallel/distributed.py", line 511, in forward
output = self.module(*inputs[0], *kwargs[0])
File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
result = self.forward(input, kwargs)
File "/content/models.py", line 495, in forward
z_slice, ids_slice = commons.rand_slice_segments(z, y_lengths, self.segment_size)
File "/content/commons.py", line 63, in rand_slice_segments
ret = slice_segments(x, ids_str, segment_size)
File "/content/commons.py", line 53, in slice_segments
ret[i] = x[i, :, idx_str:idx_end]
RuntimeError: The expanded size of the tensor (32) must match the existing size (0) at non-singleton dimension 1. Target sizes: [192, 32]. Tensor sizes: [192, 0]
QuellaMC
commented
2 years ago
INFO:zerot:{'train': {'log_interval': 50, 'eval_interval': 100, 'seed': 1234, 'epochs': 20000, 'learning_rate': 0.0002, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 32, 'fp16_run': True, 'lr_decay': 0.999875, 'segment_size': 8192, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0}, 'data': {'training_files': 'filelists/train.txt.cleaned', 'validation_files': 'filelists/val.txt.cleaned', 'text_cleaners': ['japanese_phrase_cleaners'], 'max_wav_value': 32768.0, 'sampling_rate': 22050, 'filter_length': 1024, 'hop_length': 256, 'win_length': 1024, 'n_mel_channels': 80, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 0, 'cleaned_text': True}, 'model': {'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 4, 4], 'n_layers_q': 3, 'use_spectral_norm': False}, 'model_dir': './logs/zerot'} WARNING:zerot:/content is not a git repository, therefore hash value comparison will be ignored. DEBUG:h5py._conv:Creating converter from 7 to 5 DEBUG:h5py._conv:Creating converter from 5 to 7 DEBUG:h5py._conv:Creating converter from 7 to 5 DEBUG:h5py._conv:Creating converter from 5 to 7 INFO:numexpr.utils:NumExpr defaulting to 2 threads. ./logs/zerot/G_0.pth INFO:zerot:Loaded checkpoint './logs/zerot/G_0.pth' (iteration 1) ./logs/zerot/D_0.pth INFO:zerot:Loaded checkpoint './logs/zerot/D_0.pth' (iteration 1) INFO:zerot:Train Epoch: 1 [0%] INFO:zerot:[6.072409152984619, 6.071518421173096, 0.25956135988235474, 96.23096466064453, 2.0617756843566895, 141.94737243652344, 0, 0.0002] INFO:zerot:Saving model and optimizer state at iteration 1 to ./logs/zerot/G_0.pth INFO:zerot:Saving model and optimizer state at iteration 1 to ./logs/zerot/D_0.pth Traceback (most recent call last): File "train.py", line 290, in
main()
File "train.py", line 50, in main
mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 158, in start_processes
while not context.join():
File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 119, in join
raise Exception(msg)
Exception:
-- Process 0 terminated with the following error: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/torch/multiprocessing/spawn.py", line 20, in _wrap fn(i, args) File "/content/train.py", line 117, in run train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval]) File "/content/train.py", line 144, in train_and_evaluate (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths) File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl result = self.forward(input, kwargs) File "/usr/local/lib/python3.7/dist-packages/torch/nn/parallel/distributed.py", line 511, in forward output = self.module(*inputs[0], *kwargs[0]) File "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl result = self.forward(input, kwargs) File "/content/models.py", line 495, in forward z_slice, ids_slice = commons.rand_slice_segments(z, y_lengths, self.segment_size) File "/content/commons.py", line 63, in rand_slice_segments ret = slice_segments(x, ids_str, segment_size) File "/content/commons.py", line 53, in slice_segments ret[i] = x[i, :, idx_str:idx_end] RuntimeError: The expanded size of the tensor (32) must match the existing size (0) at non-singleton dimension 1. Target sizes: [192, 32]. Tensor sizes: [192, 0]