Closed azulika closed 6 months ago
Related: https://github.com/litagin02/Style-Bert-VITS2/issues/99
Epoch 35(37%)/1000: 3%|█▊ | 5371/156008 [06:36<42:09:44, 1.01s/it]E0517 18:13:01.365890 20956 _internal.py:97] Error on request: Traceback (most recent call last): File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\serving.py", line 363, in run_wsgi execute(self.server.app) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\serving.py", line 324, in execute application_iter = app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 528, in __call__ return self._app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 569, in wrapper return wsgi_app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\security_validator.py", line 91, in __call__ return self._application(environ, start_response_proxy) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\path_prefix.py", line 68, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\experiment_id.py", line 73, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\empty_path_redirect.py", line 43, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\client_feature_flags.py", line 55, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\auth_context_middleware.py", line 38, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 551, in _route_request return self.exact_routes[clean_path](environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\wrappers\request.py", line 190, in application resp = f(*args[:-2] + (request,)) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\plugins\hparams\hparams_plugin.py", line 121, in get_experiment_route json_format.MessageToJson( TypeError: MessageToJson() got an unexpected keyword argument 'including_default_value_fields' Epoch 35(38%)/1000: 3%|█▊ | 5372/156008 [06:37<42:14:32, 1.01s/it]E0517 18:13:02.047425 20744 _internal.py:97] Error on request: Traceback (most recent call last): File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\serving.py", line 363, in run_wsgi execute(self.server.app) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\serving.py", line 324, in execute application_iter = app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 528, in __call__ return self._app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 569, in wrapper return wsgi_app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\security_validator.py", line 91, in __call__ return self._application(environ, start_response_proxy) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\path_prefix.py", line 68, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\experiment_id.py", line 73, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\empty_path_redirect.py", line 43, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\client_feature_flags.py", line 55, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\auth_context_middleware.py", line 38, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 551, in _route_request return self.exact_routes[clean_path](environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\wrappers\request.py", line 190, in application resp = f(*args[:-2] + (request,)) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\plugins\hparams\hparams_plugin.py", line 121, in get_experiment_route json_format.MessageToJson( TypeError: MessageToJson() got an unexpected keyword argument 'including_default_value_fields' Epoch 35(39%)/1000: 3%|█▊ | 5374/156008 [06:39<42:23:34, 1.01s/it]E0517 18:13:03.587944 24032 _internal.py:97] Error on request: Traceback (most recent call last): File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\serving.py", line 363, in run_wsgi execute(self.server.app) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\serving.py", line 324, in execute application_iter = app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 528, in __call__ return self._app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 569, in wrapper return wsgi_app(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\security_validator.py", line 91, in __call__ return self._application(environ, start_response_proxy) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\path_prefix.py", line 68, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\experiment_id.py", line 73, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\empty_path_redirect.py", line 43, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\client_feature_flags.py", line 55, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\auth_context_middleware.py", line 38, in __call__ return self._application(environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\backend\application.py", line 551, in _route_request return self.exact_routes[clean_path](environ, start_response) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\werkzeug\wrappers\request.py", line 190, in application resp = f(*args[:-2] + (request,)) File "E:\AI\sbv2\sbv2\Style-Bert-VITS2\venv\lib\site-packages\tensorboard\plugins\hparams\hparams_plugin.py", line 121, in get_experiment_route json_format.MessageToJson( TypeError: MessageToJson() got an unexpected keyword argument 'including_default_value_fields' Epoch 38(39%)/1000: 4%|█▉ | 5842/156008 [14:36<42:41:28, 1.02s/it]
学習は止まりませんが、途中途中のステップで定期的にこのメッセージが出ます。 今回は特定のモデルを5000ステップ目まで学習したのち、一度ターミナルを落として、その後設定を一切変更せずすぐ5000ステップ目から学習を再開したときに起きました。 今までにも何回か同様の現象が起きていて、途中ステップから再開する場合によく起きる気がします。
追記: どうやらTensorboardを開いたタイミングで上記のエラーが発生するようです。おそらくTensorboardの描画処理に関する問題なので、学習が止まらない事を見てもやはり学習自体に影響はないと思われます。
こちらでも確認しており、Tensorboardと相性が悪い別ライブラリのバージョンの問題でした。 次verで修正予定です。
Ver 2.5.0で対応しました。
Related: https://github.com/litagin02/Style-Bert-VITS2/issues/99
学習は止まりませんが、途中途中のステップで定期的にこのメッセージが出ます。 今回は特定のモデルを5000ステップ目まで学習したのち、一度ターミナルを落として、その後設定を一切変更せずすぐ5000ステップ目から学習を再開したときに起きました。 今までにも何回か同様の現象が起きていて、途中ステップから再開する場合によく起きる気がします。