ghost
commented
1 year ago Similar here, Sizes of tensors must match except in dimension 1. Expected size 42 but got size 41 for tensor number 1 in the list. on main
Closed betterftr closed 9 months ago
ghost
commented
1 year ago Similar here, Sizes of tensors must match except in dimension 1. Expected size 42 but got size 41 for tensor number 1 in the list. on main
Malapris
commented
1 year ago Sizes of tensors must match except in dimension 1. Expected size 72 but got size 71 for tensor number 1 in the list.
And it's the same with the last update... :(
I reinstalled all kohya_ss 3 times with 3 different python versions... Please help
deepfriedheroin
commented
1 year ago Same thing on dev2 and main.
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /workspace/kohya_ss/./sdxl_train_network.py:174 in <module> │
│ │
│ 171 │ args = train_util.read_config_from_file(args, parser) │
│ 172 │ │
│ 173 │ trainer = SdxlNetworkTrainer() │
│ ❱ 174 │ trainer.train(args) │
│ 175 │
│ │
│ /workspace/kohya_ss/train_network.py:775 in train │
│ │
│ 772 │ │ │ │ │ │
│ 773 │ │ │ │ │ # Predict the noise residual │
│ 774 │ │ │ │ │ with accelerator.autocast(): │
│ ❱ 775 │ │ │ │ │ │ noise_pred = self.call_unet( │
│ 776 │ │ │ │ │ │ │ args, accelerator, unet, noisy_latents, timesteps, text_enco │
│ 777 │ │ │ │ │ │ ) │
│ 778 │
│ │
│ /workspace/kohya_ss/./sdxl_train_network.py:154 in call_unet │
│ │
│ 151 │ │ vector_embedding = torch.cat([pool2, embs], dim=1).to(weight_dtype) │
│ 152 │ │ text_embedding = torch.cat([encoder_hidden_states1, encoder_hidden_states2], dim │
│ 153 │ │ │
│ ❱ 154 │ │ noise_pred = unet(noisy_latents, timesteps, text_embedding, vector_embedding) │
│ 155 │ │ return noise_pred │
│ 156 │ │
│ 157 │ def sample_images(self, accelerator, args, epoch, global_step, device, vae, tokenize │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/torch/nn/modules/module.py:1501 in │
│ _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/accelerate/utils/operations.py:521 in │
│ forward │
│ │
│ 518 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 519 │ │
│ 520 │ def forward(*args, **kwargs): │
│ ❱ 521 │ │ return model_forward(*args, **kwargs) │
│ 522 │ │
│ 523 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 524 │ forward.__wrapped__ = model_forward │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/accelerate/utils/operations.py:509 in │
│ __call__ │
│ │
│ 506 │ │ update_wrapper(self, model_forward) │
│ 507 │ │
│ 508 │ def __call__(self, *args, **kwargs): │
│ ❱ 509 │ │ return convert_to_fp32(self.model_forward(*args, **kwargs)) │
│ 510 │ │
│ 511 │ def __getstate__(self): │
│ 512 │ │ raise pickle.PicklingError( │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/torch/amp/autocast_mode.py:14 in │
│ decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(*args, **kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ /workspace/kohya_ss/library/sdxl_original_unet.py:1080 in forward │
│ │
│ 1077 │ │ h = call_module(self.middle_block, h, emb, context) │
│ 1078 │ │ │
│ 1079 │ │ for module in self.output_blocks: │
│ ❱ 1080 │ │ │ h = torch.cat([h, hs.pop()], dim=1) │
│ 1081 │ │ │ h = call_module(module, h, emb, context) │
│ 1082 │ │ │
│ 1083 │ │ h = h.type(x.dtype) │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯steps: 0%| | 0/300 [00:03<?, ?it/s, loss=0.0245]
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /workspace/kohya_ss/venv/bin/accelerate:8 in <module> │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/accelerate/commands/accelerate_cli.py:45 │
│ in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if __name__ == "__main__": │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/accelerate/commands/launch.py:918 in │
│ launch_command │
│ │
│ 915 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA │
│ 916 │ │ sagemaker_launcher(defaults, args) │
│ 917 │ else: │
│ ❱ 918 │ │ simple_launcher(args) │
│ 919 │
│ 920 │
│ 921 def main(): │
│ │
│ /workspace/kohya_ss/venv/lib/python3.10/site-packages/accelerate/commands/launch.py:580 in │
│ simple_launcher │
│ │
│ 577 │ process.wait() │
│ 578 │ if process.returncode != 0: │
│ 579 │ │ if not args.quiet: │
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) │
│ 581 │ │ else: │
│ 582 │ │ │ sys.exit(1) │
│ 583 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯'--resolution=1024,1024', '--output_dir=/workspace/training/_upscaled/_totally_cat_pics_i_promise/out', '--logging_dir=/workspace/training/_upscaled/_totally_cat_pics_i_promise/log', '--network_alpha=16', '--training_comment=dfh woman', '--save_model_as=safetensors', '--network_module=lycoris.kohya', '--network_args', 'conv_dim=8', 'conv_alpha=4', 'use_cp=False', 'algo=loha',
'--network_dropout=0', '--text_encoder_lr=0.0001', '--unet_lr=0.0001', '--network_dim=32', '--gradient_accumulation_steps=10', '--output_name=_totally_cat_pics_i_promise', '--lr_scheduler_num_cycles=10', '--no_half_vae', '--learning_rate=0.0001', '--lr_scheduler=cosine', '--train_batch_size=4', '--max_train_steps=300', '--save_every_n_epochs=1', '--mixed_precision=bf16',
'--save_precision=bf16', '--seed=1234', '--cache_latents', '--cache_latents_to_disk', '--optimizer_type=AdamW', '--max_data_loader_n_workers=0', '--bucket_reso_steps=1', '--save_every_n_steps=25', '--min_snr_gamma=10', '--xformers', '--bucket_no_upscale', '--multires_noise_iterations=8', '--multires_noise_discount=0.2']' returned non-zero exit status 1.
CiaraStrawberry
commented
1 year ago Had the same issue, only reproduces with bucketing enabled, Tried modifying the file sizes to multiples of 8, didn't help, then pre downscaled and upscaled all the images to roughly the expected pixel count and multiples of 8 and it started working correctly.
VeMeth
commented
1 year ago Did anyone of you make any further progress? I've tried the images in several different resolutions. Sometimes it even trains 5-20 steps until it crashes out with the error above.
miabrahams
commented
9 months ago I got this error because I requested a width that was not a multiple of 8 when specifying --width.
trying to finetune sdxl, latest dev2 steps: 0%| | 1/289000 [00:21<1749:16:53, 21.79s/it, loss=0.0662]╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮ │ C:\kohya_ss\sdxl_train.py:639 in │
│ │
│ 636 │ args = parser.parse_args() │
│ 637 │ args = train_util.read_config_from_file(args, parser) │
│ 638 │ │
│ ❱ 639 │ train(args) │
│ 640 │
│ │
│ C:\kohya_ss\sdxl_train.py:449 in train │
│ │
│ 446 │ │ │ │ │
│ 447 │ │ │ │ # Predict the noise residual │
│ 448 │ │ │ │ with accelerator.autocast(): │
│ ❱ 449 │ │ │ │ │ noise_pred = unet(noisy_latents, timesteps, text_embedding, vector_e │
│ 450 │ │ │ │ │
│ 451 │ │ │ │ target = noise │
│ 452 │
│ │
│ C:\kohya_ss\venv\lib\site-packages\torch\nn\modules\module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ C:\kohya_ss\venv\lib\site-packages\accelerate\utils\operations.py:521 in forward │
│ │
│ 518 │ model_forward = ConvertOutputsToFp32(model_forward) │
│ 519 │ │
│ 520 │ def forward(*args, *kwargs): │
│ ❱ 521 │ │ return model_forward(args, kwargs) │
│ 522 │ │
│ 523 │ # To act like a decorator so that it can be popped when doing `extract_model_from_pa │
│ 524 │ forward.wrapped = model_forward │
│ │
│ C:\kohya_ss\venv\lib\site-packages\accelerate\utils\operations.py:509 in call │
│ │
│ 506 │ │ update_wrapper(self, model_forward) │
│ 507 │ │
│ 508 │ def call(self, *args, kwargs): │
│ ❱ 509 │ │ return convert_to_fp32(self.model_forward(*args, *kwargs)) │
│ 510 │ │
│ 511 │ def getstate(self): │
│ 512 │ │ raise pickle.PicklingError( │
│ │
│ C:\kohya_ss\venv\lib\site-packages\torch\amp\autocast_mode.py:14 in decorate_autocast │
│ │
│ 11 │ @functools.wraps(func) │
│ 12 │ def decorate_autocast(args, kwargs): │
│ 13 │ │ with autocast_instance: │
│ ❱ 14 │ │ │ return func(*args, **kwargs) │
│ 15 │ decorate_autocast.__script_unsupported = '@autocast() decorator is not supported in │
│ 16 │ return decorate_autocast │
│ 17 │
│ │
│ C:\kohya_ss\library\sdxl_original_unet.py:1080 in forward │
│ │
│ 1077 │ │ h = call_module(self.middle_block, h, emb, context) │
│ 1078 │ │ │
│ 1079 │ │ for module in self.output_blocks: │
│ ❱ 1080 │ │ │ h = torch.cat([h, hs.pop()], dim=1) │
│ 1081 │ │ │ h = call_module(module, h, emb, context) │
│ 1082 │ │ │
│ 1083 │ │ h = h.type(x.dtype) │
╰──────────────────────────────────────────────────────────────────────────────────Sizes of tensors must match except in dimension 1. Expected size 56 but got size 55 for tensor number 1 in
the list.