Problem when running training with num_vectors_per_token>1

[slawomir-gilewski-wttech]

Hi! After modifying the config yaml file by increasing the num_vectors_per_token parameter, when I try to run the training I get an error:

Traceback (most recent call last): File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/main.py", line 596, in trainer.fit(model, data) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 771, in fit self._call_and_handle_interrupt( File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 724, in _call_and_handle_interrupt return trainer_fn(*args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 812, in _fit_impl results = self._run(model, ckpt_path=self.ckpt_path) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1237, in _run results = self._run_stage() File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1324, in _run_stage return self._run_train() File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1354, in _run_train self.fit_loop.run() File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/base.py", line 204, in run self.advance(*args, *kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 269, in advance self._outputs = self.epoch_loop.run(self._data_fetcher) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/base.py", line 204, in run self.advance(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 208, in advance batch_output = self.batch_loop.run(batch, batch_idx) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/base.py", line 204, in run self.advance(*args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance outputs = self.optimizer_loop.run(split_batch, optimizers, batch_idx) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/base.py", line 204, in run self.advance(*args, *kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 203, in advance result = self._run_optimization( File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 249, in _run_optimization closure() File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 148, in call self._result = self.closure(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 134, in closure step_output = self._step_fn() File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 427, in _training_step training_step_output = self.trainer._call_strategy_hook("training_step", step_kwargs.values()) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1766, in _call_strategy_hook output = fn(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 333, in training_step return self.model.training_step(*args, *kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/models/diffusion/ddpm.py", line 442, in training_step loss, loss_dict = self.shared_step(batch) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/perfusion/perfusion.py", line 125, in shared_step loss = self(x, c, mask=mask) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/perfusion/perfusion.py", line 129, in forward encoding = self.cond_stage_model.encode(c['c_crossattn'], embedding_manager=self.embedding_manager) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/modules/encoders/modules.py", line 285, in encode return self(text, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/modules/encoders/modules.py", line 280, in forward z = self.transformer(input_ids=tokens, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/modules/encoders/modules.py", line 259, in transformer_forward return self.text_model( File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(*args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/modules/encoders/modules.py", line 219, in text_encoder_forward hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids, File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(*args, *kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/modules/encoders/modules.py", line 143, in embedding_forward inputs_embeds = embedding_manager(input_ids, inputs_embeds) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "/home/azureuser/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/perfusion/embedding_manager.py", line 139, in forward [tokenized_text[row][:col], placeholder_token.repeat(num_vectors_for_token).to(device), AttributeError: 'int' object has no attribute 'repeat'

I figured this is probably because placeholder_token is supposed to be a tensor, therefore I fixed it by adding placeholder_token = torch.tensor(placeholder_token).to(device) line to perfusion/embedding_manager.py in the else clause after checking if self.max_vectors_per_token == 1 (line 116).

It seems to be working after that :)

[euminds]

Hi

can you meet the problem:

I tried the command

 python main.py \
    --name teddy \
    --base ./configs/perfusion_teddy.yaml \
    --basedir ./ckpt \
    -t True \
    --gpus 0,

but got the error below

Traceback (most recent call last): File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/main.py", line 461, in model = instantiate_from_config(config.model) File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/main.py", line 137, in instantiate_from_config return get_obj_from_str(config["target"])(config.get("params", dict())) File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/perfusion/perfusion.py", line 54, in init super().init(*args, kwargs) File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/models/diffusion/ddpm.py", line 565, in init self.instantiate_cond_stage(cond_stage_config) File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/models/diffusion/ddpm.py", line 640, in instantiate_cond_stage model = instantiate_from_config(config) File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/util.py", line 175, in instantiate_from_config return get_obj_from_str(config["target"])(config.get("params", dict()), *kwargs) File "/data_heat/rjt_project/Key-Locked-Rank-One-Editing-for-Text-to-Image-Personalization/ldm/modules/encoders/modules.py", line 120, in init self.tokenizer = CLIPTokenizer.from_pretrained(version) File "/home/user/miniconda3/envs/perfusion_v1/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1841, in from_pretrained return cls._from_pretrained( File "/home/user/miniconda3/envs/perfusion_v1/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2004, in _from_pretrained tokenizer = cls(init_inputs, init_kwargs) File "/home/user/miniconda3/envs/perfusion_v1/lib/python3.10/site-packages/transformers/models/clip/tokenization_clip.py", line 334, in init with open(vocab_file, encoding="utf-8") as vocab_handle: TypeError: expected str, bytes or os.PathLike object, not NoneType

Thanks