search

modelscope / scepter

SCEPTER is an open-source framework used for training, fine-tuning, and inference with generative models.
https://github.com/modelscope/scepter
Apache License 2.0
428 stars 22 forks source link

Running python scepter/tools/run_train.py --cfg scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml process, encountered the following error #47

Closed twinkleyang1 closed 1 month ago

twinkleyang1 commented 1 month ago

Thank you for your contribution, I encountered a new problem, running python scepter/tools/run_train.py --cfg scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml The error is reported as follows： 2024-10-06 16:14:48,751 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:14:48,752 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:14:48,786 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') [Info]: Loading config from scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml [Info]: System take scepter/methods/scedit/ctr/sd21_768_sce_ctr_canny.yaml as yaml, because we find yaml in this file [Info]: SEED is not set and will use default SEED as 2023; If want to change this value, please set it in your config. [Info]: Parse cfg file as { "ENV": { "BACKEND": "nccl", "SEED": 2023 }, "SOLVER": { "NAME": "LatentDiffusionSolver", "RESUME_FROM": null, "LOAD_MODEL_ONLY": true, "USE_FSDP": false, "SHARDING_STRATEGY": null, "USE_AMP": true, "DTYPE": "float16", "CHANNELS_LAST": true, "MAX_STEPS": 200, "MAX_EPOCHS": -1, "NUM_FOLDS": 1, "ACCU_STEP": 1, "EVAL_INTERVAL": 100, "RESCALE_LR": false, "WORK_DIR": "./cache/save_data/sd21_768_sce_ctr_canny", "LOG_FILE": "std_log.txt", "FILE_SYSTEM": { "NAME": "ModelscopeFs", "TEMP_DIR": "./cache/cache_data" }, "FREEZE": { "FREEZE_PART": [ "first_stage_model", "cond_stage_model", "model" ], "TRAIN_PART": [ "control_blocks" ] }, "MODEL": { "NAME": "LatentDiffusionSCEControl", "PARAMETERIZATION": "v", "TIMESTEPS": 1000, "MIN_SNR_GAMMA": null, "ZERO_TERMINAL_SNR": false, "PRETRAINED_MODEL": "ms://AI-ModelScope/stable-diffusion-2-1@v2-1_768-ema-pruned.safetensors", "IGNORE_KEYS": [], "SCALE_FACTOR": 0.18215, "SIZE_FACTOR": 8, "DEFAULT_N_PROMPT": null, "SCHEDULE_ARGS": { "NAME": "scaled_linear", "BETA_MIN": 0.00085, "BETA_MAX": 0.012 }, "USE_EMA": false, "DIFFUSION_MODEL": { "NAME": "DiffusionUNet", "IN_CHANNELS": 4, "OUT_CHANNELS": 4, "MODEL_CHANNELS": 320, "NUM_HEADS_CHANNELS": 64, "NUM_RES_BLOCKS": 2, "ATTENTION_RESOLUTIONS": [ 4, 2, 1 ], "CHANNEL_MULT": [ 1, 2, 4, 4 ], "CONV_RESAMPLE": true, "DIMS": 2, "USE_CHECKPOINT": false, "USE_SCALE_SHIFT_NORM": false, "RESBLOCK_UPDOWN": false, "USE_SPATIAL_TRANSFORMER": true, "TRANSFORMER_DEPTH": 1, "CONTEXT_DIM": 1024, "DISABLE_MIDDLE_SELF_ATTN": false, "USE_LINEAR_IN_TRANSFORMER": true, "PRETRAINED_MODEL": null, "IGNORE_KEYS": [] }, "FIRST_STAGE_MODEL": { "NAME": "AutoencoderKL", "EMBED_DIM": 4, "PRETRAINED_MODEL": null, "IGNORE_KEYS": [], "BATCH_SIZE": 4, "ENCODER": { "NAME": "Encoder", "CH": 128, "OUT_CH": 3, "NUM_RES_BLOCKS": 2, "IN_CHANNELS": 3, "ATTN_RESOLUTIONS": [], "CH_MULT": [ 1, 2, 4, 4 ], "Z_CHANNELS": 4, "DOUBLE_Z": true, "DROPOUT": 0.0, "RESAMP_WITH_CONV": true }, "DECODER": { "NAME": "Decoder", "CH": 128, "OUT_CH": 3, "NUM_RES_BLOCKS": 2, "IN_CHANNELS": 3, "ATTN_RESOLUTIONS": [], "CH_MULT": [ 1, 2, 4, 4 ], "Z_CHANNELS": 4, "DROPOUT": 0.0, "RESAMP_WITH_CONV": true, "GIVE_PRE_END": false, "TANH_OUT": false } }, "TOKENIZER": { "NAME": "OpenClipTokenizer", "LENGTH": 77 }, "COND_STAGE_MODEL": { "NAME": "FrozenOpenCLIPEmbedder", "ARCH": "ViT-H-14", "PRETRAINED_MODEL": null, "LAYER": "penultimate" }, "LOSS": { "NAME": "ReconstructLoss", "LOSS_TYPE": "l2" }, "CONTROL_MODEL": { "NAME": "CSCTuners", "PRE_HINT_IN_CHANNELS": 3, "PRE_HINT_OUT_CHANNELS": 256, "DENSE_HINT_KERNAL": 3, "SCALE": 1.0, "SC_TUNER_CFG": { "NAME": "SCTuner", "TUNER_NAME": "SCEAdapter", "DOWN_RATIO": 1.0 } }, "CONTROL_ANNO": { "NAME": "CannyAnnotator", "LOW_THRESHOLD": 100, "HIGH_THRESHOLD": 200 } }, "SAMPLE_ARGS": { "SAMPLER": "ddim", "SAMPLE_STEPS": 50, "SEED": 2023, "GUIDE_SCALE": 7.5, "GUIDE_RESCALE": 0.5, "DISCRETIZATION": "trailing", "IMAGE_SIZE": [ 768, 768 ], "RUN_TRAIN_N": false }, "OPTIMIZER": { "NAME": "AdamW", "LEARNING_RATE": 0.0001, "BETAS": [ 0.9, 0.999 ], "EPS": "1e-8", "WEIGHT_DECAY": "1e-2", "AMSGRAD": false }, "TRAIN_DATA": { "NAME": "ImageTextPairMSDataset", "MODE": "train", "MS_DATASET_NAME": "style_custom_dataset", "MS_DATASET_NAMESPACE": "damo", "MS_DATASET_SUBNAME": "3D", "PROMPT_PREFIX": "", "MS_DATASET_SPLIT": "train_short", "MS_REMAP_KEYS": { "Image:FILE": "Target:FILE" }, "REPLACE_STYLE": false, "PIN_MEMORY": true, "BATCH_SIZE": 1, "NUM_WORKERS": 4, "SAMPLER": { "NAME": "LoopSampler" }, "TRANSFORMS": [ { "NAME": "LoadImageFromFile", "RGB_ORDER": "RGB", "BACKEND": "pillow" }, { "NAME": "Resize", "SIZE": 768, "INTERPOLATION": "bilinear", "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "pillow" }, { "NAME": "CenterCrop", "SIZE": 768, "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "pillow" }, { "NAME": "ToNumpy", "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "image_preprocess" ] }, { "NAME": "ImageToTensor", "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "pillow" }, { "NAME": "Normalize", "MEAN": [ 0.5, 0.5, 0.5 ], "STD": [ 0.5, 0.5, 0.5 ], "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "torchvision" }, { "NAME": "Rename", "INPUT_KEY": [ "img", "image_preprocess" ], "OUTPUT_KEY": [ "image", "image_preprocess" ] }, { "NAME": "Select", "KEYS": [ "image", "prompt", "image_preprocess" ], "META_KEYS": [ "data_key" ] } ] }, "EVAL_DATA": { "NAME": "ImageTextPairMSDataset", "MODE": "eval", "MS_DATASET_NAME": "style_custom_dataset", "MS_DATASET_NAMESPACE": "damo", "MS_DATASET_SUBNAME": "3D", "PROMPT_PREFIX": "", "MS_DATASET_SPLIT": "train_short", "MS_REMAP_KEYS": { "Image:FILE": "Target:FILE" }, "REPLACE_STYLE": false, "PIN_MEMORY": true, "BATCH_SIZE": 10, "NUM_WORKERS": 4, "TRANSFORMS": [ { "NAME": "LoadImageFromFile", "RGB_ORDER": "RGB", "BACKEND": "pillow" }, { "NAME": "Resize", "SIZE": 768, "INTERPOLATION": "bilinear", "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "pillow" }, { "NAME": "CenterCrop", "SIZE": 768, "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "pillow" }, { "NAME": "ToNumpy", "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "image_preprocess" ] }, { "NAME": "ImageToTensor", "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "pillow" }, { "NAME": "Normalize", "MEAN": [ 0.5, 0.5, 0.5 ], "STD": [ 0.5, 0.5, 0.5 ], "INPUT_KEY": [ "img" ], "OUTPUT_KEY": [ "img" ], "BACKEND": "torchvision" }, { "NAME": "Rename", "INPUT_KEY": [ "img", "image_preprocess" ], "OUTPUT_KEY": [ "image", "image_preprocess" ] }, { "NAME": "Select", "KEYS": [ "image", "prompt", "image_preprocess" ], "META_KEYS": [ "data_key" ] } ] }, "TRAIN_HOOKS": [ { "NAME": "BackwardHook", "PRIORITY": 0 }, { "NAME": "LogHook", "LOG_INTERVAL": 50 }, { "NAME": "CheckpointHook", "INTERVAL": 100 }, { "NAME": "ProbeDataHook", "PROB_INTERVAL": 100 } ], "EVAL_HOOKS": [ { "NAME": "ProbeDataHook", "PROB_INTERVAL": 100 } ] } } 2024-10-06 16:14:55,056 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:14:55,058 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:14:55,093 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:14:55,322 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:14:55,322 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:14:55,323 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:14:55,323 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:14:55,358 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:14:55,358 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:14:55,446 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:14:55,447 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:14:55,483 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') [Info]: Now running in the distributed environment with world size 4! [Info]: PMI rank 0! [Info]: Nums of gpu 4! [Info]: Current rank 0 current devices num 4 current machine rank 0 and all world size 4 [Info]: Now running in the distributed environment with world size 4! [Info]: PMI rank 0! [Info]: Nums of gpu 4! [Info]: Current rank 3 current devices num 4 current machine rank 0 and all world size 4 [Info]: Now running in the distributed environment with world size 4! [Info]: PMI rank 0! [Info]: Nums of gpu 4! [Info]: Current rank 1 current devices num 4 current machine rank 0 and all world size 4 scepter [INFO] 2024-10-06 16:14:57,045 [File: base_solver.py Function: init at line 308] Parse work dir ./cache/save_data/sd21_768_sce_ctr_canny's prefix is /data/twinkle/app/scepter scepter [INFO] 2024-10-06 16:14:57,045 [File: diffusion_solver.py Function: init at line 113] Use default backend. scepter [INFO] 2024-10-06 16:14:57,045 [File: logger.py Function: init_logger at line 86] Running task with log file: /data/twinkle/app/scepter/./cache/save_data/sd21_768_sce_ctr_canny/std_log.txt [Info]: Now running in the distributed environment with world size 4! [Info]: PMI rank 0! [Info]: Nums of gpu 4! [Info]: Current rank 2 current devices num 4 current machine rank 0 and all world size 4 scepter [INFO] 2024-10-06 16:14:57,048 [File: base_solver.py Function: init at line 308] Parse work dir ./cache/save_data/sd21_768_sce_ctr_canny's prefix is /data/twinkle/app/scepter scepter [INFO] 2024-10-06 16:14:57,048 [File: diffusion_solver.py Function: init at line 113] Use default backend. scepter [INFO] 2024-10-06 16:14:57,049 [File: base_solver.py Function: init at line 308] Parse work dir ./cache/save_data/sd21_768_sce_ctr_canny's prefix is /data/twinkle/app/scepter scepter [INFO] 2024-10-06 16:14:57,049 [File: diffusion_solver.py Function: init at line 113] Use default backend. scepter [INFO] 2024-10-06 16:14:57,056 [File: base_solver.py Function: init at line 308] Parse work dir ./cache/save_data/sd21_768_sce_ctr_canny's prefix is /data/twinkle/app/scepter scepter [INFO] 2024-10-06 16:14:57,056 [File: diffusion_solver.py Function: init at line 113] Use default backend. 2024-10-06 16:14:57,455 - modelscope - INFO - dataset_type: 2 2024-10-06 16:14:57,457 - modelscope - INFO - dataset_type: 2 2024-10-06 16:14:57,465 - modelscope - INFO - dataset_type: 2 2024-10-06 16:14:57,468 - modelscope - INFO - dataset_type: 2 2024-10-06 16:14:58,401 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:58,401 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:58,401 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 2024-10-06 16:14:58,808 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:58,808 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:58,809 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 scepter [INFO] 2024-10-06 16:14:59,089 [File: registry.py Function: init at line 185] Built dataloader with len 9223372036854775807 2024-10-06 16:14:59,170 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:59,170 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:59,170 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 2024-10-06 16:14:59,418 - modelscope - INFO - dataset_type: 2 scepter [INFO] 2024-10-06 16:14:59,481 [File: registry.py Function: init at line 185] Built dataloader with len 9223372036854775807 2024-10-06 16:14:59,531 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:59,531 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:14:59,531 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 scepter [INFO] 2024-10-06 16:14:59,769 [File: registry.py Function: init at line 185] Built dataloader with len 9223372036854775807 2024-10-06 16:14:59,842 - modelscope - INFO - dataset_type: 2 2024-10-06 16:15:00,131 - modelscope - INFO - dataset_type: 2 2024-10-06 16:15:00,223 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:00,223 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:00,223 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 scepter [INFO] 2024-10-06 16:15:00,252 [File: registry.py Function: init at line 185] Built dataloader with len 9223372036854775807 2024-10-06 16:15:00,558 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:00,558 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:00,558 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 2024-10-06 16:15:00,577 - modelscope - INFO - dataset_type: 2 scepter [INFO] 2024-10-06 16:15:00,746 [File: registry.py Function: init at line 185] Built dataloader with len 1 2024-10-06 16:15:00,900 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:00,900 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:00,900 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 scepter [INFO] 2024-10-06 16:15:01,191 [File: registry.py Function: init at line 185] Built dataloader with len 1 2024-10-06 16:15:01,289 - modelscope - WARNING - Reusing dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:01,289 - modelscope - INFO - Generating dataset dataset_builder (/home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files) 2024-10-06 16:15:01,289 - modelscope - INFO - Reusing cached meta-data file: /home/twinkle/.cache/modelscope/hub/datasets/damo/style_custom_dataset/master/data_files/20773d60e56183e65c9ac537a8adcd83 scepter [INFO] 2024-10-06 16:15:01,442 [File: registry.py Function: init at line 185] Built dataloader with len 1 scepter [INFO] 2024-10-06 16:15:01,859 [File: registry.py Function: init at line 185] Built dataloader with len 1 scepter [INFO] 2024-10-06 16:15:07,724 [File: ldm.py Function: construct_network at line 147] all parameters:865.91M scepter [INFO] 2024-10-06 16:15:07,725 [File: ae_module.py Function: construct_model at line 76] AE Module XFORMERS_IS_AVAILBLE : True scepter [INFO] 2024-10-06 16:15:07,798 [File: ldm.py Function: construct_network at line 147] all parameters:865.91M scepter [INFO] 2024-10-06 16:15:07,799 [File: ae_module.py Function: construct_model at line 76] AE Module XFORMERS_IS_AVAILBLE : True scepter [INFO] 2024-10-06 16:15:08,336 [File: ldm.py Function: construct_network at line 147] all parameters:865.91M scepter [INFO] 2024-10-06 16:15:08,338 [File: ae_module.py Function: construct_model at line 76] AE Module XFORMERS_IS_AVAILBLE : True scepter [INFO] 2024-10-06 16:15:08,805 [File: ldm.py Function: construct_network at line 147] all parameters:865.91M scepter [INFO] 2024-10-06 16:15:08,806 [File: ae_module.py Function: construct_model at line 76] AE Module XFORMERS_IS_AVAILBLE : True 2024-10-06 16:15:19,405 - modelscope - WARNING - Model revision not specified, use revision: v1.0.0 2024-10-06 16:15:20,227 - modelscope - WARNING - Model revision not specified, use revision: v1.0.0 2024-10-06 16:15:20,660 - modelscope - WARNING - Model revision not specified, use revision: v1.0.0 2024-10-06 16:15:21,170 - modelscope - WARNING - Model revision not specified, use revision: v1.0.0 scepter [INFO] 2024-10-06 16:15:44,440 [File: ldm.py Function: init_from_ckpt at line 218] Restored from ./cache/cache_data/601abc72ccca722467955bb9c36da4b3/AI-ModelScope/stable-diffusion-2-1/v2-1_768-ema-pruned.safetensors with 86 missing and 14 unexpected keys scepter [INFO] 2024-10-06 16:15:44,441 [File: ldm.py Function: init_from_ckpt at line 222] Missing Keys: ['model.control_blocks.0.pre_hint_blocks.0.weight', 'model.control_blocks.0.pre_hint_blocks.0.bias', 'model.control_blocks.0.pre_hint_blocks.2.weight', 'model.control_blocks.0.pre_hint_blocks.2.bias', 'model.control_blocks.0.pre_hint_blocks.4.weight', 'model.control_blocks.0.pre_hint_blocks.4.bias', 'model.control_blocks.0.pre_hint_blocks.6.weight', 'model.control_blocks.0.pre_hint_blocks.6.bias', 'model.control_blocks.0.pre_hint_blocks.8.weight', 'model.control_blocks.0.pre_hint_blocks.8.bias', 'model.control_blocks.0.pre_hint_blocks.10.weight', 'model.control_blocks.0.pre_hint_blocks.10.bias', 'model.control_blocks.0.pre_hint_blocks.12.weight', 'model.control_blocks.0.pre_hint_blocks.12.bias', 'model.control_blocks.0.dense_hint_blocks.0.1.weight', 'model.control_blocks.0.dense_hint_blocks.0.1.bias', 'model.control_blocks.0.dense_hint_blocks.1.1.weight', 'model.control_blocks.0.dense_hint_blocks.1.1.bias', 'model.control_blocks.0.dense_hint_blocks.2.1.weight', 'model.control_blocks.0.dense_hint_blocks.2.1.bias', 'model.control_blocks.0.dense_hint_blocks.3.1.weight', 'model.control_blocks.0.dense_hint_blocks.3.1.bias', 'model.control_blocks.0.dense_hint_blocks.4.1.weight', 'model.control_blocks.0.dense_hint_blocks.4.1.bias', 'model.control_blocks.0.dense_hint_blocks.5.1.weight', 'model.control_blocks.0.dense_hint_blocks.5.1.bias', 'model.control_blocks.0.dense_hint_blocks.6.1.weight', 'model.control_blocks.0.dense_hint_blocks.6.1.bias', 'model.control_blocks.0.dense_hint_blocks.7.1.weight', 'model.control_blocks.0.dense_hint_blocks.7.1.bias', 'model.control_blocks.0.dense_hint_blocks.8.1.weight', 'model.control_blocks.0.dense_hint_blocks.8.1.bias', 'model.control_blocks.0.dense_hint_blocks.9.1.weight', 'model.control_blocks.0.dense_hint_blocks.9.1.bias', 'model.control_blocks.0.dense_hint_blocks.10.1.weight', 'model.control_blocks.0.dense_hint_blocks.10.1.bias', 'model.control_blocks.0.dense_hint_blocks.11.1.weight', 'model.control_blocks.0.dense_hint_blocks.11.1.bias', 'model.control_blocks.0.lsc_tuner_blocks.0.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.0.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.0.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.0.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.1.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.1.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.1.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.1.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.2.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.2.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.2.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.2.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.3.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.3.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.3.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.3.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.4.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.4.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.4.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.4.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.5.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.5.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.5.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.5.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.6.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.6.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.6.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.6.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.7.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.7.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.7.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.7.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.8.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.8.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.8.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.8.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.9.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.9.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.9.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.9.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.10.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.10.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.10.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.10.tuner_op.ln2.bias', 'model.control_blocks.0.lsc_tuner_blocks.11.tuner_op.ln1.weight', 'model.control_blocks.0.lsc_tuner_blocks.11.tuner_op.ln1.bias', 'model.control_blocks.0.lsc_tuner_blocks.11.tuner_op.ln2.weight', 'model.control_blocks.0.lsc_tuner_blocks.11.tuner_op.ln2.bias'] scepter [INFO] 2024-10-06 16:15:44,441 [File: ldm.py Function: init_from_ckpt at line 224]
Unexpected Keys: ['alphas_cumprod', 'alphas_cumprod_prev', 'betas', 'log_one_minus_alphas_cumprod', 'model_ema.decay', 'model_ema.num_updates', 'posterior_log_variance_clipped', 'posterior_mean_coef1', 'posterior_mean_coef2', 'posterior_variance', 'sqrt_alphas_cumprod', 'sqrt_one_minus_alphas_cumprod', 'sqrt_recip_alphas_cumprod', 'sqrt_recipm1_alphas_cumprod'] scepter [INFO] 2024-10-06 16:15:47,244 [File: diffusion_solver.py Function: print_model_params_status at line 745] Load trainable params 99111760 / 1402708988 = 7.07%, train part: {'control_blocks.0': 99111760}. scepter [INFO] 2024-10-06 16:15:47,245 [File: diffusion_solver.py Function: print_model_params_status at line 749] Load forzen params 1303597228 / 1402708988 = 92.93%, forzen part: {'time_embed': 2050560, 'input_blocks': 251918080, 'middle_block': 97693440, 'output_blocks': 514236480, 'out': 12164, 'encoder': 34163592, 'decoder': 49490179, 'conv1': 72, 'conv2': 20, 'model': 354032641}. scepter [INFO] 2024-10-06 16:15:47,582 [File: diffusion_solver.py Function: print_model_params_status at line 745] Load trainable params 99111760 / 1402708988 = 7.07%, train part: {'control_blocks.0': 99111760}. scepter [INFO] 2024-10-06 16:15:47,583 [File: diffusion_solver.py Function: print_model_params_status at line 749] Load forzen params 1303597228 / 1402708988 = 92.93%, forzen part: {'time_embed': 2050560, 'input_blocks': 251918080, 'middle_block': 97693440, 'output_blocks': 514236480, 'out': 12164, 'encoder': 34163592, 'decoder': 49490179, 'conv1': 72, 'conv2': 20, 'model': 354032641}. scepter [INFO] 2024-10-06 16:15:47,611 [File: diffusion_solver.py Function: print_model_params_status at line 745] Load trainable params 99111760 / 1402708988 = 7.07%, train part: {'control_blocks.0': 99111760}. scepter [INFO] 2024-10-06 16:15:47,612 [File: diffusion_solver.py Function: print_model_params_status at line 749] Load forzen params 1303597228 / 1402708988 = 92.93%, forzen part: {'time_embed': 2050560, 'input_blocks': 251918080, 'middle_block': 97693440, 'output_blocks': 514236480, 'out': 12164, 'encoder': 34163592, 'decoder': 49490179, 'conv1': 72, 'conv2': 20, 'model': 354032641}. scepter [INFO] 2024-10-06 16:15:47,630 [File: diffusion_solver.py Function: print_model_params_status at line 745] Load trainable params 99111760 / 1402708988 = 7.07%, train part: {'control_blocks.0': 99111760}. scepter [INFO] 2024-10-06 16:15:47,631 [File: diffusion_solver.py Function: print_model_params_status at line 749] Load forzen params 1303597228 / 1402708988 = 92.93%, forzen part: {'time_embed': 2050560, 'input_blocks': 251918080, 'middle_block': 97693440, 'output_blocks': 514236480, 'out': 12164, 'encoder': 34163592, 'decoder': 49490179, 'conv1': 72, 'conv2': 20, 'model': 354032641}. 0it [00:00, ?it/s]2024-10-06 16:16:03,705 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:03,707 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:03,747 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:03,758 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:03,760 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:03,825 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:03,832 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:03,833 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:03,873 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,002 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,005 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,057 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,058 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,069 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,073 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,074 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,098 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,114 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,123 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,125 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,126 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,128 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,166 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,166 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,168 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,168 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,207 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,236 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,239 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,292 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,298 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,300 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,339 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,345 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,347 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,386 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:04,502 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,503 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,547 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,585 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,586 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,602 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,604 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:04,626 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,645 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,661 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,662 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,666 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,668 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,684 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,686 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,702 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,708 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:04,738 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,792 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,793 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,836 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:04,846 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,848 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:04,888 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:04,979 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:04,982 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:05,046 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:05,050 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,050 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,052 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,052 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,058 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,060 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,098 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:05,098 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,100 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,115 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:05,116 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:05,119 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,122 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,129 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,132 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,138 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:05,183 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:05,192 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:05,271 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,273 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,288 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,289 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,312 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:05,327 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed 2024-10-06 16:16:05,364 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,365 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,404 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:05,672 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,674 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,720 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 2024-10-06 16:16:05,786 - modelscope - INFO - PyTorch version 2.0.1 Found. 2024-10-06 16:16:05,787 - modelscope - INFO - Loading ast index from /home/twinkle/.cache/modelscope/ast_indexer 2024-10-06 16:16:05,833 - modelscope - INFO - Loading done! Current index file version is 1.14.0, with md5 1966a94e015adba32f1fb346597a21b0 and a total number of 976 components indexed /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') /data/twinkle/app/scepter/scepter/modules/solver/base_solver.py:185: UserWarning: No module named 'pytorch_lightning' warnings.warn(f'{e}') 0it [00:11, ?it/s] 0it [00:11, ?it/s] 0it [00:11, ?it/s] 0it [00:12, ?it/s] Traceback (most recent call last): File "scepter/tools/run_train.py", line 62, in run() File "scepter/tools/run_train.py", line 58, in run we.init_env(cfg, logger=None, fn=run_task) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/scepter/modules/utils/distribute.py", line 431, in init_env mp.spawn(mp_worker, File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 239, in spawn return start_processes(fn, args, nprocs, join, daemon, start_method='spawn') File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 197, in start_processes while not context.join(): File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 160, in join raise ProcessRaisedException(msg, error_index, failed_process.pid) torch.multiprocessing.spawn.ProcessRaisedException:

-- Process 1 terminated with the following error: Traceback (most recent call last): File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 69, in _wrap fn(i, args) File "/data/twinkle/app/scepter/scepter/modules/utils/distribute.py", line 314, in mp_worker fn(cfg) File "/data/twinkle/app/scepter/scepter/tools/run_train.py", line 26, in run_task solver.solve() File "/data/twinkle/app/scepter/scepter/modules/solver/diffusion_solver.py", line 328, in solve self.run_train() File "/data/twinkle/app/scepter/scepter/modules/solver/diffusion_solver.py", line 341, in run_train self.run_eval() File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(args, kwargs) File "/data/twinkle/app/scepter/scepter/modules/solver/diffusion_solver.py", line 389, in run_eval results = self.run_step_eval(transfer_data_to_cuda(batch_data), File "/data/twinkle/app/scepter/scepter/modules/solver/base_solver.py", line 559, in run_step_eval results = self.model(batch_data) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(*args, kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward output = self._run_ddp_forward(*inputs, *kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward return module_to_run(inputs[0], kwargs[0]) # type: ignore[index] File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "/data/twinkle/app/scepter/scepter/modules/model/network/ldm/ldm.py", line 288, in forward return self.forward_test(kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(args, kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/amp/autocast_mode.py", line 14, in decorate_autocast return func(args, kwargs) File "/data/twinkle/app/scepter/scepter/modules/model/network/ldm/ldm_sce.py", line 133, in forward_test return super().forward_test(kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context return func(args, kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/amp/autocast_mode.py", line 14, in decorate_autocast return func(*args, kwargs) File "/data/twinkle/app/scepter/scepter/modules/model/network/ldm/ldm.py", line 318, in forward_test context = self.encode_condition(self.tokenizer(prompt).to( File "/data/twinkle/app/scepter/scepter/modules/model/network/ldm/ldm.py", line 228, in encode_condition return getattr(self.cond_stage_model, File "/data/twinkle/app/scepter/scepter/modules/model/embedder/embedder.py", line 303, in encode_text z = self.encode_with_transformer(tokens.to(we.device_id)) File "/data/twinkle/app/scepter/scepter/modules/model/embedder/embedder.py", line 278, in encode_with_transformer x = self.text_transformer_forward(x, File "/data/twinkle/app/scepter/scepter/modules/model/embedder/embedder.py", line 294, in text_transformer_forward x = r(x, attn_mask=attn_mask) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(*args, *kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/open_clip/transformer.py", line 263, in forward x = q_x + self.ls_1(self.attention(q_x=self.ln_1(q_x), k_x=k_x, v_x=v_x, attn_mask=attn_mask)) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/open_clip/transformer.py", line 250, in attention return self.attn( File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/modules/activation.py", line 1205, in forward attn_output, attn_output_weights = F.multi_head_attention_forward( File "/data/twinkle/anaconda3/envs/scepter/lib/python3.8/site-packages/torch/nn/functional.py", line 5251, in multi_head_attention_forward raise RuntimeError(f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}.") RuntimeError: The shape of the 2D attn_mask is torch.Size([77, 77]), but should be (8, 8).

/data/twinkle/anaconda3/envs/scepter/lib/python3.8/multiprocessing/resource_tracker.py:203: UserWarning: resource_tracker: There appear to be 160 leaked semaphore objects to clean up at shutdown warnings.warn('resource_tracker: There appear to be %d '