glide-the
commented
1 month ago The same data can be fine-tuned using this diffusers example script https://github.com/huggingface/diffusers/tree/main/examples/cogvideo
#!/bin/bash
export MODEL_PATH="/mnt/ceph/develop/jiawei/model_checkpoint/CogVideoX-5b-I2V"
export CACHE_PATH="~/.cache"
export OUTPUT_PATH="/mnt/ceph/develop/jiawei/model_checkpoint/hf_cogvideox_imglora_test"
export VAL_IMAGE1="/mnt/ceph/develop/jiawei/diffusers_fork_zmf/examples/cogvideo/frame0.jpg"
export VAL_IMAGE2="/mnt/ceph/develop/jiawei/diffusers_fork_zmf/examples/cogvideo/frame30.jpg"
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
GPU_IDS="0,1,2,3,4,5,6,7"
WANDB_PROJECT=DiffUsers_CogVideoX_IMAGE_test
# if you are not using wth 8 gus, change `accelerate_config_machine_single.yaml` num_processes as your gpu number
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True WANDB_API_KEY= accelerate launch --main_process_port 29501 --gpu_ids $GPU_IDS --config_file /mnt/ceph/develop/jiawei/diffusers_fork_zmf/examples/cogvideo/lora_image_k7.yaml \
train_cogvideox_image_to_video_lora.py \
--gradient_checkpointing \
--pretrained_model_name_or_path $MODEL_PATH \
--cache_dir $CACHE_PATH \
--enable_tiling \
--enable_slicing \
--instance_data_root /mnt/ceph/develop/jiawei/lora_dataset/Dance-VideoGeneration-Dataset \
--caption_column prompts.txt \
--video_column videos.txt \
--id_token 奶糖, \
--validation_prompt "奶糖, A young girl in a white blouse and navy skirt stands in a sunlit park, smiling and holding up two fingers. She's surrounded by trees and a pathway, with dappled sunlight casting shadows. A young woman in a school uniform stands on a tree-lined path, surprised, with hands raised. In the park, a woman in a white blouse with a navy collar raises her hands in a playful 'V' shape, surrounded by lush greenery and sunlight.:::奶糖, A young woman with long dark hair tied into ponytails stands in a cozy, warmly lit room, smiling gently at the camera. She takes a selfie, her hair styled in loose waves, with a playful expression. The background is a plain, light-colored wall, emphasizing her features." \
--validation_images "$VAL_IMAGE1:::$VAL_IMAGE2" \
--validation_prompt_separator ::: \
--num_validation_videos 1 \
--validation_epochs 5 \
--seed 42 \
--rank 128 \
--lora_alpha 32 \
--mixed_precision bf16 \
--output_dir $OUTPUT_PATH \
--height 480 \
--width 720 \
--fps 8 \
--max_num_frames 49 \
--skip_frames_start 0 \
--skip_frames_end 0 \
--train_batch_size 1 \
--num_train_epochs 150 \
--checkpointing_steps 1000 \
--gradient_accumulation_steps 1 \
--learning_rate 1e-3 \
--lr_scheduler cosine_with_restarts \
--lr_warmup_steps 200 \
--lr_num_cycles 1 \
--enable_slicing \
--enable_tiling \
--gradient_checkpointing \
--optimizer AdamW \
--adam_beta1 0.9 \
--adam_beta2 0.95 \
--max_grad_norm 1.0 \
--resume_from_checkpoint latest \
--report_to wandb --tracker_name $WANDB_PROJECTcompute_environment: LOCAL_MACHINE
debug: false
distributed_type: MULTI_GPU
downcast_bf16: 'no'
enable_cpu_affinity: false
gpu_ids: 0,1,2,3,5,6,7
machine_rank: 0
main_training_function: main
mixed_precision: 'no'
num_machines: 1
num_processes: 7
rdzv_backend: static
same_network: true
tpu_env: []
tpu_use_cluster: false
tpu_use_sudo: false
use_cpu: false
a-r-r-o-w
GPU count | 8 GPU type | [NVIDIA A800-SXM4-80GB
enable nccl
train script
uncompiled_2.yaml