Open l3utterfly opened 7 months ago
Working
Using the following config:
{ "bf16": { "enabled": true }, "optimizer": { "type": "AdamW", "params": { "lr": "auto", "betas": "auto", "eps": "auto", "weight_decay": "auto" } }, "scheduler": { "type": "WarmupLR", "params": { "warmup_min_lr": "auto", "warmup_max_lr": "auto", "warmup_num_steps": "auto" } }, "zero_optimization": { "stage": 3, "offload_optimizer": { "device": "cpu", "pin_memory": true }, "offload_param": { "device": "cpu", "pin_memory": true }, "overlap_comm": true, "contiguous_gradients": true, "sub_group_size": 1e9, "reduce_bucket_size": "auto", "stage3_prefetch_bucket_size": "auto", "stage3_param_persistence_threshold": "auto", "stage3_max_live_parameters": 1e9, "stage3_max_reuse_distance": 1e9, "stage3_gather_16bit_weights_on_model_save": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "steps_per_print": 2000, "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false }
I get eval loss = nan or loss = 0 after a few steps of training.
base_model: /home/layla/src/text-generation-webui/models/Mistral-7B-v0.1 base_model_config: /home/layla/src/text-generation-webui/models/Mistral-7B-v0.1 model_type: MistralForCausalLM tokenizer_type: LlamaTokenizer is_mistral_derived_model: true load_in_8bit: false load_in_4bit: false strict: false datasets: - path: /home/layla/src/Layla-datasets/datasets_formatted/base/dailydialog.topicalchat.openhermes.jsonl ds_type: json # see other options below type: sharegpt conversation: vicuna_v1.1 # datasets: # - path: /home/layla/src/Layla-datasets/datasets_formatted/airoboros_alpaca.jsonl # type: alpaca dataset_prepared_path: last_run_prepared val_set_size: 0.002 output_dir: ./out sequence_len: 8192 sample_packing: true pad_to_sequence_len: true wandb_project: wandb_entity: wandb_watch: wandb_run_id: wandb_log_model: gradient_accumulation_steps: 4 micro_batch_size: 2 num_epochs: 1 optimizer: adamw_bnb_8bit lr_scheduler: cosine learning_rate: 0.0000005 train_on_inputs: false group_by_length: false bf16: true fp16: false tf32: false gradient_checkpointing: true early_stopping_patience: resume_from_checkpoint: local_rank: logging_steps: 1 xformers_attention: flash_attention: true warmup_ratio: 0.05 eval_steps: 0.1 eval_table_size: eval_table_max_new_tokens: 128 eval_sample_packing: false save_steps: 100 debug: deepspeed: /home/layla/src/Layla-datasets/axolotl/configs/deepspeed/zero3_cpuoffload.json # multi-gpu only weight_decay: 0.0 fsdp: fsdp_config: special_tokens: bos_token: "<s>" eos_token: "</s>" unk_token: "<unk>"
No response
3.11
2d65f470d54986f2df542cb8332e31113c04112d
Are you sure it's due to cpu offload? Can you try use the regular ds3_bf16
Please check that this issue hasn't been reported before.
Expected Behavior
Working
Current behaviour
Using the following config:
I get eval loss = nan or loss = 0 after a few steps of training.
Steps to reproduce
Config yaml
Possible solution
No response
Which Operating Systems are you using?
Python Version
3.11
axolotl branch-commit
2d65f470d54986f2df542cb8332e31113c04112d
Acknowledgements