add params

[ouyangyu]

• [ ] FP16 多卡测试要配置

• [ ] 设置flow.config.collective_boxing.nccl_fusion_threshold_mb 设置为 16

• [ ] 可以尝试 flow.config.collective_boxing.nccl_fusion_max_ops 设置为 16/24

  python3 of_cnn_train_val.py \
   --train_data_dir=$DATA_ROOT/train \
   --train_data_part_num=256 \
   --val_data_dir=$DATA_ROOT/validation \
   --val_data_part_num=256 \
   --num_nodes=1 \
   --gpu_num_per_node=8 \
   --optimizer="sgd" \
   --momentum=0.875 \
   --label_smoothing=0.1 \
   --learning_rate=1.024 \
   --loss_print_every_n_iter=100 \
   --batch_size_per_device=128 \
   --val_batch_size_per_device=50 \
   --num_epoch=50 \
   --use_fp16=True \
   --nccl_fusion_threshold_mb=16 \
   --nccl_fusion_max_ops=16 \
   --model="resnet50" 2>&1 | tee ${LOGFILE}

• [ ] FP16 多卡要分别测试真实数据和合成数据，混合精度下data loader 可能会成为瓶颈

  python3 of_cnn_train_val.py \
   --train_data_dir=$DATA_ROOT/train \
   --train_data_part_num=256 \
   --num_nodes=1 \
   --gpu_num_per_node=8 \
   --optimizer="sgd" \
   --momentum=0.875 \
   --label_smoothing=0.1 \
   --learning_rate=1.024 \
   --loss_print_every_n_iter=100 \
   --batch_size_per_device=128 \
   --val_batch_size_per_device=50 \
   --num_epoch=50 \
   --use_fp16=True \
   --nccl_fusion_threshold_mb=16 \
   --nccl_fusion_max_ops=16 \
   --model="resnet50" 2>&1 | tee ${LOGFILE}

[ShawnXuan]

• AMP = ON
• XLA = ON
• start with 1n8g, try 2 or 4 nodes if possible
• max batch size

  nccl_fusion_all_reduce_use_buffer(True)

nccl_fusion_threshold_mb nccl_fusion_max_ops	4	8	16	32
16
24
32

nccl_fusion_all_reduce_use_buffer(False)

nccl_fusion_threshold_mb nccl_fusion_max_ops	4	8	16	32
16
24
32