distributed error

[gitxdf]

I trid to train the slowfast model with multigrid, unfortunatey I can not run it sucessfully.

The errors are as follows: Traceback (most recent call last): File "/home/dfxue/disk/code/SlowFast/tools/run_net.py", line 42, in main() File "/home/dfxue/disk/code/SlowFast/tools/run_net.py", line 23, in main launch_job(cfg=cfg, init_method=args.init_method, func=train) File "/home/dfxue/disk/code/SlowFast/slowfast/utils/misc.py", line 285, in launch_job daemon=daemon, File "/home/dfxue/disk/anaconda3/envs/py/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 171, in spawn while not spawn_context.join(): File "/home/dfxue/disk/anaconda3/envs/py/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 118, in join raise Exception(msg) Exception:

-- Process 2 terminated with the following error: Traceback (most recent call last): File "/home/dfxue/disk/anaconda3/envs/py/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap fn(i, *args) File "/home/dfxue/disk/code/SlowFast/slowfast/utils/multiprocessing.py", line 50, in run func(cfg) File "/home/dfxue/disk/code/SlowFast/tools/train_net.py", line 382, in train model = build_model(cfg) File "/home/dfxue/disk/code/SlowFast/slowfast/models/build.py", line 51, in build_model module=model, device_ids=[cur_device], output_device=cur_device File "/home/dfxue/disk/anaconda3/envs/py/lib/python3.7/site-packages/torch/nn/parallel/distributed.py", line 303, in init self.broadcast_bucket_size) File "/home/dfxue/disk/anaconda3/envs/py/lib/python3.7/site-packages/torch/nn/parallel/distributed.py", line 485, in _distributed_broadcast_coalesced dist._broadcast_coalesced(self.process_group, tensors, buffer_size) RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1579022060824/work/torch/lib/c10d/ProcessGroupNCCL.cpp:410, unhandled cuda error, NCCL version 2.4.8

The scripts are: python tools/run_net.py --cfg /home/dfxue/disk/code/SlowFast/configs/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.yaml NUM_GPUS 4 TRAIN.BATCH_SIZE 4 SOLVER.BASE_LR 0.0125 DATA.PATH_TO_DATA_DIR /backup/dfxue/kinetics/kinetics400_videos/ TEST.ENABLE False TENSORBOARD.ENABLE True

And the logs are as follows: Train with config: [08/24 23:31:08][INFO] train_net: 379: {'AVA': {'ANNOTATION_DIR': '/mnt/vol/gfsai-flash3-east/ai-group/users/haoqifan/ava/frame_list/', 'BGR': False, 'DETECTION_SCORE_THRESH': 0.9, 'EXCLUSION_FILE': 'ava_val_excluded_timestamps_v2.2.csv', 'FRAME_DIR': '/mnt/fair-flash3-east/ava_trainval_frames.img/', 'FRAME_LIST_DIR': '/mnt/vol/gfsai-flash3-east/ai-group/users/haoqifan/ava/frame_list/', 'FULL_TEST_ON_VAL': False, 'GROUNDTRUTH_FILE': 'ava_val_v2.2.csv', 'IMG_PROC_BACKEND': 'cv2', 'LABEL_MAP_FILE': 'ava_action_list_v2.2_for_activitynet_2019.pbtxt', 'TEST_FORCE_FLIP': False, 'TEST_LISTS': ['val.csv'], 'TEST_PREDICT_BOX_LISTS': ['ava_val_predicted_boxes.csv'], 'TRAIN_GT_BOX_LISTS': ['ava_train_v2.2.csv'], 'TRAIN_LISTS': ['train.csv'], 'TRAIN_PCA_EIGVAL': [0.225, 0.224, 0.229], 'TRAIN_PCA_EIGVEC': [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.814], [-0.5836, -0.6948, 0.4203]], 'TRAIN_PCA_JITTER_ONLY': True, 'TRAIN_PREDICT_BOX_LISTS': [], 'TRAIN_USE_COLOR_AUGMENTATION': False}, 'BENCHMARK': CfgNode({'NUM_EPOCHS': 5, 'LOG_PERIOD': 100, 'SHUFFLE': True}), 'BN': {'NORM_TYPE': 'batchnorm', 'NUM_BATCHES_PRECISE': 200, 'NUM_SPLITS': 1, 'NUM_SYNC_DEVICES': 1, 'USE_PRECISE_STATS': True, 'WEIGHT_DECAY': 0.0}, 'DATA': {'CROP_SIZE': 224, 'DECODING_BACKEND': 'pyav', 'ENSEMBLE_METHOD': 'sum', 'INPUT_CHANNEL_NUM': [3, 3], 'INV_UNIFORM_SAMPLE': False, 'MEAN': [0.45, 0.45, 0.45], 'MULTI_LABEL': False, 'NUM_FRAMES': 8, 'PATH_LABEL_SEPARATOR': ' ', 'PATH_PREFIX': '', 'PATH_TO_DATA_DIR': '/backup/dfxue/kinetics/kinetics400_videos/', 'RANDOM_FLIP': True, 'REVERSE_INPUT_CHANNEL': False, 'SAMPLING_RATE': 2, 'STD': [0.225, 0.225, 0.225], 'TARGET_FPS': 30, 'TEST_CROP_SIZE': 224, 'TRAIN_CROP_SIZE': 158, 'TRAIN_JITTER_SCALES': [256, 320]}, 'DATA_LOADER': {'ENABLE_MULTI_THREAD_DECODE': False, 'NUM_WORKERS': 1, 'PIN_MEMORY': True}, 'DEMO': {'BUFFER_SIZE': 0, 'CLIP_VIS_SIZE': 10, 'COMMON_CLASS_NAMES': ['watch (a person)', 'talk to (e.g., self, a person, a group)', 'listen to (a person)', 'touch (an object)', 'carry/hold (an object)', 'walk', 'sit', 'lie/sleep', 'bend/bow (at the waist)'], 'COMMON_CLASS_THRES': 0.7, 'DETECTRON2_CFG': 'COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml', 'DETECTRON2_THRESH': 0.9, 'DETECTRON2_WEIGHTS': 'detectron2://COCO-Detection/faster_rcnn_R_50_FPN_3x/137849458/model_final_280758.pkl', 'DISPLAY_HEIGHT': 0, 'DISPLAY_WIDTH': 0, 'ENABLE': False, 'FPS': 30, 'GT_BOXES': '', 'INPUT_FORMAT': 'BGR', 'INPUT_VIDEO': '', 'LABEL_FILE_PATH': '', 'NUM_CLIPS_SKIP': 0, 'NUM_VIS_INSTANCES': 2, 'OUTPUT_FILE': '', 'OUTPUT_FPS': -1, 'PREDS_BOXES': '', 'SLOWMO': 1, 'STARTING_SECOND': 900, 'THREAD_ENABLE': False, 'UNCOMMON_CLASS_THRES': 0.3, 'VIS_MODE': 'thres', 'WEBCAM': -1}, 'DETECTION': {'ALIGNED': True, 'ENABLE': False, 'ROI_XFORM_RESOLUTION': 7, 'SPATIAL_SCALE_FACTOR': 16}, 'DIST_BACKEND': 'nccl', 'LOG_MODEL_INFO': True, 'LOG_PERIOD': 10, 'MODEL': {'ARCH': 'slowfast', 'DROPOUT_RATE': 0.5, 'FC_INIT_STD': 0.01, 'HEAD_ACT': 'softmax', 'LOSS_FUNC': 'cross_entropy', 'MODEL_NAME': 'SlowFast', 'MULTI_PATHWAY_ARCH': ['slowfast'], 'NUM_CLASSES': 400, 'SINGLE_PATHWAY_ARCH': ['c2d', 'i3d', 'slow']}, 'MULTIGRID': {'BN_BASE_SIZE': 8, 'DEFAULT_B': 4, 'DEFAULT_S': 224, 'DEFAULT_T': 32, 'EPOCH_FACTOR': 1.5, 'EVAL_FREQ': 3, 'LONG_CYCLE': True, 'LONG_CYCLE_FACTORS': [(0.25, 0.7071067811865476), (0.5, 0.7071067811865476), (0.5, 1), (1, 1)], 'LONG_CYCLE_SAMPLING_RATE': 8, 'SHORT_CYCLE': True, 'SHORT_CYCLE_FACTORS': [0.5, 0.7071067811865476]}, 'NONLOCAL': {'GROUP': [[1, 1], [1, 1], [1, 1], [1, 1]], 'INSTANTIATION': 'dot_product', 'LOCATION': [[[], []], [[], []], [[], []], [[], []]], 'POOL': [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]]}, 'NUM_GPUS': 4, 'NUM_SHARDS': 1, 'OUTPUT_DIR': '.', 'RESNET': {'DEPTH': 50, 'INPLACE_RELU': True, 'NUM_BLOCK_TEMP_KERNEL': [[3, 3], [4, 4], [6, 6], [3, 3]], 'NUM_GROUPS': 1, 'SPATIAL_DILATIONS': [[1, 1], [1, 1], [1, 1], [1, 1]], 'SPATIAL_STRIDES': [[1, 1], [2, 2], [2, 2], [2, 2]], 'STRIDE_1X1': False, 'TRANS_FUNC': 'bottleneck_transform', 'WIDTH_PER_GROUP': 64, 'ZERO_INIT_FINAL_BN': True}, 'RNG_SEED': 0, 'SHARD_ID': 0, 'SLOWFAST': {'ALPHA': 4, 'BETA_INV': 8, 'FUSION_CONV_CHANNEL_RATIO': 2, 'FUSION_KERNEL_SZ': 7}, 'SOLVER': {'BASE_LR': 0.0125, 'DAMPENING': 0.0, 'GAMMA': 0.1, 'LRS': [8.0, 4.0, 2.0, 1.0, 0.8, 0.4, 0.2, 0.1, 0.08000000000000002, 0.04000000000000001, 0.020000000000000004, 0.010000000000000002, 0.010000000000000002, 0.0010000000000000002], 'LR_POLICY': 'steps_with_relative_lrs', 'MAX_EPOCH': 358, 'MOMENTUM': 0.9, 'NESTEROV': True, 'OPTIMIZING_METHOD': 'sgd', 'STEPS': [0, 73, 110, 142, 158, 205, 228, 248, 259, 291, 308, 322, 329, 343], 'STEP_SIZE': 1, 'WARMUP_EPOCHS': 34.0, 'WARMUP_FACTOR': 0.1, 'WARMUP_START_LR': 0.01, 'WEIGHT_DECAY': 0.0001}, 'TENSORBOARD': {'CATEGORIES_PATH': '', 'CLASS_NAMES_PATH': '', 'CONFUSION_MATRIX': {'ENABLE': False, 'FIGSIZE': [8, 8], 'SUBSET_PATH': ''}, 'ENABLE': True, 'HISTOGRAM': {'ENABLE': False, 'FIGSIZE': [8, 8], 'SUBSET_PATH': '', 'TOPK': 10}, 'LOG_DIR': '', 'MODEL_VIS': {'ACTIVATIONS': False, 'COLORMAP': 'Pastel2', 'ENABLE': False, 'GRAD_CAM': {'COLORMAP': 'viridis', 'ENABLE': True, 'LAYER_LIST': [], 'USE_TRUE_LABEL': False}, 'INPUT_VIDEO': False, 'LAYER_LIST': [], 'MODEL_WEIGHTS': False, 'TOPK_PREDS': 1}, 'PREDICTIONS_PATH': '', 'WRONG_PRED_VIS': {'ENABLE': False, 'SUBSET_PATH': '', 'TAG': 'Incorrectly classified videos.'}}, 'TEST': {'BATCH_SIZE': 64, 'CHECKPOINT_FILE_PATH': '', 'CHECKPOINT_TYPE': 'pytorch', 'DATASET': 'kinetics', 'ENABLE': False, 'NUM_ENSEMBLE_VIEWS': 10, 'NUM_SPATIAL_CROPS': 3, 'SAVE_RESULTS_PATH': ''}, 'TRAIN': {'AUTO_RESUME': True, 'BATCH_SIZE': 32, 'CHECKPOINT_FILE_PATH': '', 'CHECKPOINT_INFLATE': False, 'CHECKPOINT_PERIOD': 1, 'CHECKPOINT_TYPE': 'pytorch', 'DATASET': 'kinetics', 'ENABLE': True, 'EVAL_PERIOD': 10}}

Thanks

[gitxdf]

I run the training successfully after the machine was rebooted.

[soumyadbanik]

I run the training successfully after the machine was rebooted.

Hi @gitxdf, can you tell me what does the excluded timestamps refer? I'm making my custom AVA format dataset but don't get that

[gitxdf]

您好：    您发送的邮件已经收到。

[BonGum]

@soumyadbanik Did you figure out what that means? In the process of val，it seems to be used. And I think that in my own datasets it seems useless. And how can i quit it in the code?

[gitxdf]

您好：    您发送的邮件已经收到。