Training stops with queue error

jwnsu commented 6 years ago

Training with 8 1080 Ti GPUs, which erred out after between 100 to 700 iterations:

Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "train.py", line 50, in pin_memory
    data = data_queue.get()
  File "/usr/lib/python2.7/multiprocessing/queues.py", line 117, in get
    res = self._recv()
  File "/home/user/tf/local/lib/python2.7/site-packages/torch/multiprocessing/queue.py", line 22, in recv
    return pickle.loads(buf)
  File "/usr/lib/python2.7/pickle.py", line 1388, in loads
    return Unpickler(file).load()
  File "/usr/lib/python2.7/pickle.py", line 864, in load
    dispatch[key](self)
  File "/usr/lib/python2.7/pickle.py", line 1139, in load_reduce
    value = func(*args)
  File "/home/user/tf/local/lib/python2.7/site-packages/torch/multiprocessing/reductions.py", line 68, in rebuild_storage_fd
    fd = multiprocessing.reduction.rebuild_handle(df)
  File "/usr/lib/python2.7/multiprocessing/reduction.py", line 155, in rebuild_handle
    conn = Client(address, authkey=current_process().authkey)
  File "/usr/lib/python2.7/multiprocessing/connection.py", line 169, in Client
    c = SocketClient(address)
  File "/usr/lib/python2.7/multiprocessing/connection.py", line 308, in SocketClient
    s.connect(address)
  File "/usr/lib/python2.7/socket.py", line 228, in meth
    return getattr(self._sock,name)(*args)
error: [Errno 2] No such file or directory

Any suggestion? Thx.

jwnsu commented 6 years ago

Reduced to 4 GPUs and same error again (with a few more iterations.)

Env: ubuntu 16.04.05, Nvidia 1080 Ti, pytorch 0.4.0.

heilaw commented 6 years ago

We only tested CornerNet with Python3.6. Can you please update your Python and try it again?

jwnsu commented 6 years ago

Thx, after switch to Python3.5, training now moved past previous failure points.

It will be good for Readme to add a requirement of Python3.

YiLiangNie commented 6 years ago

Thx, after switch to Python3.5, training now moved past previous failure points.

It will be good for Readme to add a requirement of Python3.

My environment is also python3.5, but I have encountered this problem： loading all datasets... using 4 threads loading from cache file: ./cache/coco_train2014.pkl loading annotations into memory... Done (t=8.75s) creating index... index created! loading from cache file: ./cache/coco_train2014.pkl loading annotations into memory... Done (t=9.22s) creating index... index created! loading from cache file: ./cache/coco_train2014.pkl loading annotations into memory... Done (t=9.01s) creating index... index created! loading from cache file: ./cache/coco_train2014.pkl loading annotations into memory... Done (t=10.84s) creating index... index created! loading from cache file: ./cache/coco_val2014.pkl loading annotations into memory... Done (t=2.71s) creating index... index created! system config... {'batch_size': 49, 'cache_dir': './cache', 'chunk_sizes': [4, 5, 5, 5, 5, 5, 5, 5, 5, 5], 'config_dir': './config', 'data_dir': './data', 'data_rng': <mtrand.RandomState object at 0x7f9b7e81ff30>, 'dataset': 'MSCOCO', 'decay_rate': 10, 'display': 5, 'learning_rate': 0.00025, 'max_iter': 500000, 'nnet_rng': <mtrand.RandomState object at 0x7f9b7e81ff78>, 'opt_algo': 'adam', 'prefetch_size': 5, 'pretrain': None, 'result_dir': './results', 'sampling_function': 'kp_detection', 'snapshot': 5000, 'snapshot_name': 'CornerNet', 'stepsize': 450000, 'test_split': 'testdev', 'train_split': 'trainval', 'val_iter': 100, 'val_split': 'minival', 'weight_decay': False, 'weight_decay_rate': 1e-05, 'weight_decay_type': 'l2'} db config... {'ae_threshold': 0.5, 'border': 128, 'categories': 80, 'data_aug': True, 'gaussian_bump': True, 'gaussian_iou': 0.7, 'gaussian_radius': -1, 'input_size': [511, 511], 'lighting': True, 'max_per_image': 100, 'merge_bbox': False, 'nms_algorithm': 'exp_soft_nms', 'nms_kernel': 3, 'nms_threshold': 0.5, 'output_sizes': [[128, 128]], 'rand_color': True, 'rand_crop': True, 'rand_pushes': False, 'rand_samples': False, 'rand_scale_max': 1.4, 'rand_scale_min': 0.6, 'rand_scale_step': 0.1, 'rand_scales': array([0.6, 0.7, 0.8, 0.9, 1. , 1.1, 1.2, 1.3]), 'special_crop': False, 'test_scales': [1], 'top_k': 100, 'weight_exp': 8} len of db: 82783 start prefetching data... shuffling indices... start prefetching data... shuffling indices... start prefetching data... shuffling indices... start prefetching data... shuffling indices... building model... module_file: models.CornerNet start prefetching data... shuffling indices... total parameters: 201035212 setting learning rate to: 0.00025 training start... 0%| | 0/500000 [00:00<?, ?it/s] Traceback (most recent call last): File "train.py", line 195, in train(training_dbs, validation_db, args.start_iter) File "train.py", line 155, in train nnet.set_lr(learning_rate) File "/usr/lib/python3.5/contextlib.py", line 77, in exit self.gen.throw(type, value, traceback) File "/home/nyl/CornerNet-master/utils/tqdm.py", line 23, in stdout_to_tqdm raise exc File "/home/nyl/CornerNet-master/utils/tqdm.py", line 21, in stdout_to_tqdm yield save_stdout File "train.py", line 137, in train training_loss = nnet.train(*training) File "/home/nyl/CornerNet-master/nnet/py_factory.py", line 81, in train loss = self.network(xs, ys) File "/home/nyl/.local/lib/python3.5/site-packages/torch/nn/modules/module.py", line 477, in call result = self.forward(input, *kwargs) File "/home/nyl/CornerNet-master/models/py_utils/data_parallel.py", line 66, in forward inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids, self.chunk_sizes) File "/home/nyl/CornerNet-master/models/py_utils/data_parallel.py", line 77, in scatter return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim, chunk_sizes=self.chunk_sizes) File "/home/nyl/CornerNet-master/models/py_utils/scatter_gather.py", line 30, in scatter_kwargs inputs = scatter(inputs, target_gpus, dim, chunk_sizes) if inputs else [] File "/home/nyl/CornerNet-master/models/py_utils/scatter_gather.py", line 25, in scatter return scatter_map(inputs) File "/home/nyl/CornerNet-master/models/py_utils/scatter_gather.py", line 18, in scatter_map return list(zip(map(scatter_map, obj))) File "/home/nyl/CornerNet-master/models/py_utils/scatter_gather.py", line 20, in scatter_map return list(map(list, zip(map(scatter_map, obj)))) File "/home/nyl/CornerNet-master/models/py_utils/scatter_gather.py", line 15, in scatter_map return Scatter.apply(target_gpus, chunk_sizes, dim, obj) File "/home/nyl/.local/lib/python3.5/site-packages/torch/nn/parallel/_functions.py", line 87, in forward outputs = comm.scatter(input, ctx.target_gpus, ctx.chunk_sizes, ctx.dim, streams) File "/home/nyl/.local/lib/python3.5/site-packages/torch/cuda/comm.py", line 142, in scatter return tuple(torch._C._scatter(tensor, devices, chunk_sizes, dim, streams)) RuntimeError: Device index must be -1 or non-negative, got -667653224 (Device at /pytorch/torch/lib/tmp_install/include/ATen/Device.h:47) frame #0: + 0xc4968b (0x7f9bb26de68b in /home/nyl/.local/lib/python3.5/site-packages/torch/_C.cpython-35m-x86_64-linux-gnu.so) frame #1: + 0x39124b (0x7f9bb1e2624b in /home/nyl/.local/lib/python3.5/site-packages/torch/_C.cpython-35m-x86_64-linux-gnu.so) frame #2: PyCFunction_Call + 0x77 (0x4e9ba7 in python3) frame #3: PyEval_EvalFrameEx + 0x614 (0x5372f4 in python3) frame #4: python3() [0x540199] frame #5: PyEval_EvalFrameEx + 0x50b2 (0x53bd92 in python3) frame #6: PyEval_EvalCodeEx + 0x13b (0x540f9b in python3) frame #7: python3() [0x4ebd23] frame #8: PyObject_Call + 0x47 (0x5c1797 in python3) frame #9: PyEval_CallObjectWithKeywords + 0x30 (0x534d90 in python3) frame #10: THPFunction_apply(_object, _object*) + 0x38f (0x7f9bb22046af in /home/nyl/.local/lib/python3.5/site-packages/torch/_C.cpython-35m-x86_64-linux-gnu.so) frame #11: PyCFunction_Call + 0x4f (0x4e9b7f in python3) frame #12: PyEval_EvalFrameEx + 0x614 (0x5372f4 in python3) frame #13: PyEval_EvalCodeEx + 0x88a (0x5416ea in python3) frame #14: python3() [0x4ebd23] frame #15: PyObject_Call + 0x47 (0x5c1797 in python3) frame #16: python3() [0x53645f] frame #17: PyIter_Next + 0xe (0x5bfbce in python3) frame #18: PySequence_Tuple + 0xee (0x5c3b7e in python3) frame #19: PyEval_EvalFrameEx + 0x6c18 (0x53d8f8 in python3) frame #20: PyEval_EvalCodeEx + 0x88a (0x5416ea in python3) frame #21: python3() [0x4ebd23] frame #22: PyObject_Call + 0x47 (0x5c1797 in python3) frame #23: python3() [0x53645f] frame #24: PyIter_Next + 0xe (0x5bfbce in python3) frame #25: PySequence_Tuple + 0xee (0x5c3b7e in python3) frame #26: PyEval_EvalFrameEx + 0x6c18 (0x53d8f8 in python3) frame #27: python3() [0x5406df] frame #28: PyEval_EvalFrameEx + 0x54f0 (0x53c1d0 in python3) frame #29: python3() [0x5406df] frame #30: PyEval_EvalFrameEx + 0x50b2 (0x53bd92 in python3) frame #31: python3() [0x540199] frame #32: PyEval_EvalFrameEx + 0x50b2 (0x53bd92 in python3) frame #33: PyEval_EvalFrameEx + 0x4b04 (0x53b7e4 in python3) frame #34: PyEval_EvalCodeEx + 0x13b (0x540f9b in python3) frame #35: python3() [0x4ebe37] frame #36: PyObject_Call + 0x47 (0x5c1797 in python3) frame #37: PyEval_EvalFrameEx + 0x252b (0x53920b in python3) frame #38: PyEval_EvalCodeEx + 0x13b (0x540f9b in python3) frame #39: python3() [0x4ebd23] frame #40: PyObject_Call + 0x47 (0x5c1797 in python3) frame #41: python3() [0x4fb9ce] frame #42: PyObject_Call + 0x47 (0x5c1797 in python3) frame #43: python3() [0x574b36] frame #44: PyObject_Call + 0x47 (0x5c1797 in python3) frame #45: PyEval_EvalFrameEx + 0x4ec6 (0x53bba6 in python3) frame #46: PyEval_EvalCodeEx + 0x13b (0x540f9b in python3) frame #47: python3() [0x4ebe37] frame #48: PyObject_Call + 0x47 (0x5c1797 in python3) frame #49: PyEval_EvalFrameEx + 0x252b (0x53920b in python3) frame #50: python3() [0x540199] frame #51: PyEval_EvalFrameEx + 0x50b2 (0x53bd92 in python3) frame #52: python3() [0x540199] frame #53: PyEval_EvalCode + 0x1f (0x540e4f in python3) frame #54: python3() [0x60c272] frame #55: PyRun_FileExFlags + 0x9a (0x60e71a in python3) frame #56: PyRun_SimpleFileExFlags + 0x1bc (0x60ef0c in python3) frame #57: Py_Main + 0x456 (0x63fb26 in python3) frame #58: main + 0xe1 (0x4cfeb1 in python3) frame #59: __libc_start_main + 0xf0 (0x7f9bd7fa2830 in /lib/x86_64-linux-gnu/libc.so.6) frame #60: _start + 0x29 (0x5d6049 in python3)

Exception in thread Thread-1: Traceback (most recent call last): File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner self.run() File "/usr/lib/python3.5/threading.py", line 862, in run self._target(*self._args, **self._kwargs) File "train.py", line 50, in pin_memory data = data_queue.get() File "/usr/lib/python3.5/multiprocessing/queues.py", line 113, in get return ForkingPickler.loads(res) File "/home/nyl/.local/lib/python3.5/site-packages/torch/multiprocessing/reductions.py", line 151, in rebuild_storage_fd fd = df.detach() File "/usr/lib/python3.5/multiprocessing/resource_sharer.py", line 57, in detach with _resource_sharer.get_connection(self._id) as conn: File "/usr/lib/python3.5/multiprocessing/resource_sharer.py", line 87, in get_connection c = Client(address, authkey=process.current_process().authkey) File "/usr/lib/python3.5/multiprocessing/connection.py", line 493, in Client answer_challenge(c, authkey) File "/usr/lib/python3.5/multiprocessing/connection.py", line 732, in answer_challenge message = connection.recv_bytes(256) # reject large message File "/usr/lib/python3.5/multiprocessing/connection.py", line 216, in recv_bytes buf = self._recv_bytes(maxlength) File "/usr/lib/python3.5/multiprocessing/connection.py", line 407, in _recv_bytes buf = self._recv(4) File "/usr/lib/python3.5/multiprocessing/connection.py", line 379, in _recv chunk = read(handle, remaining) ConnectionResetError: [Errno 104] Connection reset by peer

Have you ever encountered it? Thx!

YiLiangNie commented 6 years ago

I changed the batch_size and chunk_sizes in config/CornerNet.json file， it works.