The checkpoint state_dict contains keys that are not used by the model

[ad45675]

hi, when I eval or load the model the below warning message will appear. is it correct for this message?

WARNING [07/29 10:39:53 fvcore.common.checkpoint]: The checkpoint state_dict contains keys that are not used by the model: modelTeacher.D_img.conv1.{bias, weight} modelTeacher.D_img.conv2.{bias, weight} modelTeacher.D_img.conv3.{bias, weight} modelTeacher.D_img.classifier.{bias, weight} modelStudent.D_img.conv1.{bias, weight} modelStudent.D_img.conv2.{bias, weight} modelStudent.D_img.conv3.{bias, weight} modelStudent.D_img.classifier.{bias, weight}

[yujheli]

@ad45675 I did not have this message when I loaded the model. Did you clone the updated code?

[ad45675]

thanks ! i clone the latest code and the problem have been solves!

But, after run about thousand iters the below error will occur!

` ERROR:adapteacher.engine.trainer:Exception during training: Traceback (most recent call last): File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 405, in train_loop self.after_step() File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/train_loop.py", line 180, in after_step h.after_step() File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/hooks.py", line 448, in after_step self._do_eval() File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/hooks.py", line 421, in _do_eval results = self._func() File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 768, in test_and_save_results_student self._last_eval_results_student = self.test(self.cfg, self.model) File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/defaults.py", line 593, in test data_loader = cls.build_test_loader(cfg, dataset_name) File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 734, in build_test_loader return build_detection_test_loader(cfg, dataset_name) File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/data/build.py", line 128, in build_detection_test_loader else None, File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/data/build.py", line 227, in get_detection_dataset_dicts dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/data/build.py", line 227, in dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/data/catalog.py", line 58, in get return f() File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/data/datasets/builtin.py", line 142, in x, y, from_json=False, to_polygons=False File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/data/datasets/cityscapes_foggy.py", line 79, in load_cityscapes_instances pool = mp.Pool(processes=max(mp.cpu_count() // get_world_size() // 2, 4)) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/context.py", line 119, in Pool context=self.get_context()) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/pool.py", line 176, in init self._repopulate_pool() File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/pool.py", line 241, in _repopulate_pool w.start() File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/process.py", line 112, in start self._popen = self._Popen(self) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/context.py", line 284, in _Popen return Popen(process_obj) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 32, in init super().init(process_obj) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/popen_fork.py", line 20, in init self._launch(process_obj) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 59, in _launch cmd, self._fds) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/util.py", line 415, in spawnv_passfds errpipe_read, errpipe_write = os.pipe() OSError: [Errno 24] Too many open files Traceback (most recent call last): File "train_net.py", line 79, in args=(args,), File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/launch.py", line 79, in launch daemon=False, File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 230, in spawn return start_processes(fn, args, nprocs, join, daemon, start_method='spawn') File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 188, in start_processes while not context.join(): File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 150, in join raise ProcessRaisedException(msg, error_index, failed_process.pid) torch.multiprocessing.spawn.ProcessRaisedException:

-- Process 1 terminated with the following error: Traceback (most recent call last): File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 59, in _wrap fn(i, args) File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/launch.py", line 125, in _distributed_worker main_func(args) File "/home/nvt05031/py37_zero/adaptive_teacher/train_net.py", line 66, in main return trainer.train() File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 386, in train self.train_loop(self.start_iter, self.max_iter) File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 405, in train_loop self.after_step() File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/train_loop.py", line 180, in after_step h.after_step() File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/hooks.py", line 448, in after_step self._do_eval() File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/hooks.py", line 421, in _do_eval results = self._func() File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 768, in test_and_save_results_student self._last_eval_results_student = self.test(self.cfg, self.model) File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/engine/defaults.py", line 593, in test data_loader = cls.build_test_loader(cfg, dataset_name) File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/engine/trainer.py", line 734, in build_test_loader return build_detection_test_loader(cfg, dataset_name) File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/data/build.py", line 128, in build_detection_test_loader else None, File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/data/build.py", line 227, in get_detection_dataset_dicts dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/data/build.py", line 227, in dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] File "/venv/py37_zero-XRWy4lKA/lib/python3.7/site-packages/detectron2/data/catalog.py", line 58, in get return f() File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/data/datasets/builtin.py", line 142, in x, y, from_json=False, to_polygons=False File "/home/nvt05031/py37_zero/adaptive_teacher/adapteacher/data/datasets/cityscapes_foggy.py", line 79, in load_cityscapes_instances pool = mp.Pool(processes=max(mp.cpu_count() // get_world_size() // 2, 4)) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/context.py", line 119, in Pool context=self.get_context()) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/pool.py", line 176, in init self._repopulate_pool() File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/pool.py", line 241, in _repopulate_pool w.start() File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/process.py", line 112, in start self._popen = self._Popen(self) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/context.py", line 284, in _Popen return Popen(process_obj) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 32, in init super().init(process_obj) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/popen_fork.py", line 20, in init self._launch(process_obj) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/popen_spawn_posix.py", line 59, in _launch cmd, self._fds) File "/opt/utility/python3.7.2/lib/python3.7/multiprocessing/util.py", line 415, in spawnv_passfds errpipe_read, errpipe_write = os.pipe() OSError: [Errno 24] Too many open files

(py37_zero-XRWy4lKA) nvt05031@oagpu7:~/py37_zero/adaptive_teacher$ /opt/utility/python3.7.2/lib/python3.7/multiprocessing/semaphore_tracker.py:144: UserWarning: semaphore_tracker: There appear to be 196 leaked semaphores to clean up at shutdown len(cache)) `

Have you ever met this error?

[yujheli]

@ad45675 I did not have this error? Were you running cityscapes when this error occured? Looks like the error happened when building cityscapes_foggy.