testkraken command crashes with master datalad

[djarecka]

Testkraken cli for workflows that use datalad will crash for future datalad versions (or current master).

testkraken /Users/dorota/testkraken/workflows4regtests/afni_dc2019/3dcopy_datalad returns:

``` Process Process-1: Traceback (most recent call last): File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/Users/dorota/soft_repo/datalad/datalad/distribution/dataset.py", line 503, in apply_func return f(**kwargs) File "/Users/dorota/soft_repo/datalad/datalad/interface/utils.py", line 371, in eval_func hooks = get_jsonhooks_from_config(ds.config if ds else dlcfg) File "/Users/dorota/soft_repo/datalad/datalad/distribution/dataset.py", line 343, in config repo = self.repo File "/Users/dorota/soft_repo/datalad/datalad/distribution/dataset.py", line 289, in repo self._repo = repo_from_path(self._path) File "/Users/dorota/soft_repo/datalad/datalad/core/local/repo.py", line 61, in repo_from_path repo = cls(path, create=False, **kw) File "/Users/dorota/soft_repo/datalad/datalad/support/repo.py", line 154, in __call__ instance = type.__call__(cls, *new_args, **new_kwargs) File "/Users/dorota/soft_repo/datalad/datalad/support/annexrepo.py", line 245, in __init__ for r in self.get_remotes(): File "/Users/dorota/soft_repo/datalad/datalad/support/annexrepo.py", line 816, in get_remotes remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only) File "/Users/dorota/soft_repo/datalad/datalad/support/gitrepo.py", line 2081, in get_remotes self.config.reload() File "/Users/dorota/soft_repo/datalad/datalad/support/gitrepo.py", line 1324, in config self._cfg = ConfigManager(dataset=self, source='any') File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 274, in __init__ self.reload(force=True) File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 327, in reload self._stores[store_id] = self._reload(runargs) File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 359, in _reload encoding='utf-8', File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 717, in _run out = self._runner.run(self._config_cmd + args, **kwargs) File "/Users/dorota/soft_repo/datalad/datalad/cmd.py", line 390, in run env=env, File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/asyncio/base_events.py", line 574, in run_until_complete self.run_forever() File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/asyncio/base_events.py", line 541, in run_forever self._run_once() File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/asyncio/base_events.py", line 1750, in _run_once event_list = self._selector.select(timeout) File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/selectors.py", line 558, in select kev_list = self._selector.control(None, max_ev, timeout) OSError: [Errno 9] Bad file descriptor Process Process-2: Traceback (most recent call last): File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap self.run() File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/multiprocessing/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/Users/dorota/soft_repo/datalad/datalad/distribution/dataset.py", line 503, in apply_func return f(**kwargs) File "/Users/dorota/soft_repo/datalad/datalad/interface/utils.py", line 371, in eval_func hooks = get_jsonhooks_from_config(ds.config if ds else dlcfg) File "/Users/dorota/soft_repo/datalad/datalad/distribution/dataset.py", line 343, in config repo = self.repo File "/Users/dorota/soft_repo/datalad/datalad/distribution/dataset.py", line 289, in repo self._repo = repo_from_path(self._path) File "/Users/dorota/soft_repo/datalad/datalad/core/local/repo.py", line 61, in repo_from_path repo = cls(path, create=False, **kw) File "/Users/dorota/soft_repo/datalad/datalad/support/repo.py", line 154, in __call__ instance = type.__call__(cls, *new_args, **new_kwargs) File "/Users/dorota/soft_repo/datalad/datalad/support/annexrepo.py", line 245, in __init__ for r in self.get_remotes(): File "/Users/dorota/soft_repo/datalad/datalad/support/annexrepo.py", line 816, in get_remotes remotes = super(AnnexRepo, self).get_remotes(with_urls_only=with_urls_only) File "/Users/dorota/soft_repo/datalad/datalad/support/gitrepo.py", line 2081, in get_remotes self.config.reload() File "/Users/dorota/soft_repo/datalad/datalad/support/gitrepo.py", line 1324, in config self._cfg = ConfigManager(dataset=self, source='any') File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 274, in __init__ self.reload(force=True) File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 327, in reload self._stores[store_id] = self._reload(runargs) File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 359, in _reload encoding='utf-8', File "/Users/dorota/soft_repo/datalad/datalad/config.py", line 717, in _run out = self._runner.run(self._config_cmd + args, **kwargs) File "/Users/dorota/soft_repo/datalad/datalad/cmd.py", line 390, in run env=env, File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/asyncio/base_events.py", line 574, in run_until_complete self.run_forever() File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/asyncio/base_events.py", line 541, in run_forever self._run_once() File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/asyncio/base_events.py", line 1750, in _run_once event_list = self._selector.select(timeout) File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/selectors.py", line 558, in select kev_list = self._selector.control(None, max_ev, timeout) OSError: [Errno 9] Bad file descriptor Traceback (most recent call last): File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/bin/testkraken", line 33, in sys.exit(load_entry_point('testkraken', 'console_scripts', 'testkraken')()) File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/site-packages/click/core.py", line 829, in __call__ return self.main(*args, **kwargs) File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/site-packages/click/core.py", line 782, in main rv = self.invoke(ctx) File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/site-packages/click/core.py", line 1066, in invoke return ctx.invoke(self.callback, **ctx.params) File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/site-packages/click/core.py", line 610, in invoke return callback(*args, **kwargs) File "/Users/dorota/testkraken/testkraken/cli.py", line 24, in main wf.run() File "/Users/dorota/testkraken/testkraken/workflowregtest.py", line 117, in run self._run_workflow_in_matrix_of_envs() File "/Users/dorota/testkraken/testkraken/workflowregtest.py", line 160, in _run_workflow_in_matrix_of_envs self._run_pydra(image=image, soft_ver_str=name) File "/Users/dorota/testkraken/testkraken/workflowregtest.py", line 218, in _run_pydra process_path_obj(value, self.data_path) File "/Users/dorota/testkraken/testkraken/data_management.py", line 213, in process_path_obj f"Datalad download failed {attempt_count} times, you may " File "/Users/dorota/miniconda3/envs/tmp_datalad_testkraken_py37/lib/python3.7/site-packages/_pytest/outcomes.py", line 121, in exit raise Exit(msg, returncode) _pytest.outcomes.Exit: Datalad download failed 2 times, you may not be connected to the internet ```

See @yarikoptic comments for suggestions.

[yarikoptic]

Yeap, note that AFAIK it is osx specific, and it wouldn't be just datalad but anything using asyncio (and doesn't create a new loop) unless you provide each process you fork a new loop