search

snap-stanford / ogb

Benchmark datasets, data loaders, and evaluators for graph machine learning
https://ogb.stanford.edu
MIT License
1.89k stars 397 forks source link

import PygNodePropPredDataset hang the program #461

Closed LukeLIN-web closed 4 months ago

LukeLIN-web commented 8 months ago

Setting:

ogb 1.3.6 , install from pip.

from ogb.nodeproppred import PygNodePropPredDataset

In my practice, this import line often hangs my program. I am not sure if this is because of multi-process or not.

Traceback (most recent call last):
  File "<string>", line 1, in <module>
    stream_input_queue_list = [mp.Manager().Queue() for i in range(device_num)]
  File "/root/share/gnninference/examples/pyg.py", line 143, in <listcomp>
    stream_input_queue_list = [mp.Manager().Queue() for i in range(device_num)]
  File "/opt/conda/lib/python3.10/multiprocessing/context.py", line 57, in Manager
    m.start()
  File "/opt/conda/lib/python3.10/multiprocessing/managers.py", line 566, in start
  File "/opt/conda/lib/python3.10/multiprocessing/spawn.py", line 116, in spawn_main
    self._address = reader.recv()
  File "/opt/conda/lib/python3.10/multiprocessing/connection.py", line 255, in recv
    buf = self._recv_bytes()
  File "/opt/conda/lib/python3.10/multiprocessing/connection.py", line 419, in _recv_bytes
    exitcode = _main(fd, parent_sentinel)
  File "/opt/conda/lib/python3.10/multiprocessing/spawn.py", line 125, in _main
    buf = self._recv(4)
  File "/opt/conda/lib/python3.10/multiprocessing/connection.py", line 384, in _recv
    prepare(preparation_data)
  File "/opt/conda/lib/python3.10/multiprocessing/spawn.py", line 236, in prepare
    chunk = read(handle, remaining)
KeyboardInterrupt
    _fixup_main_from_path(data['init_main_from_path'])
  File "/opt/conda/lib/python3.10/multiprocessing/spawn.py", line 287, in _fixup_main_from_path
    main_content = runpy.run_path(main_path,
  File "/opt/conda/lib/python3.10/runpy.py", line 289, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "/opt/conda/lib/python3.10/runpy.py", line 96, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "/opt/conda/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/root/share/gnninference/examples/pyg.py", line 12, in <module>
    from ogb.nodeproppred import PygNodePropPredDataset
  File "/opt/conda/lib/python3.10/site-packages/ogb/nodeproppred/__init__.py", line 1, in <module>
    from .evaluate import Evaluator
  File "/opt/conda/lib/python3.10/site-packages/ogb/nodeproppred/evaluate.py", line 5, in <module>
    from sklearn.metrics import roc_auc_score
  File "/opt/conda/lib/python3.10/site-packages/sklearn/__init__.py", line 82, in <module>
    from .base import clone
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 17, in <module>
    from .utils import _IS_32BIT
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/__init__.py", line 21, in <module>
    from . import _joblib
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/_joblib.py", line 7, in <module>
    import joblib
  File "/opt/conda/lib/python3.10/site-packages/joblib/__init__.py", line 113, in <module>
    from .memory import Memory, MemorizedResult, register_store_backend
  File "/opt/conda/lib/python3.10/site-packages/joblib/memory.py", line 32, in <module>
    from ._store_backends import StoreBackendBase, FileSystemStoreBackend
  File "/opt/conda/lib/python3.10/site-packages/joblib/_store_backends.py", line 15, in <module>
    from .backports import concurrency_safe_rename
  File "/opt/conda/lib/python3.10/site-packages/joblib/backports.py", line 22, in <module>
    import distutils  # noqa
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1002, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 945, in _find_spec
  File "/opt/conda/lib/python3.10/site-packages/_distutils_hack/__init__.py", line 97, in find_spec
    return method()
  File "/opt/conda/lib/python3.10/site-packages/_distutils_hack/__init__.py", line 108, in spec_for_distutils
    mod = importlib.import_module('setuptools._distutils')
  File "/opt/conda/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "/opt/conda/lib/python3.10/site-packages/setuptools/__init__.py", line 16, in <module>
    import setuptools.version
  File "/opt/conda/lib/python3.10/site-packages/setuptools/version.py", line 1, in <module>
    import pkg_resources
  File "<frozen importlib._bootstrap>", line 224, in _lock_unlock_module
  File "<frozen importlib._bootstrap>", line 120, in acquire
LukeLIN-web commented 5 months ago

joblib 1.2.0
scikit-learn 1.2.2

It repeat again on 

 File "/opt/conda/lib/python3.10/site-packages/gs-0.1-py3.10.egg/gs/__init__.py", line 8, in <module>
    from .matrix_api import Matrix, BatchMatrix
  File "/opt/conda/lib/python3.10/site-packages/gs-0.1-py3.10.egg/gs/matrix_api/__init__.py", line 1, in <module>
    from .matrix import Matrix
  File "/opt/conda/lib/python3.10/site-packages/gs-0.1-py3.10.egg/gs/matrix_api/matrix.py", line 8, in <module>
    from ..utils import create_block_from_coo, create_block_from_csc
  File "/opt/conda/lib/python3.10/site-packages/gs-0.1-py3.10.egg/gs/utils/__init__.py", line 1, in <module>
    from .load_graph import load_reddit, load_ogb
  File "/opt/conda/lib/python3.10/site-packages/gs-0.1-py3.10.egg/gs/utils/load_graph.py", line 4, in <module>
    from ogb.nodeproppred import DglNodePropPredDataset
  File "/opt/conda/lib/python3.10/site-packages/ogb/nodeproppred/__init__.py", line 1, in <module>
    from .evaluate import Evaluator
  File "/opt/conda/lib/python3.10/site-packages/ogb/nodeproppred/evaluate.py", line 5, in <module>
    from sklearn.metrics import roc_auc_score
  File "/opt/conda/lib/python3.10/site-packages/sklearn/__init__.py", line 82, in <module>
    from .base import clone
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 17, in <module>
    from .utils import _IS_32BIT
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/__init__.py", line 21, in <module>
    from . import _joblib
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/_joblib.py", line 7, in <module>
    import joblib
  File "/opt/conda/lib/python3.10/site-packages/joblib/__init__.py", line 113, in <module>
    from .memory import Memory, MemorizedResult, register_store_backend
  File "/opt/conda/lib/python3.10/site-packages/joblib/memory.py", line 32, in <module>
    from ._store_backends import StoreBackendBase, FileSystemStoreBackend
  File "/opt/conda/lib/python3.10/site-packages/joblib/_store_backends.py", line 15, in <module>
    from .backports import concurrency_safe_rename
  File "/opt/conda/lib/python3.10/site-packages/joblib/backports.py", line 22, in <module>
    import distutils  # noqa
  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1002, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 945, in _find_spec
  File "/opt/conda/lib/python3.10/site-packages/_distutils_hack/__init__.py", line 97, in find_spec
    return method()
  File "/opt/conda/lib/python3.10/site-packages/_distutils_hack/__init__.py", line 108, in spec_for_distutils
    mod = importlib.import_module('setuptools._distutils')
  File "/opt/conda/lib/python3.10/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "/opt/conda/lib/python3.10/site-packages/setuptools/__init__.py", line 16, in <module>
    import setuptools.version
  File "/opt/conda/lib/python3.10/site-packages/setuptools/version.py", line 1, in <module>
    import pkg_resources
  File "<frozen importlib._bootstrap>", line 224, in _lock_unlock_module
  File "<frozen importlib._bootstrap>", line 120, in acquire
pip install --upgrade joblib scikit-learn

can solve

LukeLIN-web commented 4 months ago

It should be solved with the PR #473