The below traceback occurs when running ./bqetl bootstrap with Python 3.12. The distutils module was removed in 3.12 and it looks like bigquery-etl depends on an older version of setuptools (via pip-tools and nodeenv) that will need to get updated.
Unfortunately it looks like pip-tools doesn't support 3.12 yet (?). So maybe the fix here is to change the supported Python version from >= 3.11 to ==3.11.
Collecting numpy==1.24.2 (from -r requirements.txt (line 958))
Downloading numpy-1.24.2.tar.gz (10.9 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.9/10.9 MB 42.7 MB/s eta 0:00:00
Installing build dependencies ... done
Getting requirements to build wheel ... done
ERROR: Exception:
Traceback (most recent call last):
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/cli/base_command.py", line 180, in exc_logging_wrapper
status = run_func(*args)
^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/cli/req_command.py", line 248, in wrapper
return func(self, options, args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/commands/install.py", line 377, in run
requirement_set = resolver.resolve(
^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 92, in resolve
result = self._result = resolver.resolve(
^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 546, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 397, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/resolvelib/resolvers.py", line 173, in _add_to_criteria
if not criterion.candidates:
^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/resolvelib/structs.py", line 156, in __bool__
return bool(self._sequence)
^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 155, in __bool__
return any(self)
^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 143, in <genexpr>
return (c for c in iterator if id(c) not in self._incompatible_ids)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 47, in _iter_built
candidate = func()
^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 208, in _make_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 293, in __init__
super().__init__(
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 156, in __init__
self.dist = self._prepare()
^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 225, in _prepare
dist = self._prepare_distribution()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 304, in _prepare_distribution
return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/operations/prepare.py", line 538, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/operations/prepare.py", line 653, in _prepare_linked_requirement
dist = _get_prepared_distribution(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/operations/prepare.py", line 69, in _get_prepared_distribution
abstract_dist.prepare_distribution_metadata(
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 48, in prepare_distribution_metadata
self._install_build_reqs(finder)
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 118, in _install_build_reqs
build_reqs = self._get_build_requires_wheel()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/distributions/sdist.py", line 95, in _get_build_requires_wheel
return backend.get_requires_for_build_wheel()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_internal/utils/misc.py", line 697, in get_requires_for_build_wheel
return super().get_requires_for_build_wheel(config_settings=cs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 166, in get_requires_for_build_wheel
return self._call_hook('get_requires_for_build_wheel', {
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 321, in _call_hook
raise BackendUnavailable(data.get('traceback', ''))
pip._vendor.pyproject_hooks._impl.BackendUnavailable: Traceback (most recent call last):
File "/home/ahal/dev/bigquery-etl/venv/lib64/python3.12/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 77, in _build_backend
obj = import_module(mod_path)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib64/python3.12/importlib/__init__.py", line 90, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1310, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 995, in exec_module
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "/tmp/pip-build-env-1vqhcnhp/overlay/lib/python3.12/site-packages/setuptools/__init__.py", line 10, in <module>
import distutils.core
ModuleNotFoundError: No module named 'distutils'
The below traceback occurs when running
./bqetl bootstrapwith Python 3.12. Thedistutilsmodule was removed in 3.12 and it looks likebigquery-etldepends on an older version ofsetuptools(viapip-toolsandnodeenv) that will need to get updated.Unfortunately it looks like
pip-toolsdoesn't support 3.12 yet (?). So maybe the fix here is to change the supported Python version from>= 3.11to==3.11.┆Issue is synchronized with this Jira Task