Isolated cloning tests fail on Apple Silicon

[smburdick]

Description of the issue

The following tests fail in Apple Silicon:

=============================================== short test summary info===============================================
FAILED dev_tools/cloned_env_test.py::test_isolated_env_cloning[a] - FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pyt...
FAILED dev_tools/cloned_env_test.py::test_isolated_env_cloning[c] - FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pyt...
FAILED dev_tools/cloned_env_test.py::test_isolated_env_cloning[b] - FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pyt...
================= 3 failed, 21238 passed, 158 skipped, 92 xfailed, 1384 warnings in 153.66s (0:02:33) =================

The issue appears to arise from

$ virtualenv -p /var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pytest/test_isolated/bin/pip install 'flynt==0.64'
/Users/sam/.virtualenvs/cirq-py3/lib/python3.11/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.
!!

        ********************************************************************************
        Please avoid running ``setup.py`` directly.
        Instead, use pypa/build, pypa/installer or other
        standards-based tools.

        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.
        ********************************************************************************

!!
  self.initialize_options()
FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pytest/test_isolated/bin/pip'

How to reproduce the issue

Run ./check/pytest

More info on the failed test:

``` ____________________________________________ test_isolated_env_cloning[b] _____________________________________________ [gw0] darwin -- Python 3.11.2 /Users/sam/.virtualenvs/cirq-py3/bin/python3 cloned_env = .base_env_creator at 0x16de40220>, param = 'b' @only_on_posix # ensure that no cirq packages are on the PYTHONPATH, this is important, otherwise # the "isolation" fails and all the cirq modules would be in the list @mock.patch.dict(os.environ, {"PYTHONPATH": ""}) @pytest.mark.parametrize('param', ['a', 'b', 'c']) def test_isolated_env_cloning(cloned_env, param): > env = cloned_env("test_isolated", "flynt==0.64") dev_tools/cloned_env_test.py:36: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ dev_tools/conftest.py:76: in base_env_creator _create_base_env(base_dir, pip_install_args) dev_tools/conftest.py:100: in _create_base_env shell_tools.run([f"{base_dir}/bin/pip", "install", *pip_install_args]) dev_tools/shell_tools.py:95: in run return subprocess.run(args, **subprocess_run_kwargs) /usr/local/Cellar/python@3.11/3.11.2_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py:548: in run with Popen(*popenargs, **kwargs) as process: /usr/local/Cellar/python@3.11/3.11.2_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py:1024: in __init__ self._execute_child(args, executable, preexec_fn, close_fds, _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = args = ['/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pytest/test_isolated/bin/pip', 'install', 'flynt==0.64'] executable = b'/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pytest/test_isolated/bin/pip', preexec_fn = None close_fds = True, pass_fds = (), cwd = None, env = None, startupinfo = None, creationflags = 0, shell = False p2cread = -1, p2cwrite = -1, c2pread = -1, c2pwrite = -1, errread = -1, errwrite = -1, restore_signals = True gid = None, gids = None, uid = None, umask = -1, start_new_session = False, process_group = -1 def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session, process_group): """Execute program (POSIX version)""" if isinstance(args, (str, bytes)): args = [args] elif isinstance(args, os.PathLike): if shell: raise TypeError('path-like args is not allowed when ' 'shell is true') args = [args] else: args = list(args) if shell: # On Android the default shell is at '/system/bin/sh'. unix_shell = ('/system/bin/sh' if hasattr(sys, 'getandroidapilevel') else '/bin/sh') args = [unix_shell, "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] sys.audit("subprocess.Popen", executable, args, cwd, env) if (_USE_POSIX_SPAWN and os.path.dirname(executable) and preexec_fn is None and not close_fds and not pass_fds and cwd is None and (p2cread == -1 or p2cread > 2) and (c2pwrite == -1 or c2pwrite > 2) and (errwrite == -1 or errwrite > 2) and not start_new_session and process_group == -1 and gid is None and gids is None and uid is None and umask < 0): self._posix_spawn(args, executable, env, restore_signals, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) return orig_executable = executable # For transferring possible exec failure from child to parent. # Data format: "exception name:hex errno:description" # Pickle is not used; it is complex and involves memory allocation. errpipe_read, errpipe_write = os.pipe() # errpipe_write must not be in the standard io 0, 1, or 2 fd range. low_fds_to_close = [] while errpipe_write < 3: low_fds_to_close.append(errpipe_write) errpipe_write = os.dup(errpipe_write) for low_fd in low_fds_to_close: os.close(low_fd) try: try: # We must avoid complex work that could involve # malloc or free in the child process to avoid # potential deadlocks, thus we do all this here. # and pass it to fork_exec() if env is not None: env_list = [] for k, v in env.items(): k = os.fsencode(k) if b'=' in k: raise ValueError("illegal environment variable name") env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) if os.path.dirname(executable): executable_list = (executable,) else: # This matches the behavior of os._execvpe(). executable_list = tuple( os.path.join(os.fsencode(dir), executable) for dir in os.get_exec_path(env)) fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) self.pid = _fork_exec( args, executable_list, close_fds, tuple(sorted(map(int, fds_to_keep))), cwd, env_list, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, restore_signals, start_new_session, process_group, gid, gids, uid, umask, preexec_fn, _USE_VFORK) self._child_created = True finally: # be sure the FD is closed no matter what os.close(errpipe_write) self._close_pipe_fds(p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) # Wait for exec to fail or succeed; possibly raising an # exception (limited in size) errpipe_data = bytearray() while True: part = os.read(errpipe_read, 50000) errpipe_data += part if not part or len(errpipe_data) > 50000: break finally: # be sure the FD is closed no matter what os.close(errpipe_read) if errpipe_data: try: pid, sts = os.waitpid(self.pid, 0) if pid == self.pid: self._handle_exitstatus(sts) else: self.returncode = sys.maxsize except ChildProcessError: pass try: exception_name, hex_errno, err_msg = ( errpipe_data.split(b':', 2)) # The encoding here should match the encoding # written in by the subprocess implementations # like _posixsubprocess err_msg = err_msg.decode() except ValueError: exception_name = b'SubprocessError' hex_errno = b'0' err_msg = 'Bad exception data from child: {!r}'.format( bytes(errpipe_data)) child_exception_type = getattr( builtins, exception_name.decode('ascii'), SubprocessError) if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) child_exec_never_called = (err_msg == "noexec") if child_exec_never_called: err_msg = "" # The error must be from chdir(cwd). err_filename = cwd else: err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) > raise child_exception_type(errno_num, err_msg, err_filename) E FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/4y/4m8lt8vj6fz5grxd3hxzg9100000gn/T/cirq-pytest/test_isolated/bin/pip' /usr/local/Cellar/python@3.11/3.11.2_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/subprocess.py:1901: FileNotFoundError ```

Cirq version 1.4.0.dev

[smburdick]

For the record, #6378 suppresses this, but doesn't fix the underlying issue.

[pavoljuhas]

I don't see this, the test passes on my side in a MacOS Miniconda environment with Python 3.11.2. The check/pytest also passes successfully in the recent CI execution here.

Please make sure all test dependencies are installed in your virtual environment as in the CI workflow setup here https://github.com/quantumlib/Cirq/blob/d33b1a71ac9721e762a2fa1b9e9871edfe187a3c/.github/workflows/ci.yml#L286-L289 and then execute the test with

$ check/pytest dev_tools/cloned_env_test.py

Please reopen if you still see the test failure.