YosysHQ / oss-cad-suite-build

Multi-platform nightly builds of open source digital design and verification tools
ISC License
774 stars 70 forks source link

xdot fails when built from scratch #48

Open robtaylor opened 1 year ago

robtaylor commented 1 year ago
  -> Copy 'xdot' source to build dir ...
  -> Copy 'python3' output to build dir ...
  -> Compiling ...
+ source /work/default/patches/python3_package.sh
+ python3_package_setup
+ export _PYTHON_PROJECT_BASE=/work/_builds/linux-x64/xdot/python3/yosyshq
+ _PYTHON_PROJECT_BASE=/work/_builds/linux-x64/xdot/python3/yosyshq
+ export PYTHONHOME=/work/_builds/linux-x64/xdot/python3/yosyshq
+ PYTHONHOME=/work/_builds/linux-x64/xdot/python3/yosyshq
+ export PYTHONPATH=/work/_outputs/linux-x64/xdot/yosyshq/lib/python3.8/site-packages:/work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8:/work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/site-packages
+ PYTHONPATH=/work/_outputs/linux-x64/xdot/yosyshq/lib/python3.8/site-packages:/work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8:/work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/site-packages
+ '[' linux == linux ']'
+ cp /usr/lib/python3.8/lib-dynload/_asyncio.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_bz2.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_codecs_cn.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_codecs_hk.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_codecs_iso2022.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_codecs_jp.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_codecs_kr.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_codecs_tw.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_contextvars.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_crypt.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_ctypes.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_ctypes_test.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_curses.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_curses_panel.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_dbm.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_decimal.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_hashlib.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_json.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_lsprof.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_lzma.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_multibytecodec.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_multiprocessing.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_opcode.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_posixshmem.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_queue.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_sqlite3.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_ssl.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_testbuffer.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_testcapi.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_testimportmultiple.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_testinternalcapi.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_testmultiphase.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_uuid.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_xxsubinterpreters.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/_xxtestfuzz.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/audioop.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/mmap.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/nis.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/ossaudiodev.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/parser.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/readline.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/resource.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/termios.cpython-38-x86_64-linux-gnu.so /usr/lib/python3.8/lib-dynload/xxlimited.cpython-38-x86_64-linux-gnu.so /work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/lib-dynload/.
+ cp -r /usr/lib/python3/dist-packages/cairo /work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/lib-dynload/.
+ python3_package_pip_install 'xdot==1.1 pycairo PyGObject'
+ install_cmd='-m pip install --target /work/_outputs/linux-x64/xdot/yosyshq/lib/python3.8/site-packages --no-cache-dir --disable-pip-version-check xdot==1.1 pycairo PyGObject'
+ '[' linux-x64 == linux-arm ']'
+ '[' linux-x64 == linux-arm64 ']'
+ '[' linux-x64 == linux-riscv64 ']'
+ '[' linux-x64 == linux-x64 ']'
+ /work/_builds/linux-x64/xdot/python3/yosyshq/py3bin/python3 -m pip install --target /work/_outputs/linux-x64/xdot/yosyshq/lib/python3.8/site-packages --no-cache-dir --disable-pip-version-check xdot==1.1 pycairo PyGObject
Collecting xdot==1.1
  Downloading xdot-1.1.tar.gz (28 kB)
Collecting pycairo
  Downloading pycairo-1.21.0.tar.gz (340 kB)
  Installing build dependencies: started
  Installing build dependencies: finished with status 'done'
  Getting requirements to build wheel: started
  Getting requirements to build wheel: finished with status 'done'
    Preparing wheel metadata: started
    Preparing wheel metadata: finished with status 'done'
Collecting PyGObject
  Downloading PyGObject-3.42.2.tar.gz (719 kB)
  Installing build dependencies: started
  Installing build dependencies: finished with status 'error'
  ERROR: Command errored out with exit status 1:
   command: /work/_builds/linux-x64/xdot/python3/yosyshq/py3bin/python3 /work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/site-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-1eatocpq/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools wheel pycairo
       cwd: None
  Complete output (45 lines):
  WARNING: The directory '/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you may want sudo's -H flag.
  Collecting setuptools
    Downloading setuptools-65.5.0-py3-none-any.whl (1.2 MB)
  Collecting wheel
    Downloading wheel-0.37.1-py2.py3-none-any.whl (35 kB)
  Collecting pycairo
    Downloading pycairo-1.21.0.tar.gz (340 kB)
    Installing build dependencies: started
    Installing build dependencies: finished with status 'done'
    Getting requirements to build wheel: started
    Getting requirements to build wheel: finished with status 'done'
      Preparing wheel metadata: started
      Preparing wheel metadata: finished with status 'done'
  Building wheels for collected packages: pycairo
    Building wheel for pycairo (PEP 517): started
    Building wheel for pycairo (PEP 517): finished with status 'error'
    ERROR: Command errored out with exit status 1:
     command: /work/_builds/linux-x64/xdot/python3/yosyshq/py3bin/python3 /work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/site-packages/pip/_vendor/pep517/_in_process.py build_wheel /tmp/tmp2kkaxjul
         cwd: /tmp/pip-install-aos4rggd/pycairo
    Complete output (19 lines):
    running bdist_wheel
    running build
    running build_py
    creating build
    creating build/lib.linux-x86_64-cpython-38
    creating build/lib.linux-x86_64-cpython-38/cairo
    copying cairo/__init__.py -> build/lib.linux-x86_64-cpython-38/cairo
    copying cairo/__init__.pyi -> build/lib.linux-x86_64-cpython-38/cairo
    copying cairo/py.typed -> build/lib.linux-x86_64-cpython-38/cairo
    running build_ext
    building 'cairo._cairo' extension
    creating build/temp.linux-x86_64-cpython-38
    creating build/temp.linux-x86_64-cpython-38/cairo
    /usr/bin/x86_64-linux-gnu-gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -DPYCAIRO_VERSION_MAJOR=1 -DPYCAIRO_VERSION_MINOR=21 -DPYCAIRO_VERSION_MICRO=0 -I/usr/include/cairo -I/usr/include/glib-2.0 -I/usr/lib/x86_64-linux-gnu/glib-2.0/include -I/usr/include/pixman-1 -I/usr/include/uuid -I/usr/include/freetype2 -I/usr/include/libpng16 -I/yosyshq/include/python3.8 -c cairo/bufferproxy.c -o build/temp.linux-x86_64-cpython-38/cairo/bufferproxy.o -Wall -Warray-bounds -Wcast-align -Wconversion -Wextra -Wformat=2 -Wformat-nonliteral -Wformat-security -Wimplicit-function-declaration -Winit-self -Winline -Wmissing-format-attribute -Wmissing-noreturn -Wnested-externs -Wold-style-definition -Wpacked -Wpointer-arith -Wreturn-type -Wshadow -Wsign-compare -Wstrict-aliasing -Wundef -Wunused-but-set-variable -Wswitch-default -Wno-missing-field-initializers -Wno-unused-parameter -Wno-unused-command-line-argument -fno-strict-aliasing -fvisibility=hidden -std=c99
    cairo/bufferproxy.c:32:10: fatal error: Python.h: No such file or directory
       32 | #include <Python.h>
          |          ^~~~~~~~~~
    compilation terminated.
    error: command '/usr/bin/x86_64-linux-gnu-gcc' failed with exit code 1
    ----------------------------------------
    ERROR: Failed building wheel for pycairo
  Failed to build pycairo
  ERROR: Could not build wheels for pycairo which use PEP 517 and cannot be installed directly
  WARNING: You are using pip version 20.2.1; however, version 22.3 is available.
  You should consider upgrading via the '/work/_builds/linux-x64/xdot/python3/yosyshq/py3bin/python3 -m pip install --upgrade pip' command.
  ----------------------------------------
ERROR: Command errored out with exit status 1: /work/_builds/linux-x64/xdot/python3/yosyshq/py3bin/python3 /work/_builds/linux-x64/xdot/python3/yosyshq/lib/python3.8/site-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-1eatocpq/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools wheel pycairo Check the logs for full command output.

==> ERROR : Script returned error code 1.

Appears to be an issue with prefix being '/yosyshq'. This makes it through to all the sysconfig setup. Maybe a solution is bindmounting /yosyshq appropriately?

robtaylor commented 1 year ago

as a note, would be ideal if there was a regular CI thats builds without using previous releases to test this sort of thing!

robtaylor commented 1 year ago

this dirty hack seems to do the job:

@@ -420,6 +420,7 @@ def executeBuild(target, arch, prefix, build_dir, output_dir, nproc, pack_source
                '--user', '{}:{}'.format(os.getuid(), os.getgid()),
                '-v', '/tmp:/tmp',
                '-v', '{}:/work'.format(cwd),
+               '-v', '{}:{}'.format(os.path.join(cwd, os.path.split(output_dir)[0], "python3",os.path.relpath(prefix,"/")), prefix),
                '-w', os.path.join('/work', os.path.relpath(build_dir, os.getcwd())),
        ]
        for i, j in env.items():
mmicko commented 1 year ago

Hi @robtaylor, thanks for report. You are right, looks like things got broken meanwhile. Found a way to prevent this by setting proper variables when compiling packages. Testing all locally now so expect it soon.

robtaylor commented 1 year ago

ah cool, thanks @mmicko!

robtaylor commented 1 year ago

Relatedly, is it a goal that pip install would work when you're in the oss-build environment? currently a user would hit the same problem there.

robtaylor commented 1 year ago

nasty hack approach needed a tweak. I'll use this locally for now, but would be great to fix this issue properly! Might be worth checking out crossenv

diff --git a/src/base.py b/src/base.py
index 47d8945..522cbbf 100644
--- a/src/base.py
+++ b/src/base.py
@@ -414,13 +414,16 @@ def executeBuild(target, arch, prefix, build_dir, output_dir, nproc, pack_source

        scriptfile.flush()

+       host_prefix = os.path.join(cwd, os.path.split(output_dir)[0], "python3",os.path.relpath(prefix,"/"))
+       os.makedirs(host_prefix, exist_ok=True)
+
        log_step("Compiling ...")
        params = ['docker',
                'run', '--rm',
                '--user', '{}:{}'.format(os.getuid(), os.getgid()),
                '-v', '/tmp:/tmp',
                '-v', '{}:/work'.format(cwd),
-               '-v', '{}:{}'.format(os.path.join(cwd, os.path.split(output_dir)[0], "python3",os.path.relpath(prefix,"/")), prefix),
+               '-v', '{}:{}'.format(host_prefix, prefix),
                '-w', os.path.join('/work', os.path.relpath(build_dir, os.getcwd())),
        ]
        for i, j in env.items():