ctypes lib build with ExternalProject

darcymason commented 11 months ago

Hi,

Very much like #11, I am trying to build a wheel with a pre-built library for use with ctypes, except that my external library (libjpeg-turbo, as a git submodule) does not allow inclusion by add_subdirectory. So I am trying with ExternalProject_Add.

I should add that I am not at all familiar with cmake, I'm learning as I go, and really appreciate any help.

Following #11, I haven't been able to get the install step working. After the ExternalProject_Add call it doesn't recognize the target name. Here is part of my pyproject.toml:

[tool.py-build-cmake.module]
directory = "src"

[tool.py-build-cmake.sdist] 
include = ["CMakeLists.txt", "src/*", "extern/**/*"]

[tool.py-build-cmake.cmake] # How to build the CMake project
build_type = "RelWithDebInfo"
build_path = "build"
source_path = "."  # "extern/libjpeg-turbo"
build_args = ["-j"]
install_components = ["python_module"]

and the whole CMakeLists.txt file at the root level

include(ExternalProject)

cmake_minimum_required(VERSION 3.3)
project(pylibjpeg_turbo)
find_package(Python3 REQUIRED COMPONENTS Development)

# tried as -P switch to INSTALL_COMMAND:  set(PROJECT_INSTALL_STEPS_FILE project_install_steps.cmake)
set(BUILD_SHARED_LIBS On)
set(LIB_INSTALL_DIR ${PY_BUILD_CMAKE_MODULE_NAME})

ExternalProject_Add(libjpeg-turbo
    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/extern/libjpeg-turbo
    BUILD_COMMAND ${CMAKE_COMMAND} --build . --target turbojpeg
    INSTALL_COMMAND ""
)

# Install the module
if (WIN32)
    install(TARGETS turbojpeg
            EXCLUDE_FROM_ALL
            RUNTIME DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}
            COMPONENT python_module)
else()
    install(TARGETS turbojpeg
            EXCLUDE_FROM_ALL
            LIBRARY DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}
            COMPONENT python_module)
endif()

Which gives

 install TARGETS given target "turbojpeg" which does not exist.

I also tried a -P install_steps.cmake file for the INSTALL_COMMAND within the ExternalProject_Add, which basically just gives a subprocess error without any useful information that I can see.

I'd be happy to just copy the built libraries using INSTALL DIRECTORY FILES_MATCHING, but then I hit another problem - my built libraries remain in a tmp path even when I specify build_path. I ran with verbose to try to get more information: python3 -m build --no-isolation . -C verbose

The first dump of the dicts gives:

cmake:
{'linux': {'args': [],
           'build_args': ['-j'],
           'build_path': '/home/dmason/git/turbo/build',
...

and similar for other platforms, but a second dump in the verbose output gives:

cmake:
{'linux': {'args': [],
           'build_args': ['-j'],
           'build_path': '/tmp/build-via-sdist-n_e1032f/pylibjpeg_turbo-0.1.0/build',

So I'm struggling with my limited knowledge and a lot of googling, how to put the pieces together which can make this happen. Really appreciate any help to get me pointed in the right direction.

A couple of BTWs:

on a user running the wheel install, I don't need libjpeg-turbo installed into system locations, I'd be happy to keep it as package data and give ctypes the full path.
I will also need to use this with cibuildwheel, if that has any bearing.

Thanks

tttapa commented 11 months ago

The problem with ExternalProject is that the external project is built and installed during the build step of the main project: i.e. after executing your CMakeLists.txt script containing the install(TARGETS ...).

Instead, you could let ExternalProject_Add install the subproject to the temporary build folder for you, and then as the install step of the main project, simply copy the files you need from the build folder to the Wheel destination.

For example:

# pyproject.toml
[build-system]
requires = ["py-build-cmake~=0.1.9a2"]
build-backend = "py_build_cmake.build"

[tool.py-build-cmake.sdist]
include = ["CMakeLists.txt", "extern/libjpeg-turbo"]

[tool.py-build-cmake.cmake]
minimum_version = "3.17"
build_type = "RelWithDebInfo"
build_args = ["-j"]
install_components = ["shlib"]

# CMakeLists.txt
cmake_minimum_required(VERSION 3.17)
project(pylibjpeg_turbo)

include(ExternalProject)

set(JPEG_INSTALL_DIR ${PROJECT_BINARY_DIR}/libjpeg-turbo-install)
ExternalProject_Add(libjpeg-turbo
    SOURCE_DIR
        ${PROJECT_SOURCE_DIR}/extern/libjpeg-turbo
    CMAKE_CACHE_ARGS
        -DBUILD_SHARED_LIBS:BOOL=On
        -DENABLE_SHARED:BOOL=On
        -DENABLE_STATIC:BOOL=Off
        -DCMAKE_INSTALL_PREFIX:STRING=${JPEG_INSTALL_DIR}
        -DCMAKE_INSTALL_LIBDIR:STRING=lib
        -DCMAKE_INSTALL_DOCDIR:STRING=doc
        -DCMAKE_PLATFORM_NO_VERSIONED_SONAME:BOOL=On
)
set(PY_BUILD_CMAKE_MODULE_NAME ${PROJECT_NAME} CACHE STRING "")
install(DIRECTORY ${JPEG_INSTALL_DIR}/lib
        DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}/libjpeg-turbo
        COMPONENT shlib
        FILES_MATCHING REGEX "(lib)?turbojpeg.(so|dll|dylib)$")
install(FILES ${JPEG_INSTALL_DIR}/doc/LICENSE.md
        DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}/libjpeg-turbo
        COMPONENT shlib)

This generates a Wheel package with the following contents:

├── pylibjpeg_turbo
│   ├── __init__.py
│   └── libjpeg-turbo
│       ├── lib
│       │   └── libturbojpeg.so
│       └── LICENSE.md
└── pylibjpeg_turbo-1.0.0.dist-info
    ├── entry_points.txt
    ├── LICENSE
    ├── METADATA
    ├── RECORD
    └── WHEEL

darcymason commented 11 months ago

This generates a Wheel package with the following contents:

Thanks so much, indeed it works perfectly on WSL2 linux locally. I'm working on a local Windows build, my first time compiling anything on this computer, it fails even at the 'simple test compile' stage, so no doubt my setup.

Meanwhile, also tried through my github workflow, the submodule is not getting recursed on any platform. This is not a py-build-cmake issue, but just reporting my results for anyone who finds this issue later, or if there are any suggestions:

D:/a/pylibjpeg-turbo/pylibjpeg-turbo/extern/libjpeg-turbo

          is not an existing non-empty directory

I added .gitmodules in the pyproject.toml:

[tool.py-build-cmake.sdist]
include = [".gitmodules", "CMakeLists.txt", "extern/libjpeg-turbo"]

And I have

      - uses: actions/checkout@v3
        with:
          submodules: true

in the workflow, but it is still not working, somehow that is not getting through to the tmp folders. I may try GIT_REPOSITORY and GIT_TAG in ExternalProject_Add instead of SOURCE_DIR, although I did read in a couple of places that using submodules was generally easier.

tttapa commented 11 months ago

This works:

darcymason commented 11 months ago

This works:

https://github.com/tttapa/py-build-cmake-libjpeg-turbo

https://github.com/tttapa/py-build-cmake-libjpeg-turbo/actions/runs/5686510361

That is truly awesome! Thank you! At first glance I didn't see much difference, but then

      - uses: actions/checkout@v3
        with:
          submodules: recursive

Here recursive instead of true, which makes sense. I'll keep digging until I understand all the differences.

Closing the issue though, clearly solved since you have posted a complete example 🚀

tttapa / py-build-cmake

ctypes lib build with ExternalProject #16