Issue: Improvement over compiled shared object.

[hwyao]

Description

When we want to integrate the LCQPow solver to some other packages e.g the ROS packages, we expect that we can use the static or shared object from this solver. However when we try to do so, we could meet multiple problems.

Expected: We use one LCQPow shared object and include its header files in our own CMakeLists.txt and everything finished. Actual: The linking is difficult, we have to link 3 objects in a correct way, otherwise we will meet many problems.

[hwyao]

Detail

Prepare function

Suppose that we have two files use_LCQPow.cpp that is same as OptimizeOnCircle.cpp, and use_LCQPow_lib.cpp with same usage but not in main() but a functionCall(). It also includes a corresponding header file use_LCQPow_lib.hpp.

#ifndef USE_LCQPOW_LIB_HPP
#define USE_LCQPOW_LIB_HPP

int functionCall(void);

#endif // USE_LCQPOW_LIB_HPP

We set folder structure:

/include
 use_LCQPow_lib.hpp
/src
 use_LCQPow_lib.cpp
 use_LCQPow.cpp
CMakeLists.txt

Minimal working example 0:

We can clone the project, use add_subdirectory and then follow the same code in CMakeLists.txt for example. Which basically works but not for ROS because gtest will be twice included and therefore is problematic.

Therefore, the following examples will consider to use ExternalProject module from CMake.

[hwyao]

Minimal working example 1:

CMake files

cmake_minimum_required(VERSION 3.24)
project(Example5-ExternalProject)

include(ExternalProject)
set(LCQPow_NAME "LCQPow")  
ExternalProject_Add(${LCQPow_NAME}
    GIT_REPOSITORY "https://github.com/hallfjonas/LCQPow"
    GIT_TAG "main"
    INSTALL_COMMAND ""
) # We this as an example of downloading the repo with this command

# just two variables as a default result of ExternalProject_Add
set(LCQPow_SOURCE_DIR ${CMAKE_BINARY_DIR}/${LCQPow_NAME}-prefix/src/${LCQPow_NAME})
set(LCQPow_BUILD_DIR ${CMAKE_BINARY_DIR}/${LCQPow_NAME}-prefix/src/${LCQPow_NAME}-build)

# import the shared library fromt the build folder
add_library(qpOASES_lib SHARED IMPORTED)
set_target_properties(qpOASES_lib PROPERTIES
  IMPORTED_LOCATION ${LCQPow_BUILD_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}qpOASES${CMAKE_SHARED_LIBRARY_SUFFIX}
  # INTERFACE_INCLUDE_DIRECTORIES ${LCQPow_BUILD_DIR}/external/src/qpoases/include 
)

add_library(osqp_lib SHARED IMPORTED)
set_target_properties(osqp_lib PROPERTIES
  IMPORTED_LOCATION ${LCQPow_BUILD_DIR}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}osqp${CMAKE_SHARED_LIBRARY_SUFFIX}
  # INTERFACE_INCLUDE_DIRECTORIES ${LCQPow_BUILD_DIR}/external/src/osqp/include
)

add_library(LCQPow_lib SHARED IMPORTED)
set_target_properties(LCQPow_lib PROPERTIES
  IMPORTED_LOCATION ${LCQPow_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}lcqpow${CMAKE_SHARED_LIBRARY_SUFFIX}
  # INTERFACE_INCLUDE_DIRECTORIES ${LCQPow_SOURCE_DIR}/include
)

#  we comment out INTERFACE_INCLUDE_DIRECTORIES because when running cmake, the folder does not exists.
#  So there will be always an ugly warning.
include_directories(
    ${LCQPow_SOURCE_DIR}/include
    ${LCQPow_BUILD_DIR}/external/src/qpoases/include
    ${LCQPow_BUILD_DIR}/external/src/osqp/include
)

# executable test
add_executable(${PROJECT_NAME}
    src/use_LCQPow.cpp
)
target_link_libraries(${PROJECT_NAME} PRIVATE
  LCQPow_lib
  osqp_lib
  qpOASES_lib
) # notice: dependency is qpOASES_lib & osqp_lib->LCQPow_lib, so the shared object depending on others goes before its dependency

# library test
add_library(${PROJECT_NAME}_lib SHARED
    src/use_LCQPow_lib.cpp
)
target_include_directories(${PROJECT_NAME}_lib
    PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
)
target_link_libraries(${PROJECT_NAME}_lib
  "$<LINK_GROUP:RESCAN,qpOASES_lib,osqp_lib,LCQPow_lib>"
) # the generator way of resolving potential dependency between different libs.
# require quite a new CMake version as shown above. Otherwise we can lower the version number there.

Running result

Notice here: if you have wrong sequence of the dependency library, it could cause error. This might be the first tricky place for people new to CMake.

For executable, by running build you will get a series of error whille linking：

/usr/bin/ld: LCQPow-prefix/src/LCQPow-build/lib/liblcqpow.so: undefined reference to `qpOASES::QProblem::~QProblem()'
...

For library, it will be slient, but you can found the problem by using ldd command and -r to check its dependency and symbol table.

> ldd -r ./libExample5-ExternalProject_lib.so 
        linux-vdso.so.1 (0x00007ffd1c5df000)
        liblcqpow.so => <root folder>/build/LCQPow-prefix/src/LCQPow-build/lib/liblcqpow.so (0x00007f24fb317000)
        libosqp.so => <root folder>/build/LCQPow-prefix/src/LCQPow-build/lib/libosqp.so (0x00007f24fb2fa000)
        libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f24fb0de000)
        libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f24faf8f000)
        libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f24faf74000)
        libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f24fad80000)
        libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f24fad7a000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f24fb340000)
undefined symbol: _ZTVN7qpOASES12SymSparseMatE  (<root folder>/build/LCQPow-prefix/src/LCQPow-build/lib/liblcqpow.so)
...

[hwyao]

Minimal working example 2:

The passing argument usage

If we watch how your repo implements the linking, we can notice that you are passing A plain library name with A link flag (target_link_libraries reference).

But actually it well get more complex by using this way because we need to dig deeper into the linker

CMake file

cmake_minimum_required(VERSION 3.24)
project(Example5-ExternalProject)

include(ExternalProject)

# Now we manually clone the repo and set the directory /submodule/LCQPow. This can be a good example of using this.
set(LCQPow_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/submodule/LCQPow)
set(LCQPow_BUILD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/submodule/LCQPow/build)

ExternalProject_Add(LCQPow
    SOURCE_DIR ${LCQPow_SOURCE_DIR}
    BINARY_DIR ${LCQPow_BUILD_DIR}
    INSTALL_COMMAND ""
) 

set(find_lib "-Wl,-rpath=${LCQPow_BUILD_DIR}/lib -L${LCQPow_BUILD_DIR}/lib")

set(qpoases_lib "-lqpOASES")
set(qpoases_include "${LCQPow_BUILD_DIR}/external/src/qpoases/include")

set(osqp_lib "-losqp")
set(osqp_include "${LCQPow_BUILD_DIR}/external/src/osqp/include")

set(LCQPow_lib "-llcqpow")
set(LCQPow_include "${LCQPow_SOURCE_DIR}/include")

include_directories( 
    PRIVATE ${LCQPow_include}
    SYSTEM ${qpoases_include}
    SYSTEM ${osqp_include}
)

# executable test
add_executable(${PROJECT_NAME}
    src/use_LCQPow.cpp
)
target_link_libraries(${PROJECT_NAME}
    PUBLIC ${find_lib} ${LCQPow_lib}
    PRIVATE ${qpoases_lib} ${osqp_lib}
) # also be careful about the dependency between libraries.

# library test
add_library(${PROJECT_NAME}_lib SHARED
    src/use_LCQPow_lib.cpp
)
target_include_directories(${PROJECT_NAME}_lib
    PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
) # because now we cannot 
target_link_libraries(${PROJECT_NAME}_lib PUBLIC 
    ${find_lib} "-Wl,--start-group" ${qpoases_lib} ${osqp_lib} ${LCQPow_lib} "-Wl,--end-group"
) # This will be explained later.

Some explanation

We can take the reference: Directory Options (Using the GNU Compiler Collection (GCC)) (-L) Link Options (Using the GNU Compiler Collection (GCC)) (-Wl) ld(1): GNU linker - Linux man page (-rpath, --start-group, --end-group)

-L: Tell gcc where to search the linked library
-Wl: Pass arguments to actual linker, in our case mostly /usr/bin/ld
-rpath: Add a directory to the runtime library search path.
--start-group, --end-group: something that also help people to resolve the dependency with linker.

In my system when we check /\<rootfolder>/build/CMakeFiles/Example5-ExternalProject.dir/link.txt, the linking command is actually with /usr/bin/c++. But the parameters are mostly same here.

Running result

This -rpath parameter is important (which does not exists in your CMakeLists as instruction). This is also a very tricky place.

If we miss out the -rpath, the building is still successful but we neither run the executable nor using the library. The ldd command will deliver:

> ldd ./Example5-ExternalProject 
        linux-vdso.so.1 (0x00007ffdad550000)
        liblcqpow.so => not found
        libqpOASES.so => not found
        libosqp.so => not found
        libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f9101e64000)
        libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f9101d15000)
        libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f9101cfa000)
        libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f9101b06000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f9102089000)

For which, the invoked dynamic loader knows who it should link to but cannot find where it is.

I also think this is somehow connected to your instruction in README:

The MATLAB interface is built automatically if matlab is successfully detected by CMake. Make sure that your linker can locate the created libraries, e.g. by exporting the library path in the same shell as the one you call matlab in: $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:\<LCQPow-dir>/build/lib $ matlab

which is actually a bit strange for people to use it.

more about -rpath

There is another way to see this problem. By watching the dynamic section of the executable / library we can see:

Without -rpath

> readelf -d Example5-ExternalProject | head -20

Dynamic section at offset 0x4c50 contains 33 entries:
  Tag        Type                         Name/Value
 0x0000000000000001 (NEEDED)             Shared library: [liblcqpow.so]
 0x0000000000000001 (NEEDED)             Shared library: [libqpOASES.so]
 0x0000000000000001 (NEEDED)             Shared library: [libosqp.so]
 0x0000000000000001 (NEEDED)             Shared library: [libstdc++.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libm.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libgcc_s.so.1]
 0x0000000000000001 (NEEDED)             Shared library: [libc.so.6]
 0x000000000000000c (INIT)               0x2000
...

With -rpath

readelf -d Example5-ExternalProject | head -20

Dynamic section at offset 0x4c40 contains 34 entries:
  Tag        Type                         Name/Value
 0x0000000000000001 (NEEDED)             Shared library: [liblcqpow.so]
 0x0000000000000001 (NEEDED)             Shared library: [libqpOASES.so]
 0x0000000000000001 (NEEDED)             Shared library: [libosqp.so]
 0x0000000000000001 (NEEDED)             Shared library: [libstdc++.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libm.so.6]
 0x0000000000000001 (NEEDED)             Shared library: [libgcc_s.so.1]
 0x0000000000000001 (NEEDED)             Shared library: [libc.so.6]
 0x000000000000001d (RUNPATH)            Library runpath: [<root path>/submodule/LCQPow/build/lib]
 0x000000000000000c (INIT)               0x2000
...

The (RUNPATH) mark link will help dynamic linker to search for the library from yourself.

You can also try tracing the library linking in runtime to see how libraries are searched with or without correct (RUNPATH).

strace ./Example5-ExternalProject 2>&1 | grep liblcqpow

Conclusion

Actually passing argument to linker is not suggested because the arguments are really system and environment dependent, which breaks our target of using CMake.

[hwyao]

Solution

I am going to pull request something to make everything easier. The pull request is under working but would be quick.