Undefined References and Relative Library Dependency Paths in CMake Configuration for mipsel

Describe the bug

When attempting to build the sample project, _basic_pubsub, using built shared libraries for _mipsel_24kcmusl, I am getting undefined references to symbols in libaws-c-io.so.

Initially, I tried to modify the CMakeList.txt for the sample project to include my tool chain. However, it has been a nightmare due to the libraries being built with relative paths to libraries each of them are dependent on, and after running make install to my tool chain staging directory, they don't seem to follow the same relative paths. This has forced me to try create a custom mipsel-openwrt-linux-g++ command to build the program.

I don't have any issues when I build and install the libraries for my host system (WSL in this case), where it builds with no issues; it sounds like I'm missing a cmake flag/variable.

I ran the following to build the shared libraries (with modified CMakeLists.txt to use the tool chain:

cmake -DCMAKE_INSTALL_PREFIX="/path/to/staging_dir" -DCMAKE_BUILD_TYPE="Release"  .

My CMakeLists.txt for basic_pub_sub in ./samples/pub_sub/basic_pub_sub:

cmake_minimum_required(VERSION 3.1)

set(staging /path/to/staging_dir)

set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR mipsel_24kc_gcc)

set(CMAKE_SYSROOT ${staging}/toolchain-mipsel_24kc_gcc-8.4.0_musl)
set(CMAKE_STAGING_PREFIX ${staging}/toolchain-mipsel_24kc_gcc-8.4.0_musl)

set(tools ${staging}/toolchain-mipsel_24kc_gcc-8.4.0_musl/bin)
set(CMAKE_C_COMPILER ${tools}/mipsel-openwrt-linux-gcc)
set(CMAKE_CXX_COMPILER ${tools}/mipsel-openwrt-linux-g++)

set(CMAKE_FIND_ROOT_PATH ../../../install/lib ${staging}/target-mipsel_24kc_musl)

set(CMAKE_INSTALL_RPATH ../../../install/lib)
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE BOTH)

project(basic-pub-sub CXX)

file(GLOB SRC_FILES
       "*.cpp"
       "../../utils/CommandLineUtils.cpp"
       "../../utils/CommandLineUtils.h"
)

add_executable(${PROJECT_NAME} ${SRC_FILES})

set_target_properties(${PROJECT_NAME} PROPERTIES
    CXX_STANDARD 11)

# Set warnings
if (MSVC)
    target_compile_options(${PROJECT_NAME} PRIVATE /W4 /WX /wd4068)
else ()
    target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wno-long-long -pedantic -Werror)
endif ()

find_package(
    aws-crt-cpp
    REQUIRED
)

# try and force the correct order
target_link_libraries(${PROJECT_NAME} 
    AWS::aws-c-common
    AWS::aws-c-cal
    AWS::aws-c-io
    AWS::aws-c-http
    AWS::aws-c-mqtt
    AWS::aws-crt-cpp
    AWS::aws-c-event-stream
    AWS::aws-c-s3
    AWS::aws-c-auth
    AWS::aws-c-compression
    AWS::aws-c-sdkutils
    AWS::aws-checksums
)

Compiler command script for _basic_pubsub in ./samples/pub_sub/basic_pub_sub:

staging_dir=/path/to/staging_dir
dir=../../..
libs=$dir/crt/aws-crt-cpp

mipsel-openwrt-linux-g++ \
    -I../../utils \
    -I$dir/crt/aws-crt-cpp/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-common/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-common/generated/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-io/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-mqtt/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-cal/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-http/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-auth/include \
    -I$dir/crt/aws-crt-cpp/crt/aws-c-sdkutils/include \
    -I$dir/crt/aws-crt-cpp/generated/include \
    -o basic_pub_sub main.cpp ../../utils/CommandLineUtils.cpp \
    -L$staging_dir/target-mipsel_24kc_musl/usr/lib \
    -L$libs \
    -L$libs/crt \
    -L$libs/crt/aws-c-auth \
    -L$libs/crt/aws-c-cal \
    -L$libs/crt/aws-c-common \
    -L$libs/crt/aws-c-compression \
    -L$libs/crt/aws-c-http \
    -L$libs/crt/aws-c-io \
    -L$libs/crt/aws-c-mqtt \
    -L$libs/crt/aws-c-s3 \
    -L$libs/crt/aws-c-sdkutils \
    -L$libs/crt/aws-checksums \
    -Wl,-rpath-link,$libs:$libs/crt:$libs/crt/aws-c-auth:$libs/crt/aws-c-cal:$libs/crt/aws-c-common:$libs/crt/aws-c-compression:$libs/crt/aws-c-http:$libs/crt/aws-c-io:$libs/crt/aws-c-mqtt:$libs/crt/aws-c-s3:$libs/crt/aws-c-sdkutils:$libs/crt/aws-checksums \
    -lssl -lcrypto \
    -laws-c-common -laws-c-cal -laws-c-http -laws-c-mqtt -laws-c-io -laws-crt-cpp

For context, here are the paths to the libraries libaws-crt-cpp.so depends on:

readelf -d libaws-crt-cpp.so

Dynamic section at offset 0x168 contains 39 entries:
  Tag        Type                         Name/Value
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-mqtt/libaws-c-mqtt.so]
 0x00000001 (NEEDED)                     Shared library: [../../lib/libaws-c-event-stream.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-s3/libaws-c-s3.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-auth/libaws-c-auth.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-http/libaws-c-http.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-io/libaws-c-io.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-compression/libaws-c-compression.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-cal/libaws-c-cal.so]
 0x00000001 (NEEDED)                     Shared library: [libcrypto.so.1.1]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-sdkutils/libaws-c-sdkutils.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-checksums/libaws-checksums.so]
 0x00000001 (NEEDED)                     Shared library: [crt/aws-c-common/libaws-c-common.so]
 0x00000001 (NEEDED)                     Shared library: [libstdc++.so.6]
 0x00000001 (NEEDED)                     Shared library: [libgcc_s.so.1]
 0x00000001 (NEEDED)                     Shared library: [libc.so]
 ...

When I run my compiler command script, the undefined symbols appear to be part of libaws-c-io.so:

aws_tls_init_static_state
aws_socket_get_bound_address
aws_socket_start_accept
aws_socket_subscribe_to_readable_events
aws_socket_assign_to_event_loop
aws_socket_get_event_loop
aws_tls_is_alpn_available
aws_tls_client_ctx_new
aws_socket_write
aws_tls_client_handler_start_negotiation
aws_socket_connect
aws_event_loop_new_default_with_options
aws_tls_handler_protocol
aws_tls_client_handler_new
aws_socket_listen
aws_socket_clean_up
aws_socket_bind
aws_socket_init
aws_shared_library_clean_up
aws_socket_stop_accept
aws_socket_close
aws_tls_server_ctx_new
aws_socket_is_open
aws_tls_clean_up_static_state
aws_socket_read
aws_shared_library_init
aws_shared_library_find_function
aws_default_dns_resolve
aws_tls_server_handler_new

Expected Behavior

Successfully compile basic_pub_sub via cmake with specified cross compiler settings. Also, being able to compile without cmake (my main project doesn't use cmake) and with shared libraries.

Current Behavior

Undefined symbols from libaws-c-io.so when compiling, where it appears linking order and directories are correct.

Shared libraries are built with relative paths to the libraries they depend on, which doesn't match the path structure after installing the libraries.

Reproduction Steps

I have provided code and commands that I have used when trying to build the libraries and sample program.

The CMakeLists.txt files are the only files I have modified, along with the bash script I used to wrapper my compiler command.

The tool chain is for the Teltonika RUT956 that uses OpenWRT for mispel (mipsel_24kc_gcc-8.4.0_musl).

Possible Solution

More information on cmake variables regarding cross compiling.

Not to use relative paths for library dependency.

Additional Information/Context

The main point is that there should not be undefined references after ensuring library order and directories are correct, regardless of it built is via cmake or not. Also, the relative paths to libraries makes it a headache to implement into a custom project.

I really hope to use this in a commercial distribute system for SCADA systems. The AWS IoT framework seems to be perfect for our application.

I would really appreciate any feedback/guidance, from this ticket, to be able to compile and run our project.

Many thanks.

SDK version used

1.30.3

Environment details (OS name and version, etc.)

host: WSL 2 (Ubuntu 22.04.2 LTS) - target: mipsel_24kc_musl

aws / aws-iot-device-sdk-cpp-v2