build bcc failed on Ubuntu with LLVM18

[muzinian]

When I was building bcc using LLVM18 on Ubuntu, I encountered various issues. The system version I was using is as follows: lsb_release -a

No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 22.04.4 LTS
Release:        22.04
Codename:       jammy

cc -v

Ubuntu clang version 18.1.8 (++20240731024944+3b5b5c1ec4a3-1~exp1~20240731145000.144)
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/bin
Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/11
Selected GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/11
Candidate multilib: .;@m64
Selected multilib: .;@m64

c++ -v

Ubuntu clang version 18.1.8 (++20240731024944+3b5b5c1ec4a3-1~exp1~20240731145000.144)
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/bin
Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/11
Selected GCC installation: /usr/bin/../lib/gcc/x86_64-linux-gnu/11
Candidate multilib: .;@m64
Selected multilib: .;@m64

At first, I made modifications to CMakeLists.txt based on the results of executing cmake ... I added

cmake_policy(set CMP0075 NEW)

within the if logic that checks for versions greater than 3.12.0, in order to avoid the CMP0075 issue. Additionally, to output the results of certain options, I added the following print statements at the end:

message(STATUS "C Compiler: ${CMAKE_C_COMPILER}")
message(STATUS "C++ Compiler: ${CMAKE_CXX_COMPILER}")
message(STATUS "Linker: ${CMAKE_LINKER}")
message(STATUS "C Compiler Flags: ${CMAKE_C_FLAGS}")
message(STATUS "C++ Compiler Flags: ${CMAKE_CXX_FLAGS}")
message(STATUS "Linker Flags: ${CMAKE_EXE_LINKER_FLAGS}")

And the result output of cmake .. is:

-- The C compiler identification is Clang 18.1.8
-- The CXX compiler identification is Clang 18.1.8
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/bin/cc - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /usr/bin/c++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Latest recognized Git tag is v0.31.0
-- Git HEAD is 3130fe8c6e0cdb08f41ecab1186cb7da38bdd9e2
-- Revision is 0.31.0+3130fe8c (major 0, minor 31, patch 0)
-- Performing Test HAVE_NO_PIE_FLAG
-- Performing Test HAVE_NO_PIE_FLAG - Success
-- Performing Test HAVE_REALLOCARRAY_SUPPORT
-- Performing Test HAVE_REALLOCARRAY_SUPPORT - Success
-- Kernel release: 5.15.0-119-generic
-- Kernel headers: /usr/src/linux-headers-5.15.0-119-generic
-- Performing Test HAVE_FFI_CALL
-- Performing Test HAVE_FFI_CALL - Success
-- Found FFI: /usr/lib/x86_64-linux-gnu/libffi.so  
-- Looking for histedit.h
-- Looking for histedit.h - found
-- Found LibEdit: /usr/include (found version "2.11") 
-- Performing Test Terminfo_LINKABLE
-- Performing Test Terminfo_LINKABLE - Success
-- Found Terminfo: /usr/lib/x86_64-linux-gnu/libtinfo.so  
-- Found ZLIB: /usr/lib/x86_64-linux-gnu/libz.so (found version "1.2.11") 
-- Found zstd: /usr/lib/x86_64-linux-gnu/libzstd.so  
-- Found LibXml2: /usr/lib/x86_64-linux-gnu/libxml2.so (found version "2.9.13") 
-- Found CURL: /usr/lib/x86_64-linux-gnu/libcurl.so (found version "7.81.0")  
-- Found LLVM: /usr/lib/llvm-18/include 18.1.8 /usr/lib/llvm-18/lib (Use LLVM_ROOT envronment variable for another version of LLVM)
-- Found BISON: /usr/bin/bison (found version "3.8.2") 
-- Found FLEX: /usr/bin/flex (found version "2.6.4") 
-- Found LibElf: /usr/lib/x86_64-linux-gnu/libelf.so  
-- Performing Test ELF_GETSHDRSTRNDX
-- Performing Test ELF_GETSHDRSTRNDX - Success
-- Found LibDebuginfod: /usr/lib/x86_64-linux-gnu/libdebuginfod.so  
-- Found LibLzma: /usr/lib/x86_64-linux-gnu/liblzma.so  
-- Found LuaJIT: /usr/lib/x86_64-linux-gnu/libluajit-5.1.a;/usr/lib/x86_64-linux-gnu/libdl.a;/usr/lib/x86_64-linux-gnu/libm.so  
-- C Compiler: /usr/bin/cc
-- C++ Compiler: /usr/bin/c++
-- Linker: /usr/bin/ld.lld
-- C Compiler Flags:  -Wall -stdlib=libc++
-- C++ Compiler Flags:  -Wall  -stdlib=libc++
-- Linker Flags: 
-- Configuring done
-- Generating done

After these operations, I performed the cmake ..and make commands according to the documentation, but ultimately encountered the following error:

In file included from /xxx/bcc/src/cc/bcc_syms.cc:36:
In file included from /xxx/bcc/src/cc/common.h:19:
In file included from /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/memory:77:
In file included from /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr.h:53:
In file included from /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:62:
In file included from /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/concurrence.h:34:
In file included from /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/exception:153:
In file included from /usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/exception_ptr.h:38:
/usr/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/cxxabi_init_exception.h:70:7: error: functions that differ only in their return type cannot be overloaded
   69 |       __cxa_refcounted_exception*
      |       ~~~~~~~~~~~~~~~~~~~~~~~~~~~
   70 |       __cxa_init_primary_exception(void *object, std::type_info *tinfo,
      |       ^
/usr/lib/llvm-18/include/cxxabi.h:51:1: note: previous declaration is here
   50 | extern _LIBCXXABI_FUNC_VIS __cxa_exception*
      |                            ~~~~~~~~~~~~~~~~
   51 | __cxa_init_primary_exception(void* object, std::type_info* tinfo, void(_LIBCXXABI_DTOR_FUNC* dest)(void*)) throw();
      | ^
1 warning and 1 error generated

Based on the error message, I suspected there might be a conflict in the libraries (I don't have much knowledge of C/C++ itself, this is just a simple guess based on google and ChatGPT). Therefore, I modified the CMakeLists.txt file, making changes to the following parts:

set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -stdlib=libc++")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall ${CXX_ISYSTEM_DIRS} -stdlib=libc++")

It would then report a massive number of undefined reference errors at the stage of [ 39%] Linking CXX executable CGroupTest.For example:

usr/bin/ld: ../../src/cc/libbcc.a(bpf_module.cc.o): in function `ebpf::BPFModule::dump_ir(llvm::Module&)':
bpf_module.cc:(.text+0x112d): undefined reference to `llvm::PassBuilder::PassBuilder(llvm::TargetMachine*, llvm::PipelineTuningOptions, std::__1::optional<llvm::PGOOptions>, llvm::PassInstrumentationCallbacks*)'
/usr/bin/ld: bpf_module.cc:(.text+0x1203): undefined reference to `llvm::PrintModulePass::PrintModulePass(llvm::raw_ostream&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, bool, bool)'
/usr/bin/ld: ../../src/cc/libbcc.a(bpf_module.cc.o): in function `ebpf::BPFModule::run_pass_manager(llvm::Module&)':
bpf_module.cc:(.text+0x179a): undefined reference to `llvm::PassBuilder::PassBuilder(llvm::TargetMachine*, llvm::PipelineTuningOptions, std::__1::optional<llvm::PGOOptions>, llvm::PassInstrumentationCallbacks*)'
/usr/bin/ld: bpf_module.cc:(.text+0x18e8): undefined reference to `llvm::PrintModulePass::PrintModulePass(llvm::raw_ostream&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, bool, bool)'
/usr/bin/ld: ../../src/cc/libbcc.a(bpf_module.cc.o): in function `ebpf::BPFModule::finalize()':
bpf_module.cc:(.text+0x39e9): undefined reference to `llvm::EngineBuilder::EngineBuilder(std::__1::unique_ptr<llvm::Module, std::__1::default_delete<llvm::Module> >)'

The above is just a part of the entire error output, which is approximately 30,000 lines long and mostly consists of various undefined reference errors. What confuses me is that the previous cmake output already shows that the choosenlinker is ld.lld, but here it seems the linker is still /usr/bin/ld.And even if I only modified the CMAKE_CXX_FLAGS part, the same error was reported.

[muzinian]

And the cmake version I use is 3.22.1. I installed LLVM according to the Automatic installation script on the LLVM apt package page and manually created a symbolic link for ld.lld to ld.lld-18(version is Ubuntu LLD 18.1.8 (compatible with GNU linkers)). The ld version is GNU ld (GNU Binutils for Ubuntu) 2.38

[ShawnZhong]

I am able to compile BCC with LLVM 18.1.3. Here is my output for cmake if that helps:

cmake output

``` CC=clang CXX=clang++ cmake -B build . CMake Deprecation Warning at CMakeLists.txt:3 (cmake_minimum_required): Compatibility with CMake < 3.5 will be removed from a future version of CMake. Update the VERSION argument value or use a ... suffix to tell CMake that the project does not need compatibility with older versions. -- The C compiler identification is Clang 18.1.3 -- The CXX compiler identification is Clang 18.1.3 -- Detecting C compiler ABI info -- Detecting C compiler ABI info - done -- Check for working C compiler: /usr/bin/clang - skipped -- Detecting C compile features -- Detecting C compile features - done -- Detecting CXX compiler ABI info -- Detecting CXX compiler ABI info - done -- Check for working CXX compiler: /usr/bin/clang++ - skipped -- Detecting CXX compile features -- Detecting CXX compile features - done -- Currently on Git tag v0.31.0 -- Git HEAD is -- Revision is 0.31.0 (major 0, minor 31, patch 0) -- Performing Test HAVE_NO_PIE_FLAG -- Performing Test HAVE_NO_PIE_FLAG - Success -- Performing Test HAVE_REALLOCARRAY_SUPPORT -- Performing Test HAVE_REALLOCARRAY_SUPPORT - Success -- Kernel release: 6.10.6-orbstack-00249-g92ad2848917c -- Kernel headers: KERNELHEADERS_DIR-NOTFOUND -- Performing Test HAVE_FFI_CALL -- Performing Test HAVE_FFI_CALL - Success -- Found FFI: /usr/lib/aarch64-linux-gnu/libffi.so -- Could NOT find LibEdit (missing: LibEdit_INCLUDE_DIRS LibEdit_LIBRARIES) -- Performing Test Terminfo_LINKABLE -- Performing Test Terminfo_LINKABLE - Success -- Found Terminfo: /usr/lib/aarch64-linux-gnu/libtinfo.so -- Found ZLIB: /usr/lib/aarch64-linux-gnu/libz.so (found version "1.3") -- Found zstd: /usr/lib/aarch64-linux-gnu/libzstd.so -- Found LibXml2: /usr/lib/aarch64-linux-gnu/libxml2.so (found version "2.9.14") -- Could NOT find CURL (missing: CURL_LIBRARY CURL_INCLUDE_DIR) -- Found LLVM: /usr/lib/llvm-18/include 18.1.3 (Use LLVM_ROOT envronment variable for another version of LLVM) -- Could NOT find BISON (missing: BISON_EXECUTABLE) -- Could NOT find FLEX (missing: FLEX_EXECUTABLE) -- Found LibElf: /usr/lib/aarch64-linux-gnu/libelf.so -- Performing Test ELF_GETSHDRSTRNDX -- Performing Test ELF_GETSHDRSTRNDX - Success -- Could NOT find LibDebuginfod (missing: LIBDEBUGINFOD_LIBRARIES LIBDEBUGINFOD_INCLUDE_DIRS) -- Could NOT find LibLzma (missing: LIBLZMA_LIBRARIES LIBLZMA_INCLUDE_DIRS) -- Could NOT find LuaJIT (missing: LUAJIT_LIBRARIES LUAJIT_INCLUDE_DIR) -- Configuring done (1.3s) -- Generating done (0.2s) ```

[muzinian]

I am able to compile BCC with LLVM 18.1.3. Here is my output for cmake if that helps:

cmake output

CC=clang CXX=clang++ cmake -B build .
CMake Deprecation Warning at CMakeLists.txt:3 (cmake_minimum_required):
  Compatibility with CMake < 3.5 will be removed from a future version of
  CMake.

  Update the VERSION argument <min> value or use a ...<max> suffix to tell
  CMake that the project does not need compatibility with older versions.

-- The C compiler identification is Clang 18.1.3
-- The CXX compiler identification is Clang 18.1.3
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /usr/bin/clang - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /usr/bin/clang++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Currently on Git tag v0.31.0
-- Git HEAD is 
-- Revision is 0.31.0 (major 0, minor 31, patch 0)
-- Performing Test HAVE_NO_PIE_FLAG
-- Performing Test HAVE_NO_PIE_FLAG - Success
-- Performing Test HAVE_REALLOCARRAY_SUPPORT
-- Performing Test HAVE_REALLOCARRAY_SUPPORT - Success
-- Kernel release: 6.10.6-orbstack-00249-g92ad2848917c
-- Kernel headers: KERNELHEADERS_DIR-NOTFOUND
-- Performing Test HAVE_FFI_CALL
-- Performing Test HAVE_FFI_CALL - Success
-- Found FFI: /usr/lib/aarch64-linux-gnu/libffi.so  
-- Could NOT find LibEdit (missing: LibEdit_INCLUDE_DIRS LibEdit_LIBRARIES) 
-- Performing Test Terminfo_LINKABLE
-- Performing Test Terminfo_LINKABLE - Success
-- Found Terminfo: /usr/lib/aarch64-linux-gnu/libtinfo.so  
-- Found ZLIB: /usr/lib/aarch64-linux-gnu/libz.so (found version "1.3")  
-- Found zstd: /usr/lib/aarch64-linux-gnu/libzstd.so  
-- Found LibXml2: /usr/lib/aarch64-linux-gnu/libxml2.so (found version "2.9.14") 
-- Could NOT find CURL (missing: CURL_LIBRARY CURL_INCLUDE_DIR) 
-- Found LLVM: /usr/lib/llvm-18/include 18.1.3 (Use LLVM_ROOT envronment variable for another version of LLVM)
-- Could NOT find BISON (missing: BISON_EXECUTABLE) 
-- Could NOT find FLEX (missing: FLEX_EXECUTABLE) 
-- Found LibElf: /usr/lib/aarch64-linux-gnu/libelf.so  
-- Performing Test ELF_GETSHDRSTRNDX
-- Performing Test ELF_GETSHDRSTRNDX - Success
-- Could NOT find LibDebuginfod (missing: LIBDEBUGINFOD_LIBRARIES LIBDEBUGINFOD_INCLUDE_DIRS) 
-- Could NOT find LibLzma (missing: LIBLZMA_LIBRARIES LIBLZMA_INCLUDE_DIRS) 
-- Could NOT find LuaJIT (missing: LUAJIT_LIBRARIES LUAJIT_INCLUDE_DIR) 
-- Configuring done (1.3s)
-- Generating done (0.2s)

According to the documentation, bison and flex are required. Why is it that in your output, bison and flex were not found, yet the build was still successful? Did I misunderstand something? Please point out my mistake, thank you.