Open arcivanov opened 2 years ago
I guess the workaround would be to force llvm-tblgen
and clang-tblgen
to link statically, but I can't seem to find a way to do that.
The problem is reproducible for the stock LLVM 2-stage bootstrap build as well, so stage1 install location is NOT relevant.
Here's the example:
twostage.cmake
# Common
set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld;lldb" CACHE STRING "")
set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
set(CLANG_BOOTSTRAP_PASSTHROUGH "CMAKE_INSTALL_PREFIX;PYTHON_HOME;PYTHON_EXECUTABLE;Python3_EXECUTABLE;LLVM_TARGETS_TO_BUILD;LLVM_PARALLEL_COMPILE_JOBS;LLVM_PARALLEL_LINK_JOBS" CACHE STRING "")
set(LLVM_PARALLEL_LINK_JOBS 1 CACHE STRING "")
# Stage 1
set(LLVM_BUILD_TOOLS OFF CACHE BOOL "")
set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
# Stage 2
set(BOOTSTRAP_LLVM_BUILD_TOOLS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_DUMP ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_CLANG_LINK_CLANG_DYLIB ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LTO THIN CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_TERMINFO OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBC ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_ENABLE_LIBCXX ON CACHE BOOL "")
set(BOOTSTRAP_CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_LINKER lld CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
set(BOOTSTRAP_CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "")
set(BOOTSTRAP_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "")
set(BOOTSTRAP_SANITIZER_CXX_ABI libc++ CACHE STRING "")
set(BOOTSTRAP_SANITIZER_TEST_CXX libc++ CACHE STRING "")
set(BOOTSTRAP_LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "")
set(BOOTSTRAP_LIBCXX_USE_COMPILER_RT YES CACHE BOOL "")
set(BOOTSTRAP_LIBCXXABI_USE_COMPILER_RT YES CACHE BOOL "")
set(BOOTSTRAP_LIBCXXABI_USE_LLVM_UNWINDER YES CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_BINUTILS_SYMLINKS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_CCTOOLS_SYMLINKS ON CACHE BOOL "")
set(BOOTSTRAP_LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")
set(BOOTSTRAP_CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
set(BOOTSTRAP_CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG " CACHE STRING "")
set(BOOTSTRAP_CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
set(BOOTSTRAP_CMAKE_ASM_FLAGS_RELWITHDEBINFO "-O3 -glldb -DNDEBUG" CACHE STRING "")
$ find . -name \*-tblgen
./tools/clang/stage2-bins/bin/llvm-tblgen
./bin/llvm-tblgen
./bin/clang-tblgen
FAILED: include/llvm/IR/IntrinsicsR600.h /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/include/llvm/IR/IntrinsicsR600.h
cd /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins && /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/llvm-tblgen -gen-intrinsic-enums -intrinsic-prefix=r600 -I /home/user/Documents/src/llvm/llvm-project/llvm/include/llvm/IR -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/include -I/home/user/Documents/src/llvm/llvm-project/llvm/include /home/user/Documents/src/llvm/llvm-project/llvm/include/llvm/IR/Intrinsics.td --write-if-changed -o include/llvm/IR/IntrinsicsR600.h -d include/llvm/IR/IntrinsicsR600.h.d
/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/llvm-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory
Actually the same thing happens with lldb-tblgen once you get to it.
[12/15] Performing build step for 'stage2'
[3220/4875] Building CommandOptions.inc...
FAILED: tools/lldb/source/Commands/CommandOptions.inc /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/source/Commands/CommandOptions.inc
cd /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins && /home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/lldb-tblgen -gen-lldb-option-defs -I /home/user/Documents/src/llvm/llvm-project/lldb/source/Commands -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/source -I/home/user/Documents/src/llvm/llvm-project/lldb/include -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/include -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/include -I/home/user/Documents/src/llvm/llvm-project/llvm/include -I/home/user/.pyenv/versions/3.9.9/include/python3.9 -I/home/user/Documents/src/llvm/llvm-project/clang/include -I/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/tools/lldb/../clang/include -I/usr/include -I/usr/include/libxml2 -I/home/user/Documents/src/llvm/llvm-project/lldb/source/. /home/user/Documents/src/llvm/llvm-project/lldb/source/Commands/Options.td --write-if-changed -o tools/lldb/source/Commands/CommandOptions.inc -d tools/lldb/source/Commands/CommandOptions.inc.d
/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/lldb-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory
And
/home/user/Documents/src/llvm/llvm.twostage.build/tools/clang/stage2-bins/bin/clang-ast-dump: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory
But this can't be overwritten with -DXYZ_TABLEGEN=<path>
, so now another solution has to be found.
For future reference, I ended up fixing this by building stage1 with
LDFLAGS="-Wl,-rpath=. -Wl,--disable-new-dtags"
which causes all the stage1 binaries to encode the path to each library they link against into the binary itself using RPATH
(as suggested briefly here). The second flag is specifically so that RPATH
is set rather than RUNPATH
, because the former takes precedence over LD_LIBRARY_PATH
. This in turn means you can then run the stage2 build with LD_LIBRARY_PATH
set. Specifically:
LD_LIBRARY_PATH="$stage_2_build/lib:$stage_1_install/lib:$LD_LIBRARY_PATH" make
@jonhoo Let me see if it works in my scenario. Thanks so much!!
Ah, you're using make. I need to find a similar way to do it with Ninja.
Also, to leave searchable breadcrumbs for others who may be running into this issue, it's also possible you run into this issue in the form of an error like:
[ 95%] ASTNodeAPI.json
../../../../bin/clang-ast-dump: relocation error: ../../../../bin/clang-ast-dump: symbol _ZTVN4llvm2cl6parserINSt3__112basic_stringIcNS2_11char_traitsIcEENS2_9allocatorIcEEEEEE, version LLVM_14 not defined in file libLLVM-14.so with link time reference
or this one:
../../../../bin/llvm-tblgen: error while loading shared libraries: libc++.so.1: cannot open shared object file: No such file or directory
There is also some additional discussion in https://discourse.llvm.org/t/bootstrap-build-with-llvm-runtimes-clang-shared-libc-build-failure-stage2-bins-due-to-libc-so-not-found/60141
Ah, you're using make. I need to find a similar way to do it with Ninja.
I think the exact same thing should work for ninja, just s/make/ninja/
in that command.
Thanks. I wonder if I can convert it into a cmake-based solution.
Thanks. With some caveats I was able to use LDFLAGS/LD_LIBRARY_PATH approach, although it required fixing #54955. I'll post the workup once I'm done with #54955
# Stage 1
cmake3 --build "$BUILD_DIR" --target clang-bootstrap-deps
# Stage 2 Prep
HOST_TARGET=$("$BUILD_DIR"/bin/llvm-config --host-target)
STAGE1_LIB="$BUILD_DIR/lib"
STAGE2_LIB="$BUILD_DIR/tools/clang/stage2-bins/lib"
#Stage 2
LDFLAGS="-L$STAGE2_LIB/$HOST_TARGET -L$STAGE2_LIB -L$STAGE1_LIB/$HOST_TARGET -L$STAGE1_LIB" \
LD_LIBRARY_PATH="$STAGE2_LIB/$HOST_TARGET:$STAGE2_LIB:$STAGE1_LIB/$HOST_TARGET:$STAGE1_LIB" \
cmake3 --build "$BUILD_DIR" --target stage2
?
?
??
?
??
???!!
Any updates on this? Facing the similar problem - were able to bypass it by incorporating your instruction @arcivanov (big thanks for that!). Yet, an ability to use built-in bootstrap and not divide the build into 2 stages manually w/ workarounds looks more appealing.
I'm using the workaround in https://github.com/llvm/llvm-project/issues/53561#issuecomment-1133752590
can confirm that there still is no proper 3-stage bootstrap of a self hosting llvm and that the -Wl,-rpath=. -Wl,--disable-new-dtags
on stage1 and then the LD_PRELOAD
of stage1 libs on stage2 method is necessary.
I have a custom two-stage build.
Stage 1 build LLVM 13.0.1 toolchain via GCC on Fedora 35 x86. Stage 2 build LLVM 13.0.1 toolchain using Stage 1 relying completely on LLVM (libc++, compiler-rt, libunwind everywhere).
Stage 1 builds and functions fine. Stage 2 fails to build while invoking any target with
llvm-tblgen
in its path, because it's built beforelibc++
it requires. Ifllvm-tblgen
is specified to be remote via "-DLLVM_TABLEGEN" then the identical thing happens toclang-tblgen
.IMPORTANT: The CMAKE_INSTALL_PREFIX of Stage 1 is on PATH when building Stage 2 but is in a user home directory
Stage 1 CMAKE cache:
Stage 2 CMAKE cache:
Stage 1 and Stage 2 are built virtually identically. Stage 2 is built as follows, where
LLVM_DISTRO_CONF
is the CMAKE cache file above:The build fails with the above
llvm-tblgen
orclang-tblgen
error.Example: