Closed Anefu closed 4 weeks ago
What version of Abseil are you using? This looks like you might have an older one than 25.x requires
https://github.com/protocolbuffers/protobuf/issues/12292#issuecomment-1921936350 It helped me solve the same problem
What version of Abseil are you using? This looks like you might have an older one than 25.x requires
Abseil version: 20240116.1
#12292 (comment) It helped me solve the same problem
Thanks @wizetLee but this still doesn't help.
How are you including protobuf in cmake? I believe the problem back then was how find_package was being used. i.e. this:
find_package(protobuf REQUIRED)
does not work because it uses the deprecated FindProtobuf file we don't own. Instead, you need
find_package(protobuf REQUIRED CONFIG)
Here's a portion of my CMakeList.txt:
option(protobuf_MODULE_COMPATIBLE TRUE)
find_package(Protobuf CONFIG REQUIRED)
find_program(_PROTOBUF_PROTOC protoc)
find_package(Boost 1.83 COMPONENTS system thread REQUIRED)
find_package(absl CONFIG REQUIRED)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20 -fsanitize=address")
get_filename_component(api_proto "websockets/api.proto" ABSOLUTE)
get_filename_component(api_proto_path "${api_proto}" PATH)
add_custom_command(
OUTPUT "${api_proto_srcs}" "${api_proto_hdrs}"
COMMAND ${_PROTOBUF_PROTOC}
ARGS
--cpp_out "${PROJECT_BINARY_DIR}"
-I "${api_proto_path}"
"${api_proto}"
DEPENDS "${api_proto}")
add_library(proto
${api_proto_srcs}
${api_proto_hdrs}
)
target_link_libraries(proto
PUBLIC protobuf::libprotobuf
)
How are you including protobuf in cmake? I believe the problem back then was how find_package was being used. i.e. this:
find_package(protobuf REQUIRED)
does not work because it uses the deprecated FindProtobuf file we don't own. Instead, you need
find_package(protobuf REQUIRED CONFIG)
Thanks, I think it would be useful to make this more prominent in the docs and also make clear that the variables defined in https://cmake.org/cmake/help/latest/module/FindProtobuf.html are not available. On that note, what are the canonical Protobuf CMake variables we can actually use? We are still struggling with this.
Overall, this is causing significant confusion and breakage with many open-source repositories trying to use Protobuf.
Not sure if this is the right place to ask, but I am having exactly the same issue while building a program against Tensorflow, which uses Protobuf.
cd /Users/chazeon/Documents/Projects/deepmd-kit/source/op && /Users/chazeon/miniforge3/envs/deepmd-dev/bin/cmake -E cmake_link_script CMakeFiles/deepmd_op.dir/link.txt --verbose=1
/Users/chazeon/miniforge3/envs/deepmd-dev/bin/arm64-apple-darwin20.0.0-clang++ -ftree-vectorize -fPIC -fstack-protector-strong -O2 -pipe -stdlib=libc++ -fvisibility-inlines-hidden -fmessage-length=0 -isystem /Users/chazeon/miniforge3/envs/deepmd-dev/include -fopenmp=libomp -I -O3 -DNDEBUG -arch arm64 -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk -bundle -Wl,-headerpad_max_install_names -Wl,-headerpad_max_install_names -Wl,-dead_strip_dylibs -Wl,-rpath,/Users/chazeon/miniforge3/envs/deepmd-dev/lib -L/Users/chazeon/miniforge3/envs/deepmd-dev/lib -o libdeepmd_op.so CMakeFiles/deepmd_op.dir/add_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/copy_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/custom_op.cc.o CMakeFiles/deepmd_op.dir/descrpt.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_ef.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_ef_para.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_ef_vert.cc.o CMakeFiles/deepmd_op.dir/descrpt_se_a_mask.cc.o CMakeFiles/deepmd_op.dir/dotmul_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/ewald_recp.cc.o CMakeFiles/deepmd_op.dir/flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/gelu_multi_device.cc.o CMakeFiles/deepmd_op.dir/map_aparam.cc.o CMakeFiles/deepmd_op.dir/map_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/matmul_fitnet_nvnmd.cc.o CMakeFiles/deepmd_op.dir/matmul_flt2fix_nvnmd.cc.o CMakeFiles/deepmd_op.dir/matmul_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/mul_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/neighbor_stat.cc.o CMakeFiles/deepmd_op.dir/pair_tab.cc.o CMakeFiles/deepmd_op.dir/pairwise.cc.o CMakeFiles/deepmd_op.dir/prod_env_mat_multi_device.cc.o CMakeFiles/deepmd_op.dir/prod_env_mat_multi_device_nvnmd.cc.o CMakeFiles/deepmd_op.dir/prod_force.cc.o CMakeFiles/deepmd_op.dir/prod_force_multi_device.cc.o CMakeFiles/deepmd_op.dir/prod_force_se_a_mask.cc.o CMakeFiles/deepmd_op.dir/prod_virial.cc.o CMakeFiles/deepmd_op.dir/prod_virial_multi_device.cc.o CMakeFiles/deepmd_op.dir/quantize_nvnmd.cc.o CMakeFiles/deepmd_op.dir/soft_min.cc.o CMakeFiles/deepmd_op.dir/soft_min_force.cc.o CMakeFiles/deepmd_op.dir/soft_min_virial.cc.o CMakeFiles/deepmd_op.dir/tabulate_multi_device.cc.o CMakeFiles/deepmd_op.dir/tanh4_flt_nvnmd.cc.o CMakeFiles/deepmd_op.dir/unaggregated_grad.cc.o CMakeFiles/deepmd_op.dir/optimizer/parallel.cc.o -Wl,-rpath,/Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow -Wl,-rpath,/Users/chazeon/Documents/Projects/deepmd-kit/source/lib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/libomp.dylib -lProtobuf /Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow/libtensorflow_framework.2.dylib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow/libtensorflow_cc.2.dylib ../lib/libdeepmd.dylib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/python3.11/site-packages/tensorflow/../../..//libprotobuf.24.4.0.dylib /Users/chazeon/miniforge3/envs/deepmd-dev/lib/libomp.dylib
Undefined symbols for architecture arm64:
"void absl::lts_20230802::log_internal::LogMessage::CopyToEncodedBuffer<(absl::lts_20230802::log_internal::LogMessage::StringType)0>(std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
"absl::lts_20230802::log_internal::LogMessage::operator<<(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&)", referenced from:
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
"absl::lts_20230802::log_internal::LogMessageFatal::LogMessageFatal(char const*, int, std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
"absl::lts_20230802::log_internal::LogMessageFatal::~LogMessageFatal()", referenced from:
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [2]>(char const (&) [2]) const in parallel.cc.o
tensorflow::AttrValue const& google::protobuf::Map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, tensorflow::AttrValue>::at<char [8]>(char const (&) [8]) const in parallel.cc.o
"absl::lts_20230802::hash_internal::MixingHashState::LowLevelHashImpl(unsigned char const*, unsigned long)", referenced from:
absl::lts_20230802::hash_internal::MixingHashState::combine_contiguous(absl::lts_20230802::hash_internal::MixingHashState, unsigned char const*, unsigned long) in parallel.cc.o
"absl::lts_20230802::hash_internal::MixingHashState::CombineLargeContiguousImpl64(unsigned long long, unsigned char const*, unsigned long)", referenced from:
absl::lts_20230802::hash_internal::MixingHashState::combine_contiguous(absl::lts_20230802::hash_internal::MixingHashState, unsigned char const*, unsigned long) in parallel.cc.o
"absl::lts_20230802::hash_internal::MixingHashState::kSeed", referenced from:
google::protobuf::internal::KeyMapBase<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>::FindHelper(std::__1::basic_string_view<char, std::__1::char_traits<char>>, absl::lts_20230802::container_internal::btree_iterator<absl::lts_20230802::container_internal::btree_node<absl::lts_20230802::container_internal::map_params<google::protobuf::internal::VariantKey, google::protobuf::internal::NodeBase*, std::__1::less<google::protobuf::internal::VariantKey>, google::protobuf::internal::MapAllocator<std::__1::pair<google::protobuf::internal::VariantKey const, google::protobuf::internal::NodeBase*>>, 256, false>>, std::__1::pair<google::protobuf::internal::VariantKey const, google::protobuf::internal::NodeBase*>&, std::__1::pair<google::protobuf::internal::VariantKey const, google::protobuf::internal::NodeBase*>*>*) const in parallel.cc.o
google::protobuf::internal::KeyMapBase<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>::TransferList(google::protobuf::internal::KeyNode<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>>*) in parallel.cc.o
"absl::lts_20230802::Status::UnrefNonInlined(unsigned long)", referenced from:
absl::lts_20230802::Status::~Status() in add_flt_nvnmd.cc.o
absl::lts_20230802::Status::~Status() in copy_flt_nvnmd.cc.o
absl::lts_20230802::Status::~Status() in custom_op.cc.o
DescrptOp<Eigen::ThreadPoolDevice, float>::DescrptOp(tensorflow::OpKernelConstruction*) in descrpt.cc.o
absl::lts_20230802::Status::~Status() in descrpt.cc.o
DescrptOp<Eigen::ThreadPoolDevice, double>::DescrptOp(tensorflow::OpKernelConstruction*) in descrpt.cc.o
DescrptSeAEfOp<Eigen::ThreadPoolDevice, float>::DescrptSeAEfOp(tensorflow::OpKernelConstruction*) in descrpt_se_a_ef.cc.o
...
"absl::lts_20230802::Status::Status(absl::lts_20230802::StatusCode, std::__1::basic_string_view<char, std::__1::char_traits<char>>)", referenced from:
absl::lts_20230802::Status tsl::errors::Internal<char const*, char const*, char const*, char const*, char const*, int>(char const*, char const*, char const*, char const*, char const*, int) in custom_op.cc.o
absl::lts_20230802::Status tsl::errors::ResourceExhausted<char const*, char const*, char const*, char const*, char const*, int>(char const*, char const*, char const*, char const*, char const*, int) in custom_op.cc.o
absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt.cc.o
absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_ef.cc.o
absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_ef_para.cc.o
absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_ef_vert.cc.o
absl::lts_20230802::Status tsl::errors::InvalidArgument<char const*>(char const*) in descrpt_se_a_mask.cc.o
...
ld: symbol(s) not found for architecture arm64
clang-16: error: linker command failed with exit code 1 (use -v to see invocation)
make[2]: *** [op/CMakeFiles/deepmd_op.dir/build.make:751: op/libdeepmd_op.so] Error 1
make[2]: Leaving directory '/Users/chazeon/Documents/Projects/deepmd-kit/source'
make[1]: *** [CMakeFiles/Makefile2:277: op/CMakeFiles/deepmd_op.dir/all] Error 2
make[1]: Leaving directory '/Users/chazeon/Documents/Projects/deepmd-kit/source'
make: *** [Makefile:136: all] Error 2
my Tensorflow and Protobuf is installed via mamba
(deepmd-dev) (.env) chazeon@Chenxings-MBP [23:06:17] [~/Documents/Projects/deepmd-kit/source] [devel *]
-> % mamba list | grep tensorflow
libtensorflow 2.15.0 cpu_h9d8c315_2 conda-forge
libtensorflow_cc 2.15.0 cpu_h1998a8a_2 conda-forge
tensorflow 2.15.0 cpu_py311h9b82b1a_2 conda-forge
tensorflow-base 2.15.0 cpu_py311he034567_2 conda-forge
tensorflow-estimator 2.15.0 cpu_py311hcceb7c1_2 conda-forge
(deepmd-dev) (.env) chazeon@Chenxings-MBP [23:06:46] [~/Documents/Projects/deepmd-kit/source] [devel *]
-> % mamba list | grep protobuf
libprotobuf 4.24.4 h810fc01_0 conda-forge
protobuf 4.24.4 py311h4d1eceb_0 conda-forge
I have to link the following libraries explicitly to make the build work:
find_package(absl REQUIRED)
target_link_libraries(
${libname} PRIVATE
absl::log_internal_message
absl::log_internal_check_op
absl::status
)
I notice that the symbol
void absl::lts_20240116::log_internal::LogMessage::CopyToEncodedBuffer<(absl::lts_20240116::log_internal::LogMessage::StringType)0>(std::__1::basic_string_view<char, std::__1::char_traits<char>>)
is not publicly exposed from libprotobuf.dylib
.
We triage inactive PRs and issues in order to make it easier to find active work. If this issue should remain active or becomes active again, please add a comment.
This issue is labeled inactive
because the last activity was over 90 days ago.
We triage inactive PRs and issues in order to make it easier to find active work. If this issue should remain active or becomes active again, please reopen it.
This issue was closed and archived because there has been no new activity in the 14 days since the inactive
label was added.
What version of protobuf and what language are you using? Version: 25.3_1 Language: C++
What operating system (Linux, Windows, ...) and version? Mac, running MacOS 14.4.1 Sonoma What runtime / compiler are you using (e.g., python version or gcc version) Apple clang version 15.0.0 (clang-1500.3.9.4) Target: arm64-apple-darwin23.4.0 Thread model: posix InstalledDir: /Library/Developer/CommandLineTools/usr/bin What did you do? Steps to reproduce the behavior:
What did you expect to see Successful compilation What did you see instead?
Make sure you include information that can help us debug (full error message, exception listing, stack trace, logs).
Anything else we should know about your project / environment