search

xmake-io / xmake-repo

📦 An official xmake package repository
https://xrepo.xmake.io
Apache License 2.0
684 stars 398 forks source link

照着原来的arrow7.0.0写了个能在win下通过的，不过没法添加lz4之类的 #4033

Open Altina-oz opened 5 months ago

Altina-oz commented 5 months ago

你在什么场景下需要该功能？

库里原先的apache-arrow是7.0.0 linux版本的，仿照改写了个能在win下16.1.0版本的，不过貌似lz4之类的会挂（读写不压缩的parquet，feather，csv，compute什么的能过），其他的三方库我不确定至少lz4链接会链接不到（static, win11, msvc2022 c++20 release, xmake 2,9.1 dev, 报错见下方）

描述可能的解决方案

package("apache-arrow")
    set_description("Apache arrow")

    add_urls("https://github.com/apache/arrow/archive/refs/tags/apache-arrow-$(version).tar.gz")
    add_versions('16.1.0', '9762d9ecc13d09de2a03f9c625a74db0d645cb012de1e9a10dfed0b4ddc09524')

    add_configs("csv",      {description = "CSV reader module", default = true, type = "boolean"})
    add_configs("json",     {description = "JSON reader module", default = true, type = "boolean"})
    add_configs("compute",   {description = "Build all computational kernel functions", default = true, type = "boolean"})
    add_configs("dataset",  {description = "Dataset API, implies the Filesystem API", default = true, type = "boolean"})
    --add_configs("filesystem",      {description = "Filesystem API for accessing local and remote filesystems", default = true, type = "boolean"})
    --add_configs("mimalloc", {description = "Build the Arrow mimalloc-based allocator", default = true, type = "boolean"})
    --add_configs("jemalloc", {description = "Build the Arrow jemalloc-based allocator", default = false, type = "boolean"})
    add_configs("parquet",  {description = "Apache Parquet libraries and Arrow integration", default = true, type = "boolean"})
    -- If true, arrow will look for shared libraries for third party dependency.
    -- The pyarrow python package creates shared library that links in all necessary thirdparty static libraries.
    add_configs("shared_dep", {description = "Use shared library for dependency", default = false, type = "boolean"})

    -- Some libraries are required for build with our default config settings.
    local configdeps = {
        re2 = "re2", utf8proc = "utf8proc",
        -- compression libraries
        brotli = "brotli", bz2 = "bzip2", snappy = "snappy", lz4 = "lz4", zlib = "zlib", zstd = "zstd",
    }
    for config, dep in pairs(configdeps) do

        add_configs(config, {description = "Enable " .. dep .. " support.", default = false, type = "boolean"})
    end

    add_deps("cmake", "boost")

    if is_plat("bsd") then
        add_syslinks("pthread", "execinfo")
    elseif is_plat("linux") then
        add_syslinks("pthread")
    elseif is_plat("windows") then 
        add_syslinks("Ole32", "shell32","kernel32","Ws2_32")
    end
    on_load(function (package)

        if package:config("shared") or not is_plat("windows") then
            package:add("links", "arrow", "arrow_bundled_dependencies")
        else 
            package:add("links","arrow_static","arrow_acero_static","arrow_bundled_dependencies")

        end

        if package:config("dataset") then
            if package:config("shared") or not is_plat("windows") then
                package:add("links", "arrow_dataset")
            else 
                package:add("links","arrow_dataset_static")
            end
        end

        for name, dep in pairs(configdeps) do
            if package:config(name) then
                package:add("deps", dep)
            end
        end

        if package:config("parquet")  then
            if package:config("shared") or not is_plat("windows") then
                package:add("links", "parquet")
            else 
                package:add("links","parquet_static")
            end

            package:add("deps", "thrift")
        end
        if package:config("json") then
            package:add("deps", "rapidjson")
        end

    end)

    on_install(function (package)
        local configs = {
            "-DARROW_BUILD_TESTS=OFF",
            "-DARROW_DEPENDENCY_SOURCE=BUNDLED",
            "-DARROW_SIMD_LEVEL=AVX2"
        }
        package:addenv("ARROW_THRIFT_BUILD_VERSION", "0.20.0")
        table.insert(configs, "-DCMAKE_BUILD_TYPE=" .. (package:debug() and "Debug" or "Release"))

        local shared = package:config("shared")

        if is_plat("windows") then 
            table.insert(configs, "-DARROW_STATIC -DARROW_FLIGHT_STATIC -DARROW_FLIGHT_SQL_STATIC -DARROW_BUILD_STATIC=" .. (shared and "OFF" or "ON"))
        else
            table.insert(configs, "-DARROW_BUILD_STATIC=" .. (shared and "OFF" or "ON"))
        end
        table.insert(configs, "-DARROW_BUILD_SHARED=" .. (shared and "ON" or "OFF"))
        table.insert(configs, "-DARROW_DEPENDENCY_USE_SHARED=" .. (package:config("shared_dep") and "ON" or "OFF"))

        for config, enabled in pairs(package:configs()) do
            if not package:extraconf("configs", config, "builtin") and configdeps[config] == nil then
                table.insert(configs, "-DARROW_" .. string.upper(config)  .. "=" .. (enabled and "ON" or "OFF"))
            end
        end

        for config, dep in pairs(configdeps) do
            table.insert(configs, "-DARROW_WITH_" .. string.upper(config)  .. "=" .. (package:config(config) and "ON" or "OFF"))
        end

        -- To fix arrow src/arrow/CMakeLists.txt:538, when CMAKE_SYSTEM_NAME set but CMAKE_SYSTEM_PROCESSOR is not causing error.
        table.insert(configs, "-DCMAKE_SYSTEM_PROCESSOR=" .. (package:is_arch("x86_64") and "x86_64" or "x86"))

        os.cd("cpp")

        import("package.tools.cmake").install(package, configs)
    end)

    on_test(function (package)

    end)

以上可以不压缩地读写和运行（至少不依赖三方库的部分）计算模块等

描述你认为的候选方案

arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_compress_default，函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_decompress_safe，函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::Decompress(__int64,unsigned char const *,__int64,unsigned char *)" (?Decompress@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_compressBound，函数 "public: virtual __int64 __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::MaxCompressedLen(__int64,unsigned char const *)" (?MaxCompressedLen@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA_J_JPEBE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_isError，函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_getErrorName，函数 "class arrow::Status __cdecl arrow::util::internal::`anonymous namespace'::LZ4Error(unsigned __int64,char const *)" (?LZ4Error@?A0xa3932215@internal@util@arrow@@YA?AVStatus@4@_KPEBD@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressionLevel_max，函数 "public: virtual int __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::maximum_compression_level(void)const " (?maximum_compression_level@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEBAHXZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressFrameBound，函数 "public: virtual __int64 __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::MaxCompressedLen(__int64,unsigned char const *)" (?MaxCompressedLen@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA_J_JPEBE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressFrame，函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_createCompressionContext，函数 "public: virtual class arrow::Result<class std::shared_ptr<class arrow::util::Compressor> > __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::MakeCompressor(void)" (?MakeCompressor@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@V?$shared_ptr@VCompressor@util@arrow@@@std@@@5@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_freeCompressionContext，函数 "public: virtual __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::~LZ4Compressor(void)" (??1LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressBegin，函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressBound，函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressUpdate，函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::CompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UCompressResult@Compressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_flush，函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::FlushResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::Flush(__int64,unsigned char *)" (?Flush@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UFlushResult@Compressor@util@arrow@@@5@_JPEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_compressEnd，函数 "public: virtual class arrow::Result<struct arrow::util::Compressor::EndResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Compressor::End(__int64,unsigned char *)" (?End@LZ4Compressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UEndResult@Compressor@util@arrow@@@5@_JPEAE@Z) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_createDecompressionContext，函数 "public: virtual class arrow::Result<class std::shared_ptr<class arrow::util::Decompressor> > __cdecl arrow::util::internal::`anonymous namespace'::Lz4FrameCodec::MakeDecompressor(void)" (?MakeDecompressor@Lz4FrameCodec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@V?$shared_ptr@VDecompressor@util@arrow@@@std@@@5@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_freeDecompressionContext，函数 "public: virtual __cdecl arrow::util::internal::`anonymous namespace'::LZ4Decompressor::~LZ4Decompressor(void)" (??1LZ4Decompressor@?A0xa3932215@internal@util@arrow@@UEAA@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_decompress，函数 "public: virtual class arrow::Result<struct arrow::util::Decompressor::DecompressResult> __cdecl arrow::util::internal::`anonymous namespace'::LZ4Decompressor::Decompress(__int64,unsigned char const *,__int64,unsigned char *)" (?Decompress@LZ4Decompressor@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@UDecompressResult@Decompressor@util@arrow@@@5@_JPEBE0PEAE@Z) 中引用了该 符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4F_resetDecompressionContext，函数 "public: virtual class arrow::Status __cdecl arrow::util::internal::`anonymous namespace'::LZ4Decompressor::Reset(void)" (?Reset@LZ4Decompressor@?A0xa3932215@internal@util@arrow@@UEAA?AVStatus@5@XZ) 中引用了该符号
arrow_static.lib(compression_lz4.obj) : error LNK2019: 无法解析的外部符号 __imp_LZ4_compress_HC，函数 "public: virtual class arrow::Result<__int64> __cdecl arrow::util::internal::`anonymous namespace'::Lz4Codec::Compress(__int64,unsigned char const *,__int64,unsigned char *)" (?Compress@Lz4Codec@?A0xa3932215@internal@util@arrow@@UEAA?AV?$Result@_J@5@_JPEBE0PEAE@Z) 中引用了该符号

其他信息

No response

Issues-translate-bot commented 5 months ago

Bot detected the issue body's language is not English, translate it automatically.

Title: Based on the original arrow7.0.0, I wrote one that can pass under win, but I can't add lz4 or the like.

SnowinterCat commented 2 months ago

__imp是Windows导入动态库符号的前缀，相当于你给了一个lz4的静态库进去，但是你的arrow需要一个lz4的动态库，所以找不到符号。你看看cmake或者arrow里面控制lz4的导入符号的部分，看一下控制宏是什么。再用类似以下的方式，把宏追加进cmake里面，来支持lz4的静态库：

if not package:dep("lz4"):config("shared") then
  io.replace("CMakeLists.txt", "target_compile_definitions(${arrow某个模块} PUBLIC ${模块中的其他宏})", "target_compile_definitions(${arrow某个模块} PUBLIC ${模块中的其他宏} ${追加的宏})")
end