search

hyunsik / hdfs-rs

libhdfs binding and wrapper APIs for Rust
Apache License 2.0
33 stars 10 forks source link

`exec.sh cargo test` failed #4

Open sunchao opened 6 years ago

sunchao commented 6 years ago

I'm trying to use this Hadoop 2.7.2. Compiled the native lib on Mac. cargo build works fine, but when exec.sh cargo test I got this:

➜  hdfs-rs (master) ✗ ./exec.sh cargo test
   Compiling hdfs v0.0.4 (file:///Users/chao/git/hdfs-rs)
warning: lint raw_pointer_derive has been removed: using derive with raw pointers is ok
   --> src/dfs.rs:273:9
    |
273 | #[allow(raw_pointer_derive)]
    |         ^^^^^^^^^^^^^^^^^^
    |
    = note: #[warn(renamed_and_removed_lints)] on by default

warning: lint raw_pointer_derive has been removed: using derive with raw pointers is ok
   --> src/dfs.rs:273:9
    |
273 | #[allow(raw_pointer_derive)]
    |         ^^^^^^^^^^^^^^^^^^

warning: unused import: `itertools::Itertools`
   --> src/dfs.rs:787:7
    |
787 |   use itertools::Itertools;
    |       ^^^^^^^^^^^^^^^^^^^^
    |
    = note: #[warn(unused_imports)] on by default

error: linking with `cc` failed: exit code: 1
  |
  = note: "cc" "-m64" "-L" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.158wz1xuz2s5vbhj.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.15ocno9e0gz60klj.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.16u6js6g0l3k1ic6.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.17erb31xn70viq4b.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.181cuta0v63atwcm.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1e698527iprj2v8f.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1ha8q1iwal888w5t.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1im38lueib99jsk0.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1jnlhyxj59jycbjv.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1qko0s7o0yku8wi7.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1y16o1qfye96o7m0.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.1zwd8n7bcl3vhvvh.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.22hy4jf5cexj8ofe.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.23ce453q2vda0cq6.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.23tqyymcb18u96mb.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2bmdyctmqwyrczo2.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2cf3hj3i603y6nxm.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2jqywn86b2gsqohu.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2lyh15q6cjwzy18c.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2oiwwqohid6mabhc.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2w6xsa2hbi53xhak.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2xnvmuhjbhd7vxcm.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2xs38k3nu4oe9uo6.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.2xwdgiv55wx7wpkg.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3ayaeypdcro9d6yk.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3cutt3lg8vmui89m.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3g94tobrpgum5dk6.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3ik0x0hz6l66cx38.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3nb415nst569tn21.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3oqznv5oym7l3m4e.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3rngp6bm2u2q5z0y.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3thsdxsfmyg8gwej.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3vck0z67htzlxjio.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.3wta9ctgdrpkmlpr.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.41fhlsd7519xuco5.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.436dotimmrgzkwfa.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.43v6g0y2xsxoggnt.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.44bsbddupzfao2om.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.45pc7c65foh9i35f.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.48721dc4k5qxei0u.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.49a7n47po4ttqjl7.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.49lx1q7cxvpykyv0.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.49qmpuk8w1opmi5q.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4brwxiorhmgsb6oc.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4ezmh1vbs95c5ack.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4g47d2sx6m33abop.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4oc10dk278mpk1vy.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4rzh6npf1tuwcxx9.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4sw9hz7i7fiueihz.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4v3lwapkm7lhovox.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4xq48u46a1pwiqn7.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4xuhwz5yy6q4gvgg.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4yh8x2b62dcih00t.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4ypvbwho0bu5tnww.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.4z8gk4dt64uq1bs.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.51s1w397y42gpez1.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.56dly8q07ws8ucdq.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.572bgozz9kcwe4zq.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.57k06xfugllsc526.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.5beqqbexwb143v6a.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.5by9xggibmun5kn9.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.5gf6du7k58s78kob.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.63mmpc50mjcb0lb.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.8xzrsc1ux72v29j.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.98g0d9x8aw3akpe.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.9elsx31vb4it187.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.c6lbtaiefvx3wya.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.d6p2df93qm1h3gm.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.e19kx9t7xhzajfl.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.mb501chi8fnqgmm.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.oa3rad818d8sgn4.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.y08g5q2x813c4wx.rcgu.o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.z9ox7biyn1otfln.rcgu.o" "-o" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55" "/Users/chao/git/hdfs-rs/target/debug/deps/hdfs-2b714980d01b0f55.crate.allocator.rcgu.o" "-Wl,-dead_strip" "-nodefaultlibs" "-L" "/Users/chao/git/hdfs-rs/target/debug/deps" "-L" "/opt/hadoop/lib/native" "-L" "/Users/chao/git/hdfs-rs/target/debug/build/hdfs-6b27ce521f6b3b8b/out" "-L" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib" "-l" "hdfs" "-l" "hdfs" "-Wl,-force_load,/Users/chao/git/hdfs-rs/target/debug/build/hdfs-6b27ce521f6b3b8b/out/libminidfs.a" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libtest-0d97595764cf9b64.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libterm-0f60ec9e394e0248.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libgetopts-6f2ac69fc328f830.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/liburl-1012528998a3d996.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/libmatches-d634ed9077d128bf.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/libuuid-2cdedf5331d88c7d.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/librand-67af2eb6503806d0.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/librand-c95782c5d16a1bc5.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/liblibc-0819fa82d85a83bd.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/librustc_serialize-a4d600dec7edbf71.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/liblog-5e9334cd63f74e97.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/liblog-41e198eccb985521.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/libcfg_if-2bd44b22855dd866.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/liblibc-017538ef75185d0f.rlib" "/Users/chao/git/hdfs-rs/target/debug/deps/libitertools-a36391e000121329.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libstd-81ad5fb7ec87f0fb.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libpanic_unwind-2e56e130d8e4c4c5.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/liballoc_jemalloc-65f3bd01304e8cb7.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libunwind-63ece1d61d0caa09.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/liballoc_system-1f539c0f734c73b4.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/liblibc-72ea87627c504b3f.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/liballoc-f664b1dcf152920f.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libcore-39fcd66b77381104.rlib" "/Users/chao/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/x86_64-apple-darwin/lib/libcompiler_builtins-c6bbc2262c448bc9.rlib" "-l" "c" "-l" "m" "-l" "System" "-l" "resolv" "-l" "pthread" "-l" "c" "-l" "m"
  = note: Undefined symbols for architecture x86_64:
            "_hadoopConfSetStr", referenced from:
                _nmdConfigureShortCircuit in libminidfs.a(native_mini_dfs.o)
            "_hdfsDisableDomainSocketSecurity", referenced from:
                _nmdConfigureShortCircuit in libminidfs.a(native_mini_dfs.o)
            "_newRuntimeError", referenced from:
                _nmdConfigureShortCircuit in libminidfs.a(native_mini_dfs.o)
            "_getJNIEnv", referenced from:
                _nmdCreate in libminidfs.a(native_mini_dfs.o)
                _nmdFree in libminidfs.a(native_mini_dfs.o)
                _nmdShutdown in libminidfs.a(native_mini_dfs.o)
                _nmdGetNameNodePort in libminidfs.a(native_mini_dfs.o)
            "_newJavaStr", referenced from:
                _nmdCreate in libminidfs.a(native_mini_dfs.o)
            "_constructNewObjectOfClass", referenced from:
                _nmdCreate in libminidfs.a(native_mini_dfs.o)
            "_invokeMethod", referenced from:
                _nmdCreate in libminidfs.a(native_mini_dfs.o)
                _nmdShutdown in libminidfs.a(native_mini_dfs.o)
                _nmdGetNameNodePort in libminidfs.a(native_mini_dfs.o)
            "_printExceptionAndFree", referenced from:
                _nmdCreate in libminidfs.a(native_mini_dfs.o)
                _nmdShutdown in libminidfs.a(native_mini_dfs.o)
                _nmdGetNameNodePort in libminidfs.a(native_mini_dfs.o)
            "_printPendingExceptionAndFree", referenced from:
                _nmdCreate in libminidfs.a(native_mini_dfs.o)
          ld: symbol(s) not found for architecture x86_64
          clang: error: linker command failed with exit code 1 (use -v to see invocation)

error: aborting due to previous error

error: Could not compile `hdfs`.

To learn more, run the command again with --verbose.

is version 2.7.2 supported by this library? do I need to have a running Hadoop environment on my Mac in order to run the tests? Thanks.

yahoNanJing commented 3 years ago

Any update for this issue? I also meet the same issue.

yahoNanJing commented 3 years ago

It's because the related functions are not exported in the hdfs library. It can be checked by nm libhdfs.dylib | grep " t " Here, t means local function which is not exported

yahoNanJing commented 3 years ago

Based on hdfs-rs, I created another crate fs-hdfs