Building without tools, and tests

[baryluk]

I am trying to convert from cmake base build system to bazel.

I was building previously onload as a root and installing it system wide, either on a host or in a docker.

As part of bazel build, I wish to make the build hermetic and isolated. (not use libraries and toolchain from the host).

Already converted all dependencies, with onload / ef_vi being the last one to tackle.

First I added libcap2 to bazel, as it is a dependency of cplane (unless compiled with NO_CAPS define).

Still, I have issue with that.

Here is my setup

WORKSPACE:

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
    name = "rules_foreign_cc",
    # url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
    # sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
    # strip_prefix = "rules_foreign_cc-0.9.0",
    strip_prefix = "rules_foreign_cc-main",
    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/main.zip",
)

load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
rules_foreign_cc_dependencies()

all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""

# libcap is a dependency of onload
http_archive(
    name = "mylibcap",
    urls = ["https://mirrors.edge.kernel.org/pub/linux/libs/security/linux-privs/libcap2/libcap-2.67.tar.xz"],
    sha256 = "ce9b22fdc271beb6dae7543da5f74cf24cb82e6848cfd088a5a069dec5ea5198",
    strip_prefix = "libcap-2.67",
    build_file_content = all_content,
)

http_archive(
    name = "myonload",
    # master with 8.1 branch, and some glibc fixes (i.e. __fstatx issues)
    urls = ["https://github.com/Xilinx-CNS/onload/archive/1c45de39d43c682ee85c489d2ed624fb17fa653b.zip"],
    sha256 = "964097dba011791bb1821197babd6772b03e49c6b4d21911e94ea5b1b5015e58",
    strip_prefix = "onload-1c45de39d43c682ee85c489d2ed624fb17fa653b",
    build_file_content = all_content,
)

extra/libcap/BUILD:

package(default_visibility = ["//:__subpackages__"])

load("@rules_foreign_cc//foreign_cc:defs.bzl", "make")

make(
    name = "libcap_make",
    lib_name = "libcap",
    lib_source = "@mylibcap//:all",
    out_static_libs = [
        "libcap.a",
    ],
    out_lib_dir = "lib64",
    out_shared_libs = [
        "libcap.so.2.67",
    ],
    # out_include_dir = "libcap/include",
    out_include_dir = "usr/include",
    visibility = ["//visibility:public"],
    args = ["-C", "libcap", "FAKEROOT=$$INSTALLDIR$$"],
)

cc_library(
   name = "libcap",
   # copts = ["-Iexternal/libcap/libcap/include"],
   deps = ["libcap_make"],
)

extra/onload/BUILD:

package(default_visibility = ["//:__subpackages__"])

genrule(
    name = "onload_genrule",
    srcs = [
      "@myonload//:all",
      "//extra/libcap:libcap",
    ],
    outs = [
      "onload/install/lib/asd.a",  # TODO
      "onload/install/lib/asd.so",
      "onload/install/include/asd/asd.h",
    ],
    # --no-debug-info
    cmd = "cd external/myonload/ && mkdir install && ./scripts/onload_build --user && ./scripts/onload_install --userfiles --noldconfig --includedir=`pwd`/install/include --lib64dir=`pwd`/install/lib --bindir=`pwd`/install/bin --sbindir=`pwd`/install/sbin",

  # --sbindir=*) sbindir=${1#--sbindir=};;
  # --usrsbindir=*) usrsbindir=${1#--usrsbindir=};;
  # --bindir=*) bindir=${1#--bindir=};;
  # --libexecdir=*) libexecdir=${1#--libexecdir=};;
  # --sysconfdir=*) sysconfdir=${1#--sysconfdir=};;
  # --includedir=*) includedir=${1#--includedir=};;
  # --usrlocaldir=*) usrlocaldir=${1#--usrlocaldir=};;
  # --usrdir=*) usrdir=${1#--usrdir=};;
  # --lib32dir=*) lib32dir=${1#--lib32dir=};;
  # --lib64dir=*) lib64dir=${1#--lib64dir=};;
)

cc_library(
    name = "onload",
    # copts = = ["-Iexternal/onload/src/include"],
    srcs = [":onload_genrule"],
    visibility = ["//visibility:public"],
)

m/BUILD

cc_library(
  name = "example",
  srcs = [
    "example.cc",
  ],
  deps = [
    "//extra/libcap:libcap",
  ],
)

m/example.cc

#include <sys/capability.h>

Building libcap works:

witek:~/onload-example$ bazel build //extra/libcap:libcap --sandbox_debug --verbose_failures
INFO: Analyzed target //extra/libcap:libcap (0 packages loaded, 0 targets configured).
INFO: Found 1 target...
Target //extra/libcap:libcap up-to-date (nothing to build)
INFO: Elapsed time: 0.170s, Critical Path: 0.00s
INFO: 1 process: 1 internal.
INFO: Build completed successfully, 1 total action
witek@:~/onload-example$ find bazel-out/ | grep -E 'sys/cap|libcap.(a|so)'
bazel-out/k8-fastbuild/bin/extra/libcap/libcap/lib64/libcap.a
bazel-out/k8-fastbuild/bin/extra/libcap/libcap/lib64/libcap.so.2.67
bazel-out/k8-fastbuild/bin/extra/libcap/copy_libcap/libcap/lib64/libcap.a
bazel-out/k8-fastbuild/bin/extra/libcap/copy_libcap/libcap/lib64/libcap.so.2.67
bazel-out/k8-fastbuild/bin/extra/libcap/copy_libcap/libcap/lib64/libcap.so.2
bazel-out/k8-fastbuild/bin/extra/libcap/copy_libcap/libcap/lib64/libcap.so
bazel-out/k8-fastbuild/bin/extra/libcap/copy_libcap/libcap/usr/include/sys/capability.h
bazel-out/k8-fastbuild/bin/_solib_k8/_U_S_Sextra_Slibcap_Clibcap_Umake___Uextra_Slibcap_Slibcap_Slib64/libcap.so.2.67
witek@:~/onload-example$ 

as does binaries using them:

witek@:~/onload-example$  blaze build //m:example --sandbox_debug
INFO: Analyzed target //m:example (1 packages loaded, 3 targets configured).
INFO: Found 1 target...
Target //m:example up-to-date:
  bazel-bin/m/libexample.a
  bazel-bin/m/libexample.so
INFO: Elapsed time: 0.229s, Critical Path: 0.01s
INFO: 1 process: 1 internal.
INFO: Build completed successfully, 1 total action
witek@:~/onload-example$ 

But onload not:

witek@:~/onload-example$ bazel build //extra/onload:onload_genrule --sandbox_debug --verbose_failures

1678884263.252827959: src/main/tools/linux-sandbox-pid1.cc:491: calling fork...
1678884263.253153282: src/main/tools/linux-sandbox-pid1.cc:521: child started with PID 2
+ nice cc -m64 -O2 -g -shared -fPIC -Wl,-soname,libonload_ext.so.1 onload_ext.o -o libonload_ext.so.1.2.0
fatal: not a git repository (or any of the parent directories): .git
fatal: not a git repository (or any of the parent directories): .git
fatal: not a git repository (or any of the parent directories): .git
+ nice cc -m64 -O2 -g -lrt -shared -fPIC -Wl,-soname,libciul.so.1 ci_ul_pt_tx.o ci_ul_pt_rx.o ci_ul_vi_init.o ci_ul_ef10_event.o ci_ul_ef10_vi.o ci_ul_ef100_event.o ci_ul_ef100_vi.o ci_ul_efxdp_vi.o ci_ul_efct_vi.o ci_ul_ef10_evtimer.o ci_ul_logging.o ci_ul_checksum.o ci_ul_open.o ci_ul_event_q.o ci_ul_event_q_put.o ci_ul_pt_endpoint.o ci_ul_filter.o ci_ul_vi_set.o ci_ul_memreg.o ci_ul_pd.o ci_ul_ef_app_cluster.o ci_ul_pio.o ci_ul_vi_layout.o ci_ul_vi_stats.o ci_ul_vi_prime.o ci_ul_capabilities.o ci_ul_smartnic_exts.o ci_ul_ctpio.o -o libciul.so.1.1.1
+ nice cc -m64 -O2 -g -nostartfiles -shared -fPIC ci_tp_unix_startup.o ci_tp_unix_log_fn.o ci_tp_unix_sys.o ci_tp_unix_sockcall_intercept.o ci_tp_unix_onload_ext_intercept.o ci_tp_unix_zc_intercept.o ci_tp_unix_zc_hlrx.o ci_tp_unix_tmpl_intercept.o ci_tp_unix_stackname.o ci_tp_unix_stackopt.o ci_tp_unix_fdtable.o ci_tp_unix_protocol_manager.o ci_tp_unix_closed_fd.o ci_tp_unix_tcp_fd.o ci_tp_unix_udp_fd.o ci_tp_unix_pipe_fd.o ci_tp_unix_nonsock.o ci_tp_unix_epoll_fd.o ci_tp_unix_epoll_fd_b.o ci_tp_unix_netif_init.o ci_tp_unix_exec.o ci_tp_unix_environ.o ci_tp_unix_common_fcntl.o ci_tp_unix_wqlock.o ci_tp_unix_poll_select.o ci_tp_unix_passthrough_fd.o ci_tp_unix_utils.o ci_tp_unix_vfork_intercept.o ./../../../lib/transport/common/libcitpcommon0.a ./../../../lib/transport/ip/libciip0.a ./../../../lib/cplane/libcplane0.a ./../../../lib/citools/libcitools1.a ./../../../lib/ciul/libciul1.a -e onload_version_msg -Wl,--version-script=/home/witek/.cache/bazel/_bazel_witek/52ee13dadb77498b9e4fc952a2e4201b/sandbox/linux-sandbox/226/execroot/__main__/external/myonload/src/lib/transport/unix/exports.map -lm -lpthread -lrt -ldl -lanl -o libcitransport0.so
/home/witek/.cache/bazel/_bazel_witek/52ee13dadb77498b9e4fc952a2e4201b/sandbox/linux-sandbox/226/execroot/__main__/external/myonload/src/tools/cplane/server.c:13:10: fatal error: sys/capability.h: No such file or directory
   13 | #include <sys/capability.h>
      |          ^~~~~~~~~~~~~~~~~~
compilation terminated.
make[2]: *** [/home/witek/.cache/bazel/_bazel_witek/52ee13dadb77498b9e4fc952a2e4201b/sandbox/linux-sandbox/226/execroot/__main__/external/myonload/mk/after.mk:151: server.o] Error 1
make[2]: *** Waiting for unfinished jobs....
make[1]: *** [/home/witek/.cache/bazel/_bazel_witek/52ee13dadb77498b9e4fc952a2e4201b/sandbox/linux-sandbox/226/execroot/__main__/external/myonload/src/tools/mmake.mk:20: all] Error 2
make: *** [/home/witek/.cache/bazel/_bazel_witek/52ee13dadb77498b9e4fc952a2e4201b/sandbox/linux-sandbox/226/execroot/__main__/external/myonload/src/mmake.mk:25: all] Error 2
onload_build: ERROR: Failed to build 64-bit user-level components.
...

(some other components do compile, but cplane server does not).

Considering that I do not actually need to build tools to compile my project, instead of fixing it, my preference would be to not build tools, only libraries that I need to link against. I have a separate build to build tools and install them system-wide, but the my binaries that require onload, do not need to rebuild these tools. So skipping build of tools (and tests) would help improve build times.

It is possible to pass CC_FLAGS and pass it to the build, still it is rather complicated, so just not building tools and tests is a better option.

[baryluk]

Any ideas? The current build system is very convoluted and non-standard, so I do not even know how to workaround it.