libxdp and libbpf 1.4.6 RLIMIT_MEMLOCK is set to big enough value

[vincentmli]

Hi @tohojo switch to rlimit mlock issue. now xdp-tools is built with libbpf 1.4.6 and same code here without rlimit mlock setting in https://github.com/vincentmli/suricata/blob/suricata-bpfire-stack-smashing/src/util-ebpf.c#L358-L474

int EBPFLoadMultiXDPFile(AFPIfaceConfig *aconf, const char *path, uint32_t prio,
                 int *val, struct ebpf_timeout_config *config)
{
    int err;
    char buf[129];
    struct xdp_program *p = NULL;
    struct bpf_object *bpfobj = NULL;
    struct bpf_program *bpfprog = NULL;
    struct bpf_map *map = NULL;

    const char *iface = aconf->iface;
    unsigned int run_prio = (unsigned int)prio;

    DECLARE_LIBXDP_OPTS(xdp_program_opts, xdp_opts, 0);

    if (iface == NULL)
        return -1;
    LiveDevice *livedev = LiveGetDevice(iface);
    if (livedev == NULL)
        return -1;

    unsigned int ifindex = if_nametoindex(iface);
    if (ifindex == 0) {
        SCLogError(SC_ERR_INVALID_VALUE,
                "Unknown interface '%s'", iface);
        return -1;
    }

    if (!strcmp(aconf->xdp_filter_file, path)) {
        if (config->flags & EBPF_XDP_CODE && config->flags & EBPF_PINNED_MAPS) {
        /* We try to get our flow table maps and if we have them we can simply return */
            if (EBPFLoadPinnedMaps(livedev, config) == 0) {
                SCLogInfo("Loaded pinned maps, will use already loaded eBPF filter");
                return 1;
            }
        }
    }

    if (! path) {
        SCLogError(SC_ERR_INVALID_VALUE, "No file defined to load eBPF from");
        return -1;
    }

    xdp_opts.open_filename = path;
    p = xdp_program__create(&xdp_opts);
    err = libxdp_get_error(p);
    if (err) {
        libxdp_strerror(err, buf, sizeof(buf));
        SCLogError(SC_ERR_INVALID_VALUE, "Unable to create multi xdp program on '%s': %s (%d)",
            iface, buf, err);
       p = NULL;
       goto error_out;
     }

    bpfobj = xdp_program__bpf_obj(p);
    err = libxdp_get_error(bpfobj);
    if (err) {
        libxdp_strerror(err, buf, sizeof(buf));
        SCLogError(SC_ERR_INVALID_VALUE, "Unable to get multi XDP bpf object on '%s': %s (%d)",
            iface, buf, err);
       p = NULL;
       goto error_out;
     }

    if (config->flags & EBPF_XDP_HW_MODE) {
        bpf_object__for_each_program(bpfprog, bpfobj) {
            bpf_program__set_ifindex(bpfprog, ifindex);
        }
        bpf_map__for_each(map, bpfobj) {
            bpf_map__set_ifindex(map, ifindex);
        }
    }

    if (run_prio) {
        err = xdp_program__set_run_prio(p, run_prio);
        if (err != 0) {
            libxdp_strerror(err, buf, sizeof(buf));
            SCLogWarning(SC_ERR_INVALID_VALUE, "Unable to set run prio multi XDP on '%s': %s (%d)",
                iface, buf, err);
        }
    }

    err = xdp_program__set_chain_call_enabled(p, XDP_PASS, true);
    if (err) {
        libxdp_strerror(err, buf, sizeof(buf));
        SCLogError(SC_ERR_INVALID_VALUE, "Unable to set multi XDP chain call action on '%s': %s (%d)",
                iface, buf, err);
        goto error_out;
    }

    xdp_program__print_chain_call_actions(p, buf, sizeof(buf));
    SCLogInfo("XDP program: %s. Run prio: %d. Chain call actions: %s\n",
                         path, xdp_program__run_prio(p), buf);

    err = xdp_program__attach(p, ifindex, aconf->xdp_mode, 0);
    if (err != 0) {
        libxdp_strerror(err, buf, sizeof(buf));
        SCLogError(SC_ERR_INVALID_VALUE, "Unable to attach multi XDP on '%s': %s (%d)",
                iface, buf, err);
        goto error_out;
    }

    /* Store map data from bpf object */
    if (!strcmp(aconf->xdp_filter_file, path)) {
        if ( EBPFStoreBpfObjMapsData(iface, bpfobj, config) == 0)
            SCLogInfo("Stored %s map data successfully!", path);
    }

    SCLogInfo("Successfully loaded eBPF file '%s' on '%s'", path, iface);

    *val = xdp_program__fd(p);
    return 0;

error_out:
    xdp_program__close(p);
    return -1;
}

run suricata in BPFire result in error, it seems it can't even load /usr/lib/bpf/xdp-dispatcher.o

[root@bpfire ~]# suricata -c /etc/suricata/suricata-xdp.yaml --af-packet -vvv

libbpf: elf: skipping unrecognized data section(7) .xdp_run_config
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libxdp: Failed to load dispatcher: Operation not permitted
libxdp: Falling back to loading single prog without dispatcher
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp_filter.bpf'
libbpf: elf: skipping unrecognized data section(7) .xdp_run_config
libxdp: couldn't get program fd: Operation not permittedlibbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libxdp: Failed to load dispatcher: Operation not permitted
libxdp: Falling back to loading single prog without dispatcher
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: Error in bpf_object__probe_loading():Operation not permitted(1). Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.
libbpf: failed to load object '/usr/lib/bpf/xdp_filter.bpf'

BPFire kernel runs upstream stable kernel 6.10.11. I tried to manually set mlock memory by ulimit -l unlimited at command line and then run suricata from command line again, same error.

[vincentmli]

suricata has config

# Run suricata as user and group.
run-as:
  user: suricata
  group: suricata

attempted to change to user and group to root, no effect. I have no issue run suricata in Ubuntu 22.04, there maybe some OS level setting in BPFire that is stopping this ? tried to use debug method in https://lpc.events/event/11/contributions/933/attachments/887/1729/LPC%202021%20BPF%20User%20Experience%20Rough%20Edges.pdf, but not sure what I should look for :).

[vincentmli]

error slightly changed after setting kernel.unprivileged_bpf_disabled = 0

[root@bpfire bin]# suricata -c /etc/suricata/suricata-xdp.yaml --af-packet -vvv

libbpf: elf: skipping unrecognized data section(7) .xdp_run_config
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: prog 'xdp_dispatcher': BPF program load failed: Operation not permitted
libbpf: prog 'xdp_dispatcher': failed to load: -1
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: prog 'xdp_dispatcher': BPF program load failed: Operation not permitted
libbpf: prog 'xdp_dispatcher': failed to load: -1
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libxdp: Failed to load dispatcher: Operation not permitted
libxdp: Falling back to loading single prog without dispatcher
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: prog 'xdp_dispatcher': BPF program load failed: Operation not permitted
libbpf: prog 'xdp_dispatcher': failed to load: -1
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: map 'cpu_map': failed to create: Operation not permitted(-1)
libbpf: failed to load object '/usr/lib/bpf/xdp_filter.bpf'
libbpf: elf: skipping unrecognized data section(7) .xdp_run_config
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: prog 'xdp_dispatcher': BPF program load failed: Operation not permitted
libbpf: prog 'xdp_dispatcher': failed to load: -1
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: prog 'xdp_dispatcher': BPF program load failed: Operation not permitted
libbpf: prog 'xdp_dispatcher': failed to load: -1
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libxdp: Failed to load dispatcher: Operation not permitted
libxdp: Falling back to loading single prog without dispatcher
libbpf: elf: skipping unrecognized data section(7) xdp_metadata
libbpf: prog 'xdp_dispatcher': BPF program load failed: Operation not permitted
libbpf: prog 'xdp_dispatcher': failed to load: -1
libbpf: failed to load object '/usr/lib/bpf/xdp-dispatcher.o'
libbpf: map 'cpu_map': failed to create: Operation not permitted(-1)
libbpf: failed to load object '/usr/lib/bpf/xdp_filter.bpf'

I have kernel config below, set CONFIG_BPF_UNPRIV_DEFAULT_OFF to n ?

[root@bpfire boot]# grep 'CONFIG_BPF_UNPRIV_DEFAULT_OFF' /boot/config-6.10.11-ipfire 
CONFIG_BPF_UNPRIV_DEFAULT_OFF=y

[vincentmli]

here is the strace -s1024 -f suricata -c /etc/suricata/suricata-xdp.yaml --af-packet -vvv after removing CONFIG_BPF_UNPRIV_DEFAULT_OFF=y kernel config, it looks user id 101 suricata is the user to load/attach the XDP program xdp_filter.bpf and has EPERM (Operation not permitted) with memlock. maybe I should keep CONFIG_BPF_UNPRIV_DEFAULT_OFF=y and just set kernel.unprivileged_bpf_disabled = 0 and do the strace.

3297  setresgid(101, 101, 101)          = 0
3297  setgroups(0, NULL)                = 0
3297  setresuid(101, 101, 101)          = 0
3297  prctl(PR_SET_KEEPCAPS, 0)         = 0
3297  capset({version=_LINUX_CAPABILITY_VERSION_3, pid=3297}, {effective=1<<CAP_NET_ADMIN|1<<CAP_NET_RAW|1<<CAP_SYS_NICE, permitted=1<<CAP_NET_ADMIN|1<<CAP_NET_RAW|1<<CAP_SYS_NICE, inheritable=0}) = 0
3297  prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0) = 0
3297  sendto(3, "<174>Oct 17 19:08:39 suricata: dropped the caps for main thread", 63, MSG_NOSIGNAL, NULL, 0) = 63
3297  prctl(PR_GET_DUMPABLE)            = 0 (SUID_DUMP_DISABLE)
3297  prctl(PR_SET_DUMPABLE, SUID_DUMP_USER) = 0

3297  bpf(BPF_OBJ_GET, {pathname="/sys/fs/bpf/suricata-red0-flow_table_v4", bpf_fd=0, file_flags=0, path_fd=0}, 20) = -1 ENOENT (No such file or directory)
3297  prlimit64(0, RLIMIT_MEMLOCK, {rlim_cur=RLIM64_INFINITY, rlim_max=RLIM64_INFINITY}, NULL) = -1 EPERM (Operation not permitted)
3297  sendto(3, "<171>Oct 17 19:08:39 suricata: [ERRCODE: SC_ERR_MEM_ALLOC(1)] - Unable to lock memory: Operation not permitted (1)", 114, MSG_NOSIGNAL, NULL, 0) = 114
3297  sendto(3, "<172>Oct 17 19:08:39 suricata: [ERRCODE: SC_ERR_INVALID_VALUE(130)] - Error when loading XDP filter file", 104, MSG_NOSIGNAL, NULL, 0) = 104
3297  socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)) = 5

3297  bpf(BPF_OBJ_GET, {pathname="/sys/fs/bpf/suricata-green0-flow_table_v4", bpf_fd=0, file_flags=0, path_fd=0}, 20) = -1 ENOENT (No such file or directory)
3299  <... clock_nanosleep resumed>NULL) = 0
3297  prlimit64(0, RLIMIT_MEMLOCK, {rlim_cur=RLIM64_INFINITY, rlim_max=RLIM64_INFINITY},  <unfinished ...>
3299  clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=0, tv_nsec=100000},  <unfinished ...>
3297  <... prlimit64 resumed>NULL)      = -1 EPERM (Operation not permitted)
3297  sendto(3, "<171>Oct 17 19:08:40 suricata: [ERRCODE: SC_ERR_MEM_ALLOC(1)] - Unable to lock memory: Operation not permitted (1)", 114, MSG_NOSIGNAL, NULL, 0 <unfinished ...>

3298  <... clock_nanosleep resumed>NULL) = 0
3297  <... sendto resumed>)             = 114
3298  clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=0, tv_nsec=100000},  <unfinished ...>
3297  sendto(3, "<172>Oct 17 19:08:40 suricata: [ERRCODE: SC_ERR_INVALID_VALUE(130)] - Error when loading XDP filter file", 104, MSG_NOSIGNAL, NULL, 0 <unfinished ...>
3299  <... clock_nanosleep resumed>NULL) = 0
3297  <... sendto resumed>)             = 104

[tohojo]

Don't think the unprivileged BPF setting will help you. You need CAP_BPF in the init namespace to use BPF, and at least CAP_NET_ADMIN in whatever namespace you're running. May need CAP_SYS_ADMIN as well. So basically, you'll need to run as root...

[vincentmli]

I am not running in namespace or container, BPFire is a standalone complete OS that is fork of open source firewall IPFire, like pfsense, openwrt...the problem is after suricata started, it switches to non-root user suricata that does not have capability as root. needs to figure out how to load/attach XDP program for non-root user, in this sense, it is sort of like namespace/container.

[vincentmli]

but I wonder why I don't have the issue when running suricata in Ubuntu that requires no special capability setting for non root user suricata.

[vincentmli]

but I wonder why I don't have the issue when running suricata in Ubuntu that requires no special capability setting for non root user suricata.

it turned out in Ubuntu, suricata runs as root, which can workaround this operation permission issue. sorry for the noise :).

# Run Suricata with a specific user-id and group-id:
#run-as:
#  user: suri
#  group: suri