Open irevoire opened 5 years ago
aspsk
commented
5 years ago @irevoire Your code works for me. What exactly is not working in your case? Can you post the strace output? Does your network device support native mode (I am not sure how -A works, so you can try -S and -N)?
irevoire
commented
5 years ago Hello,
My device should support native mode it's a 82599ES 10-Gigabit SFI/SFP+ nic.
Here is the output with -S
% sudo strace ./xdp_loader -d enp4s0f0 -S
execve("./xdp_loader", ["./xdp_loader", "-d", "enp4s0f0", "-S"], 0x7fff2044b9a8 /* 16 vars */) = 0
brk(NULL) = 0x562dc78e3000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffe43d0c990) = -1 EINVAL (Argument invalide)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (Aucun fichier ou dossier de ce type)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=161920, ...}) = 0
mmap(NULL, 161920, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fcf1e208000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/libelf.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=100184, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fcf1e206000
mmap(NULL, 102424, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fcf1e1ec000
mmap(0x7fcf1e1ef000, 65536, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fcf1e1ef000
mmap(0x7fcf1e1ff000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x7fcf1e1ff000
mmap(0x7fcf1e204000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fcf1e204000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360o\2\0\0\0\0\0"..., 832) = 832
lseek(3, 64, SEEK_SET) = 64
read(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784) = 784
lseek(3, 848, SEEK_SET) = 848
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
lseek(3, 880, SEEK_SET) = 880
read(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\250\257l\201\313(\243{\363\245F\227\v\366B$"..., 68) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2133648, ...}) = 0
lseek(3, 64, SEEK_SET) = 64
read(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784) = 784
lseek(3, 848, SEEK_SET) = 848
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
lseek(3, 880, SEEK_SET) = 880
read(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\250\257l\201\313(\243{\363\245F\227\v\366B$"..., 68) = 68
mmap(NULL, 1844408, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fcf1e029000
mprotect(0x7fcf1e04e000, 1654784, PROT_NONE) = 0
mmap(0x7fcf1e04e000, 1351680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7fcf1e04e000
mmap(0x7fcf1e198000, 299008, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16f000) = 0x7fcf1e198000
mmap(0x7fcf1e1e2000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b8000) = 0x7fcf1e1e2000
mmap(0x7fcf1e1e8000, 13496, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fcf1e1e8000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=91912, ...}) = 0
mmap(NULL, 2187280, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fcf1de12000
mprotect(0x7fcf1de28000, 2093056, PROT_NONE) = 0
mmap(0x7fcf1e027000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7fcf1e027000
close(3) = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fcf1de0f000
arch_prctl(ARCH_SET_FS, 0x7fcf1de0f740) = 0
mprotect(0x7fcf1e1e2000, 12288, PROT_READ) = 0
mprotect(0x7fcf1e027000, 4096, PROT_READ) = 0
mprotect(0x7fcf1e204000, 4096, PROT_READ) = 0
mprotect(0x562dc75aa000, 4096, PROT_READ) = 0
mprotect(0x7fcf1e25a000, 4096, PROT_READ) = 0
munmap(0x7fcf1e208000, 161920) = 0
brk(NULL) = 0x562dc78e3000
brk(0x562dc7904000) = 0x562dc7904000
access("/proc/net", R_OK) = 0
access("/proc/net/unix", R_OK) = 0
socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0) = 3
ioctl(3, SIOCGIFINDEX, {ifr_name="enp4s0f0", }) = 0
close(3) = 0
openat(AT_FDCWD, "learning_kern.o", O_RDONLY) = 3
fcntl(3, F_GETFD) = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=19608, ...}) = 0
pread64(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\1\0\367\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 64, 0) = 64
pread64(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1408, 18200) = 1408
pread64(3, "\0.debug_abbrev\0.text\0.rel.BTF.ex"..., 308, 17888) = 308
pread64(3, "\277\26\0\0\0\0\0\0aa\4\0\0\0\0\0ag\0\0\0\0\0\0\267\2\0\0\0\0\0\0"..., 1464, 64) = 1464
pread64(3, "\310\3\0\0\0\0\0\0\1\0\0\0{\0\0\0`\5\0\0\0\0\0\0\1\0\0\0y\0\0\0", 32, 14352) = 32
pread64(3, "\6\0\0\0\4\0\0\0\20\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\0\0"..., 56, 1528) = 56
pread64(3, "GPL\0", 4, 1584) = 4
pread64(3, "clang version 8.0.1 (tags/RELEAS"..., 999, 1588) = 999
pread64(3, "\0\0\0\0\0\0\0\0\10\0\0\0\0\0\0\0\1\0Q\10\0\0\0\0\0\0\0\270\5\0\0\0"..., 664, 2587) = 664
pread64(3, "\1\21\1%\16\23\5\3\16\20\27\33\16\21\1\22\6\0\0\0024\0\3\16I\23?\31:\v;\v"..., 416, 3251) = 416
pread64(3, "\321\6\0\0\4\0\0\0\0\0\10\1\0\0\0\0\f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1749, 3667) = 1749
pread64(3, "\6\0\0\0\0\0\0\0\n\0\0\0t\0\0\0\f\0\0\0\0\0\0\0\n\0\0\0\2\0\0\0"..., 2256, 14384) = 2256
pread64(3, "\300\3\0\0\0\0\0\0P\4\0\0\0\0\0\0\250\5\0\0\0\0\0\0\270\5\0\0\0\0\0\0"..., 48, 5416) = 48
pread64(3, "\0", 1, 5464) = 1
pread64(3, "\237\353\1\0\30\0\0\0\0\0\0\0\304\5\0\0\304\5\0\0\275\t\0\0\0\0\0\0\0\0\0\2"..., 3993, 5465) = 3993
mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fcf1ddce000
bpf(BPF_BTF_LOAD, {btf="\237\353\1\0\30\0\0\0\0\0\0\0\304\5\0\0\304\5\0\0\275\t\0\0\0\0\0\0\0\0\0\2"..., btf_log_buf=NULL, btf_size=3993, btf_log_size=0, btf_log_level=0}, 112) = 4
munmap(0x7fcf1ddce000, 266240) = 0
pread64(3, "\237\353\1\0\30\0\0\0\0\0\0\0\24\0\0\0\24\0\0\0\254\4\0\0\10\0\0\0\1\0\0\0"..., 1240, 9458) = 1240
pread64(3, "$\0\0\0\0\0\0\0\0\0\0\0r\0\0\08\0\0\0\0\0\0\0\0\0\0\0r\0\0\0"..., 1200, 16640) = 1200
pread64(3, "\f\0\0\0\377\377\377\377\4\0\10\0\10|\v\0\24\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 40, 10704) = 40
pread64(3, "\24\0\0\0\0\0\0\0\n\0\0\0v\0\0\0\30\0\0\0\0\0\0\0\1\0\0\0r\0\0\0", 32, 17840) = 32
pread64(3, "o\2\0\0\4\0\373\0\0\0\10\1\1\373\16\r\0\1\1\1\1\0\0\0\1\0\0\1.\0./"..., 627, 10744) = 627
pread64(3, "\n\1\0\0\0\0\0\0\1\0\0\0r\0\0\0", 16, 17872) = 16
pread64(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\311\0\0\0\4\0\361\377"..., 2976, 11376) = 2976
close(3) = 0
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_SOCKET_FILTER, insn_cnt=2, insns=0x7ffe43d0c4e0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=0, func_info_rec_size=0, func_info=NULL, func_info_cnt=0, line_info_rec_size=0, line_info=NULL, line_info_cnt=0}, 112) = 3
close(3) = 0
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_SOCKET_FILTER, insn_cnt=2, insns=0x7ffe43d0c4e0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="test", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=0, func_info_rec_size=0, func_info=NULL, func_info_cnt=0, line_info_rec_size=0, line_info=NULL, line_info_cnt=0}, 112) = 3
close(3) = 0
bpf(BPF_MAP_CREATE, {map_type=BPF_MAP_TYPE_PERCPU_ARRAY, key_size=4, value_size=16, max_entries=5, map_flags=0, inner_map_fd=0, map_name="xdp_stats_map", map_ifindex=0, btf_fd=0, btf_key_type_id=0, btf_value_type_id=0}, 112) = 3
bpf(BPF_MAP_CREATE, {map_type=BPF_MAP_TYPE_DEVMAP, key_size=4, value_size=4, max_entries=256, map_flags=0, inner_map_fd=0, map_name="tx_port", map_ifindex=0, btf_fd=0, btf_key_type_id=0, btf_value_type_id=0}, 112) = 5
brk(0x562dc794c000) = 0x562dc794c000
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_XDP, insn_cnt=183, insns=0x562dc78e54c0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x562dc78e5c00, func_info_cnt=1, line_info_rec_size=16, line_info=0x562dc78e4ec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_XDP, insn_cnt=183, insns=0x562dc78e54c0, license="GPL", log_level=1, log_size=262144, log_buf="", kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x562dc78e5c00, func_info_cnt=1, line_info_rec_size=16, line_info=0x562dc78e4ec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)
write(2, "libbpf: load bpf program failed:"..., 57libbpf: load bpf program failed: Operation not permitted
) = 57
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_KPROBE, insn_cnt=183, insns=0x562dc78e54c0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x562dc78e5c00, func_info_cnt=1, line_info_rec_size=16, line_info=0x562dc78e4ec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)
write(2, "libbpf: failed to load program '"..., 44libbpf: failed to load program 'xdp_router'
) = 44
close(3) = 0
close(5) = 0
close(4) = 0
write(2, "libbpf: failed to load object 'l"..., 48libbpf: failed to load object 'learning_kern.o'
) = 48
close(4) = -1 EBADF (Mauvais descripteur de fichier)
write(2, "ERR: loading BPF-OBJ file(learni"..., 67ERR: loading BPF-OBJ file(learning_kern.o) (-22): Invalid argument
) = 67
write(2, "ERR: loading file: learning_kern"..., 35ERR: loading file: learning_kern.o
) = 35
exit_group(40) = ?
+++ exited with 40 +++And here is the output with -N:
% sudo strace ./xdp_loader -d enp4s0f0 -N
execve("./xdp_loader", ["./xdp_loader", "-d", "enp4s0f0", "-N"], 0x7ffdbbc53dd8 /* 16 vars */) = 0
brk(NULL) = 0x557545119000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffdf91ae560) = -1 EINVAL (Argument invalide)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (Aucun fichier ou dossier de ce type)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=161920, ...}) = 0
mmap(NULL, 161920, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7faaead72000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/libelf.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=100184, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7faaead70000
mmap(NULL, 102424, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7faaead56000
mmap(0x7faaead59000, 65536, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7faaead59000
mmap(0x7faaead69000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x7faaead69000
mmap(0x7faaead6e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7faaead6e000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360o\2\0\0\0\0\0"..., 832) = 832
lseek(3, 64, SEEK_SET) = 64
read(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784) = 784
lseek(3, 848, SEEK_SET) = 848
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
lseek(3, 880, SEEK_SET) = 880
read(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\250\257l\201\313(\243{\363\245F\227\v\366B$"..., 68) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2133648, ...}) = 0
lseek(3, 64, SEEK_SET) = 64
read(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784) = 784
lseek(3, 848, SEEK_SET) = 848
read(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32) = 32
lseek(3, 880, SEEK_SET) = 880
read(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\250\257l\201\313(\243{\363\245F\227\v\366B$"..., 68) = 68
mmap(NULL, 1844408, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7faaeab93000
mprotect(0x7faaeabb8000, 1654784, PROT_NONE) = 0
mmap(0x7faaeabb8000, 1351680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7faaeabb8000
mmap(0x7faaead02000, 299008, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16f000) = 0x7faaead02000
mmap(0x7faaead4c000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b8000) = 0x7faaead4c000
mmap(0x7faaead52000, 13496, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7faaead52000
close(3) = 0
openat(AT_FDCWD, "/usr/lib/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=91912, ...}) = 0
mmap(NULL, 2187280, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7faaea97c000
mprotect(0x7faaea992000, 2093056, PROT_NONE) = 0
mmap(0x7faaeab91000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7faaeab91000
close(3) = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7faaea979000
arch_prctl(ARCH_SET_FS, 0x7faaea979740) = 0
mprotect(0x7faaead4c000, 12288, PROT_READ) = 0
mprotect(0x7faaeab91000, 4096, PROT_READ) = 0
mprotect(0x7faaead6e000, 4096, PROT_READ) = 0
mprotect(0x557544792000, 4096, PROT_READ) = 0
mprotect(0x7faaeadc4000, 4096, PROT_READ) = 0
munmap(0x7faaead72000, 161920) = 0
brk(NULL) = 0x557545119000
brk(0x55754513a000) = 0x55754513a000
access("/proc/net", R_OK) = 0
access("/proc/net/unix", R_OK) = 0
socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0) = 3
ioctl(3, SIOCGIFINDEX, {ifr_name="enp4s0f0", }) = 0
close(3) = 0
openat(AT_FDCWD, "learning_kern.o", O_RDONLY) = 3
fcntl(3, F_GETFD) = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=19608, ...}) = 0
pread64(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\1\0\367\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 64, 0) = 64
pread64(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1408, 18200) = 1408
pread64(3, "\0.debug_abbrev\0.text\0.rel.BTF.ex"..., 308, 17888) = 308
pread64(3, "\277\26\0\0\0\0\0\0aa\4\0\0\0\0\0ag\0\0\0\0\0\0\267\2\0\0\0\0\0\0"..., 1464, 64) = 1464
pread64(3, "\310\3\0\0\0\0\0\0\1\0\0\0{\0\0\0`\5\0\0\0\0\0\0\1\0\0\0y\0\0\0", 32, 14352) = 32
pread64(3, "\6\0\0\0\4\0\0\0\20\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\0\0"..., 56, 1528) = 56
pread64(3, "GPL\0", 4, 1584) = 4
pread64(3, "clang version 8.0.1 (tags/RELEAS"..., 999, 1588) = 999
pread64(3, "\0\0\0\0\0\0\0\0\10\0\0\0\0\0\0\0\1\0Q\10\0\0\0\0\0\0\0\270\5\0\0\0"..., 664, 2587) = 664
pread64(3, "\1\21\1%\16\23\5\3\16\20\27\33\16\21\1\22\6\0\0\0024\0\3\16I\23?\31:\v;\v"..., 416, 3251) = 416
pread64(3, "\321\6\0\0\4\0\0\0\0\0\10\1\0\0\0\0\f\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1749, 3667) = 1749
pread64(3, "\6\0\0\0\0\0\0\0\n\0\0\0t\0\0\0\f\0\0\0\0\0\0\0\n\0\0\0\2\0\0\0"..., 2256, 14384) = 2256
pread64(3, "\300\3\0\0\0\0\0\0P\4\0\0\0\0\0\0\250\5\0\0\0\0\0\0\270\5\0\0\0\0\0\0"..., 48, 5416) = 48
pread64(3, "\0", 1, 5464) = 1
pread64(3, "\237\353\1\0\30\0\0\0\0\0\0\0\304\5\0\0\304\5\0\0\275\t\0\0\0\0\0\0\0\0\0\2"..., 3993, 5465) = 3993
mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7faaea938000
bpf(BPF_BTF_LOAD, {btf="\237\353\1\0\30\0\0\0\0\0\0\0\304\5\0\0\304\5\0\0\275\t\0\0\0\0\0\0\0\0\0\2"..., btf_log_buf=NULL, btf_size=3993, btf_log_size=0, btf_log_level=0}, 112) = 4
munmap(0x7faaea938000, 266240) = 0
pread64(3, "\237\353\1\0\30\0\0\0\0\0\0\0\24\0\0\0\24\0\0\0\254\4\0\0\10\0\0\0\1\0\0\0"..., 1240, 9458) = 1240
pread64(3, "$\0\0\0\0\0\0\0\0\0\0\0r\0\0\08\0\0\0\0\0\0\0\0\0\0\0r\0\0\0"..., 1200, 16640) = 1200
pread64(3, "\f\0\0\0\377\377\377\377\4\0\10\0\10|\v\0\24\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 40, 10704) = 40
pread64(3, "\24\0\0\0\0\0\0\0\n\0\0\0v\0\0\0\30\0\0\0\0\0\0\0\1\0\0\0r\0\0\0", 32, 17840) = 32
pread64(3, "o\2\0\0\4\0\373\0\0\0\10\1\1\373\16\r\0\1\1\1\1\0\0\0\1\0\0\1.\0./"..., 627, 10744) = 627
pread64(3, "\n\1\0\0\0\0\0\0\1\0\0\0r\0\0\0", 16, 17872) = 16
pread64(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\311\0\0\0\4\0\361\377"..., 2976, 11376) = 2976
close(3) = 0
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_SOCKET_FILTER, insn_cnt=2, insns=0x7ffdf91ae0b0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=0, func_info_rec_size=0, func_info=NULL, func_info_cnt=0, line_info_rec_size=0, line_info=NULL, line_info_cnt=0}, 112) = 3
close(3) = 0
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_SOCKET_FILTER, insn_cnt=2, insns=0x7ffdf91ae0b0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="test", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=0, func_info_rec_size=0, func_info=NULL, func_info_cnt=0, line_info_rec_size=0, line_info=NULL, line_info_cnt=0}, 112) = 3
close(3) = 0
bpf(BPF_MAP_CREATE, {map_type=BPF_MAP_TYPE_PERCPU_ARRAY, key_size=4, value_size=16, max_entries=5, map_flags=0, inner_map_fd=0, map_name="xdp_stats_map", map_ifindex=0, btf_fd=0, btf_key_type_id=0, btf_value_type_id=0}, 112) = 3
bpf(BPF_MAP_CREATE, {map_type=BPF_MAP_TYPE_DEVMAP, key_size=4, value_size=4, max_entries=256, map_flags=0, inner_map_fd=0, map_name="tx_port", map_ifindex=0, btf_fd=0, btf_key_type_id=0, btf_value_type_id=0}, 112) = 5
brk(0x557545182000) = 0x557545182000
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_XDP, insn_cnt=183, insns=0x55754511b4c0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x55754511bc00, func_info_cnt=1, line_info_rec_size=16, line_info=0x55754511aec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_XDP, insn_cnt=183, insns=0x55754511b4c0, license="GPL", log_level=1, log_size=262144, log_buf="", kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x55754511bc00, func_info_cnt=1, line_info_rec_size=16, line_info=0x55754511aec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)
write(2, "libbpf: load bpf program failed:"..., 57libbpf: load bpf program failed: Operation not permitted
) = 57
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_KPROBE, insn_cnt=183, insns=0x55754511b4c0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x55754511bc00, func_info_cnt=1, line_info_rec_size=16, line_info=0x55754511aec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)
write(2, "libbpf: failed to load program '"..., 44libbpf: failed to load program 'xdp_router'
) = 44
close(3) = 0
close(5) = 0
close(4) = 0
write(2, "libbpf: failed to load object 'l"..., 48libbpf: failed to load object 'learning_kern.o'
) = 48
close(4) = -1 EBADF (Mauvais descripteur de fichier)
write(2, "ERR: loading BPF-OBJ file(learni"..., 67ERR: loading BPF-OBJ file(learning_kern.o) (-22): Invalid argument
) = 67
write(2, "ERR: loading file: learning_kern"..., 35ERR: loading file: learning_kern.o
) = 35
exit_group(40) = ?
+++ exited with 40 +++I see that your file have BTF information. The code you posted doesn't compile with BTF info included.
Your strace output:
bpf(BPF_BTF_LOAD, {btf="\237\353\1\0\30\0\0\0\0\0\0\0\304\5\0\0\304\5\0\0\275\t\0\0\0\0\0\0\0\0\0\2"..., btf_log_buf=NULL, btf_size=3993, btf_log_size=0, btf_log_level=0}, 112) = 4
...
bpf(BPF_PROG_LOAD, {prog_type=BPF_PROG_TYPE_XDP, insn_cnt=183, insns=0x55754511b4c0, license="GPL", log_level=0, log_size=0, log_buf=NULL, kern_version=KERNEL_VERSION(0, 0, 0), prog_flags=0, prog_name="xdp_router_func", prog_ifindex=0, expected_attach_type=BPF_CGROUP_INET_INGRESS, prog_btf_fd=4, func_info_rec_size=8, func_info=0x55754511bc00, func_info_cnt=1, line_info_rec_size=16, line_info=0x55754511aec0, line_info_cnt=74}, 112) = -1 EPERM (Opération non permise)So probably try using more recent kernel / libbpf?
irevoire
commented
5 years ago Well I'm using a kernel 5:
% uname -r
5.2.4-arch1-1-ARCHAnd isn't the libbpf directly in the repository ? I copy / pasted it from this repository.
aspsk
commented
5 years ago The libbpf library inside xdp-tutorial is a git submodule pointing to the https://github.com/libbpf/libbpf repo, and it is fixed at some old version (like March 2019). So you might try to using a newer one. However, looks more like a kernel or some other type of misconfiguration, as your code works for me fine.
Are you able to compile and load a simple XDP_PASS program from the same environment which breaks on the learning_kern.o file? Say, are you able to clone a new xdp-tutorial and load the XDP_PASS program from the first lesson?
irevoire
commented
5 years ago Just updated the libbpf to the last version.
I also updated my kernel to linux-5.2.5.arch1-1, but it doesn't change anything.
Also it seems like I have no problem for a simple program
irevoire@irevoire ~/xdp-tutorial/basic01-xdp-pass % sudo ./xdp_pass_user --dev lo --skb-mode
Success: Loading XDP prog name:xdp_prog_simple(id:23) on device:lo(ifindex:1)
irevoire@irevoire ~/xdp-tutorial/basic01-xdp-pass % sudo ./xdp_pass_user --dev enp4s0f0 --skb-mode
Success: Loading XDP prog name:xdp_prog_simple(id:32) on device:enp4s0f0(ifindex:5)Also I tried to run the code like that:
sudo ip link set dev enp4s0f0 xdpgeneric obj learning_kern.o sec xdp_router
sudo ip link set dev enp4s0f1 xdpgeneric obj learning_kern.o sec xdp_routerIt does load the program without error. If I load a program doing only
return XDP_DROPEverything is dropped.
But when I load the learning switch the packet are not forwarded :(
aspsk
commented
5 years ago Is forwarding enabled on your machine?
$ sudo sysctl net.ipv4.conf.all.forwarding=1
$ sudo sysctl net.ipv6.conf.all.forwarding=1Well it is enabled for ipv4 and this is what I used in my tests:
% sudo sysctl net.ipv4.conf.all.forwarding
net.ipv4.conf.all.forwarding = 1
% sudo sysctl net.ipv6.conf.all.forwarding
net.ipv6.conf.all.forwarding = 0I'll retry with ipv6 enabled later but I don't think it will do a difference
aspsk
commented
5 years ago You also can just add some debug printing in your program (I believe xdp_stats won't work as is, because ip places maps differently), see https://github.com/xdp-project/xdp-tutorial/tree/master/tracing03-xdp-debug-print . For example, you can see what is the return value of the bpf_fib_lookup function to understand why it is not forwarding packets.
Also note that all interfaces interacting in your forwarding scheme should have an XDP programs attached (maybe a dummy XDP_PASS). For veth interfaces programs should be attached to both ends.
irevoire
commented
5 years ago Thanks for the answers, right now I need to move, but tuesday I'll try this and update the issue.
irevoire
commented
5 years ago Hello back, so I patched my code like that: https://github.com/irevoire/xdp-learning/commit/ed622c8ebb1e1f71c1d9c747a777072a0a73b0fc I did it in another branch so you can run it yourself.
Then I got this:
% sudo cat /sys/kernel/debug/tracing/trace_pipe
EMT-0-8341 [003] ..s1 8511.085303: 0: bpf fib lookup: 4 <idle>-0 [003] ..s2 8514.927704: 0: bpf fib lookup: 4 <idle>-0 [003] ..s2 8522.093392: 0: bpf fib lookup: 4 <idle>-0 [003] ..s2 8535.917251: 0: bpf fib lookup: 4 <idle>-0 [003] ..s2 8563.827820: 0: bpf fib lookup: 4
^C@irevoire sorry, don't have too much time to debug this (and it works for me). In the last output you provided bpf_fib_lookup returns 4 which is BPF_FIB_LKUP_RET_NOT_FWDED (see the possible return values here: https://github.com/torvalds/linux/blob/master/include/uapi/linux/bpf.h#L3440), so for those packets your program should have been returned XDP_PASS and do not make attempts to redirect them. (BTW, add a newline character in your bpf_printk call for readability.)
If you want it to actually forward packets (those, for which bpf_fib_lookup returned BPF_FIB_LKUP_RET_SUCCESS (0), you also need to setup the redirect map properly. As you are using ip to load the program, you can use bpftool to setup the map manually, like this:
Find the id of the program attached to your device:
# ip -d link show dev woo
8: woo@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 xdpgeneric qdisc noqueue state UP mode DEFAULT group default qlen 1000
link/ether f2:e8:78:bc:08:94 brd ff:ff:ff:ff:ff:ff link-netns woo promiscuity 0
veth addrgenmode eui64 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
prog/xdp id 396 tag 424908ae3c0516ab jitedList the maps attached to the program:
# bpftool prog show id 396
396: xdp tag 424908ae3c0516ab gpl
loaded_at 2019-08-06T14:16:08+0000 uid 0
xlated 1480B jited 820B memlock 4096B map_ids 202,203One of the two maps should be of type devmap:
# bpftool map show id 203
203: devmap flags 0x80
key 4B value 4B max_entries 256 memlock 4096BSetup it to map indexes to NICs (won't work as is if you have ifnums > 255):
# for i in `ip l | awk -F: '/^[0-9]+:/ {print $1}'`; do bpftool map update id 203 key $i 0 0 0 value $i 0 0 0; doneCheck that this worked (I have 4 interfaces):
# bpftool map dump id 203 | grep -v '<no entry>'
key: 01 00 00 00 value: 01 00 00 00
key: 02 00 00 00 value: 02 00 00 00
key: 06 00 00 00 value: 06 00 00 00
key: 08 00 00 00 value: 08 00 00 00
Found 4 elementsAs of your original problem (failing to load the program using xdp_loader), it might be helpful to have the log of all the steps you performed from cloning the xdp-tutorial repo to failing to load the xdp_prog_kern_03.o from the packet-solutions using the xdp_loader program.
irevoire
commented
5 years ago Ok so:
% sudo ip -d link show dev enp4s0f0
7: enp4s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 xdpgeneric qdisc mq state UP mode DEFAULT group default qlen 1000
link/ether 00:1b:21:bf:0e:34 brd ff:ff:ff:ff:ff:ff promiscuity 0 minmtu 68 maxmtu 9710 addrgenmode eui64 numtxqueues 64 numrxqueues 64 gso_max_size 65536 gso_max_segs 65535
prog/xdp id 17 tag 353083d61bf9812f jited Here I can see that my interface have the id 17, I ran the same command for my enp4s0f0 and it have the id 18.
Then:
% sudo bpftool prog show id 17
17: xdp tag 353083d61bf9812f gpl
loaded_at 2019-08-06T17:04:06+0200 uid 0
xlated 1616B jited 891B memlock 4096B map_ids 18,19But then:
% sudo bpftool map show id 17
17: lpm_trie flags 0x1
key 20B value 8B max_entries 1 memlock 4096BHere I don't have the devmap, and for the other interface I don't get the same result:
% sudo bpftool map show id 18
18: percpu_array flags 0x0
key 4B value 16B max_entries 5 memlock 4096BI must admit that I don't get what is happening here :(
I'll reboot and then describe exactly all the step to run xdp_prog_kern_03.o.
aspsk
commented
5 years ago % sudo bpftool prog show id 17 17: xdp tag 353083d61bf9812f gpl loaded_at 2019-08-06T17:04:06+0200 uid 0 xlated 1616B jited 891B memlock 4096B map_ids 18,19But then:
% sudo bpftool map show id 17 17: lpm_trie flags 0x1 key 20B value 8B max_entries 1 memlock 4096B
Because your map_ids are 18,19. So your devmap ID is actually 19, not 17.
I'll reboot and then describe exactly all the step to run
xdp_prog_kern_03.o.
In fact, I was able to reproduce your problem on my Arch machine. Kernel seems to have no problems, as we can load the program using ip, so the problem is with libbpf, most probably. Will try to debug it if I find time.
irevoire
commented
5 years ago Here is the complete trace of what I've done to test the xdp_prog_kern_03:
irevoire@irevoire ~ % cdt
/tmp/tmp.CJhOJ3cIlR
irevoire@irevoire /tmp/tmp.CJhOJ3cIlR % git clone https://github.com/xdp-project/xdp-tutorial --recursive
Clonage dans 'xdp-tutorial'...
remote: Enumerating objects: 52, done.
remote: Counting objects: 100% (52/52), done.
remote: Compressing objects: 100% (36/36), done.
remote: Total 1843 (delta 19), reused 39 (delta 15), pack-reused 1791
Réception d'objets: 100% (1843/1843), 423.14 KiB | 1.28 MiB/s, fait.
Résolution des deltas: 100% (1137/1137), fait.
Sous-module 'libbpf' (https://github.com/libbpf/libbpf/) enregistré pour le chemin 'libbpf'
Clonage dans '/tmp/tmp.CJhOJ3cIlR/xdp-tutorial/libbpf'...
remote: Enumerating objects: 33, done.
remote: Counting objects: 100% (33/33), done.
remote: Compressing objects: 100% (22/22), done.
remote: Total 1051 (delta 19), reused 13 (delta 11), pack-reused 1018
Réception d'objets: 100% (1051/1051), 490.94 KiB | 1.30 MiB/s, fait.
Résolution des deltas: 100% (673/673), fait.
Chemin de sous-module 'libbpf' : '7a431904c8797619f89f469cba0f46d3814e38fc' extrait
irevoire@irevoire /tmp/tmp.CJhOJ3cIlR % cd xdp-tutorial/packet03-redirecting/
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % cp ../packet-solutions/xdp_prog_kern_03.c xdp_prog_kern.c (git)-[master]
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % make (git)-[master]
make[1] : on entre dans le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/libbpf/src »
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c bpf.c -o bpf.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c btf.c -o btf.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c libbpf.c -o libbpf.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c libbpf_errno.c -o libbpf_errno.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c netlink.c -o netlink.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c nlattr.c -o nlattr.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c str_error.c -o str_error.o
cc -I. -I../include -I../include/uapi -g -O2 -Werror -Wall -c libbpf_probes.c -o libbpf_probes.o
ar rcs libbpf.a bpf.o btf.o libbpf.o libbpf_errno.o netlink.o nlattr.o str_error.o libbpf_probes.o
make[1] : on quitte le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/libbpf/src »
make[1] : on entre dans le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/libbpf/src »
if [ ! -d 'root/usr/include/bpf' ]; then install -d -m 755 'root/usr/include/bpf'; fi; install bpf.h libbpf.h btf.h -m 644 'root/usr/include/bpf'
make[1] : on quitte le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/libbpf/src »
make -C ../common
make[1] : on entre dans le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/common »
gcc -g -Wall -I../libbpf/src//root/usr/include/ -I../headers -c -o common_params.o common_params.c
gcc -g -Wall -I../libbpf/src//root/usr/include/ -I../headers -c -o common_user_bpf_xdp.o common_user_bpf_xdp.c
gcc -g -Wall -I../libbpf/src//root/usr/include/ -I../headers -c -o common_libbpf.o common_libbpf.c
make[1] : on quitte le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/common »
cc -Wall -I../libbpf/src//root/usr/include/ -g -I/usr/include/x86_64-linux-gnu -I../headers/ -L../libbpf/src/ -o xdp_prog_user ../common/common_params.o ../common/common_user_bpf_xdp.o \
xdp_prog_user.c -l:libbpf.a -lelf
xdp_prog_user.c: Dans la fonction « parse_mac »:
xdp_prog_user.c:57:1: attention: « return » manquant dans une fonction devant retourner une valeur [-Wreturn-type]
57 | }
| ^
clang -S \
-target bpf \
-D __BPF_TRACING__ \
-I../libbpf/src//root/usr/include/ -g -I/usr/include/x86_64-linux-gnu -I../headers/ \
-Wall \
-Wno-unused-value \
-Wno-pointer-sign \
-Wno-compare-distinct-pointer-types \
-Werror \
-O2 -emit-llvm -c -g -o xdp_prog_kern.ll xdp_prog_kern.c
llc -march=bpf -filetype=obj -o xdp_prog_kern.o xdp_prog_kern.ll
make -C ../common/../basic04-pinning-maps xdp_loader
make[1] : on entre dans le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/basic04-pinning-maps »
cc -Wall -I../libbpf/src//root/usr/include/ -g -I/usr/include/x86_64-linux-gnu -I../headers/ -L../libbpf/src/ -o xdp_loader ../common//common_libbpf.o ../common//common_params.o ../common//common_user_bpf_xdp.o \
xdp_loader.c -l:libbpf.a -lelf
make[1] : on quitte le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/basic04-pinning-maps »
cp ../common/../basic04-pinning-maps/xdp_loader xdp_loader
make -C ../common/../basic04-pinning-maps xdp_stats
make[1] : on entre dans le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/basic04-pinning-maps »
cc -Wall -I../libbpf/src//root/usr/include/ -g -I/usr/include/x86_64-linux-gnu -I../headers/ -L../libbpf/src/ -o xdp_stats ../common//common_libbpf.o ../common//common_params.o ../common//common_user_bpf_xdp.o \
xdp_stats.c -l:libbpf.a -lelf
make[1] : on quitte le répertoire « /tmp/tmp.CJhOJ3cIlR/xdp-tutorial/basic04-pinning-maps »
cp ../common/../basic04-pinning-maps/xdp_stats xdp_stats
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % sudo ip link set enp4s0f0 up (git)-[master]
[sudo] Mot de passe de irevoire :
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % sudo ip link set enp4s0f1 up (git)-[master]
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % sudo sysctl net.ipv4.conf.all.forwarding=1 (git)-[master]
net.ipv4.conf.all.forwarding = 1
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % sudo sysctl net.ipv6.conf.all.forwarding=1 (git)-[master]
net.ipv6.conf.all.forwarding = 1
irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % sudo ./xdp_loader -d enp4s0f0 -S (git)-[master]
libbpf: failed to create map (name: 'redirect_params'): Operation not permitted
libbpf: failed to load object 'xdp_prog_kern.o'
ERR: loading BPF-OBJ file(xdp_prog_kern.o) (-22): Invalid argument
ERR: loading file: xdp_prog_kern.o
40 irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % sudo ./xdp_loader -d enp4s0f0 -N (git)-[master] :(
libbpf: failed to create map (name: 'redirect_params'): Operation not permitted
libbpf: failed to load object 'xdp_prog_kern.o'
ERR: loading BPF-OBJ file(xdp_prog_kern.o) (-22): Invalid argument
ERR: loading file: xdp_prog_kern.o
40 irevoire@irevoire ..cIlR/xdp-tutorial/packet03-redirecting % Ok so I reran all the commands but this time I checked I was really using the map id and not the interface id :grimacing: (sorry for that).
irevoire@irevoire ~/learning/src % sudo ip link set dev enp4s0f0 xdpgeneric obj learning_kern.o sec xdp_router (git)-[trace_fib]
[sudo] Mot de passe de irevoire :
irevoire@irevoire ~/learning/src % sudo ip link set dev enp4s0f1 xdpgeneric obj learning_kern.o sec xdp_router (git)-[trace_fib]
irevoire@irevoire ~/learning/src % sudo ip -d link show dev enp4s0f0 (git)-[trace_fib]
6: enp4s0f0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 xdpgeneric qdisc mq state DOWN mode DEFAULT group default qlen 1000
link/ether 00:1b:21:bf:0e:34 brd ff:ff:ff:ff:ff:ff promiscuity 0 minmtu 68 maxmtu 9710 addrgenmode eui64 numtxqueues 64 numrxqueues 64 gso_max_size 65536 gso_max_segs 65535
prog/xdp id 13 tag c6fc9e1041e88708 jited
irevoire@irevoire ~/learning/src % sudo ip -d link show dev enp4s0f1 (git)-[trace_fib]
9: enp4s0f1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 xdpgeneric qdisc mq state DOWN mode DEFAULT group default qlen 1000
link/ether 00:1b:21:bf:0e:36 brd ff:ff:ff:ff:ff:ff promiscuity 0 minmtu 68 maxmtu 9710 addrgenmode eui64 numtxqueues 64 numrxqueues 64 gso_max_size 65536 gso_max_segs 65535
prog/xdp id 14 tag c6fc9e1041e88708 jited
irevoire@irevoire ~/learning/src % sudo bpftool prog show id 13 (git)-[trace_fib]
13: xdp tag c6fc9e1041e88708 gpl
loaded_at 2019-08-06T18:55:25+0200 uid 0
xlated 1600B jited 884B memlock 4096B map_ids 12,13
irevoire@irevoire ~/learning/src % sudo bpftool prog show id 14 (git)-[trace_fib]
14: xdp tag c6fc9e1041e88708 gpl
loaded_at 2019-08-06T18:55:31+0200 uid 0
xlated 1600B jited 884B memlock 4096B map_ids 14,15
irevoire@irevoire ~/learning/src % sudo bpftool map show id 12 (git)-[trace_fib] :(
12: percpu_array flags 0x0
key 4B value 16B max_entries 5 memlock 4096B
irevoire@irevoire ~/learning/src % sudo bpftool map show id 13 (git)-[trace_fib]
13: devmap flags 0x0
key 4B value 4B max_entries 256 memlock 4096B
irevoire@irevoire ~/learning/src % sudo bpftool map show id 15 (git)-[trace_fib]
15: devmap flags 0x0
key 4B value 4B max_entries 256 memlock 4096BSo here my map id should be either 13 or 15 if I understood well.
Then:
root@irevoire /home/irevoire/learning/src # for i in `ip l | awk -F: '/^[0-9]+:/ {print $1}'`; do bpftool map update id 13 key $i 0 0 0 value $i 0 0 0; done
irevoire@irevoire ~/learning/src % sudo bpftool map dump id 13 | grep -v '<no entry>' (git)-[trace_fib]
[sudo] Mot de passe de irevoire :
key: 01 00 00 00 value: 01 00 00 00
key: 02 00 00 00 value: 02 00 00 00
key: 03 00 00 00 value: 03 00 00 00
key: 04 00 00 00 value: 04 00 00 00
key: 05 00 00 00 value: 05 00 00 00
key: 06 00 00 00 value: 06 00 00 00
key: 07 00 00 00 value: 07 00 00 00
key: 08 00 00 00 value: 08 00 00 00
key: 09 00 00 00 value: 09 00 00 00
key: 0a 00 00 00 value: 0a 00 00 00
Found 10 elementsHere I have my 10 interfaces so I retried running a ping between the two interface after giving them ip address and I still got the same error:
irevoire@irevoire /tmp/tmp.khIW6K56pi % sudo cat /sys/kernel/debug/tracing/trace_pipe :(
<idle>-0 [007] ..s2 2196.447013: 0: bpf fib lookup: 4
<idle>-0 [007] .Ns2 2200.761904: 0: bpf fib lookup: 4
<idle>-0 [007] ..s2 2209.466985: 0: bpf fib lookup: 4
<idle>-0 [007] ..s2 2229.186947: 0: bpf fib lookup: 4
^CI'm wondering, which distribution are you using ?
aspsk
commented
5 years ago Don't forget to run this command for the second map:
# for i in `ip l | awk -F: '/^[0-9]+:/ {print $1}'`; do bpftool map update id 15 key $i 0 0 0 value $i 0 0 0; doneand to enable forwarding on host:
echo 1 > /proc/sys/net/ipv4/conf/all/forwarding
echo 1 > /proc/sys/net/ipv6/conf/all/forwardingI've checked that this (xdp_router program from packet-solutions) works for me on Arch (kernel is 5.2.6-arch1-1-ARCH). I've redirected packets between two test environments, as in the tutorial.
Also, I don't know what the test you are running. Be sure that packets you are sending really need to be redirected from one of your interfaces to the other.
netoptimizer
commented
5 years ago Is forwarding enabled on your machine?
$ sudo sysctl net.ipv4.conf.all.forwarding=1 $ sudo sysctl net.ipv6.conf.all.forwarding=1
We should make it more clear that this is needed (e.g. via a comment in the code).
Return value: BPF_FIB_LKUP_RET_FWD_DISABLED == 5
P.s. I'm going to update xdp_fwd_kern.c (in kernel tree).
netoptimizer
commented
5 years ago irevoire@irevoire /tmp/tmp.khIW6K56pi % sudo cat /sys/kernel/debug/tracing/trace_pipe :( <idle>-0 [007] ..s2 2196.447013: 0: bpf fib lookup: 4 <idle>-0 [007] .Ns2 2200.761904: 0: bpf fib lookup: 4 <idle>-0 [007] ..s2 2209.466985: 0: bpf fib lookup: 4 <idle>-0 [007] ..s2 2229.186947: 0: bpf fib lookup: 4 ^C
Based on irevoire/xdp-learning@ed622c8 you are printing the return code from bpf_fib_lookup(). The return code values are defined here.
Return code 4 => BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */
Thus, bpf_fib_lookup() clearly are refusing to forward your packet...
When I read the kernel code for bpf_ipv4_fib_lookup(), then this return code BPF_FIB_LKUP_RET_NOT_FWDED can be returned in a number of places. Thus, its not easy to say exactly why...
The first/next step is to assure that your IPv4 forwarding setup works without the XDP program loaded.
irevoire
commented
5 years ago Ok so I'll try to describe my setup:
On the top it's the interface enp1s0fX which do not support ebpf offloading.
So first I connect it like that, up all the interface and then ping myself from the two pair of interface. To be sure the packet goes in the cable I setup a VM on one of the two interface with an IP and give another in the same network on my host machine.
This way I'm sure all the interface / cable works.
Then I connect everything like this:
Load the learning program on the enp4s0fX interfaces (which are the two other supporting ebpf). And that's how I did all the previous tests.
Also yes, the forwarding is enabled in all the previous test once it was mentioned:
irevoire@irevoire ~ % sudo sysctl net.ipv4.conf.all.forwarding
net.ipv4.conf.all.forwarding = 1
irevoire@irevoire ~ % sudo sysctl net.ipv6.conf.all.forwarding
net.ipv6.conf.all.forwarding = 1Hello,
After following all the previous exercise I had a hard time to make the last one works. So I just tried to use the solution in the packet-solutions directory and it was still not working with this error when loading the code:
% sudo ./xdp_loader -d enp4s0f0 -A [sudo] Mot de passe de irevoire : libbpf: load bpf program failed: Operation not permitted libbpf: failed to load program 'xdp_router' libbpf: failed to load object 'learning_kern.o' ERR: loading BPF-OBJ file(learning_kern.o) (-22): Invalid argument ERR: loading file: learning_kern.oHere I've made a minimal directory with exactly the code I ran: https://github.com/irevoire/xdp-learning I don't know if I missed something or if it's a problem in my config.
@irevoire I probably found the reason why you can't load your programs using xdp_loader on Arch—the low default ulimit -l (the maximum size a process may lock into memory). Try doing this instead of your sudo command:
% sudo su
# ulimit -l unlimited
# ./xdp_loader -d enp4s0f0 -AHello @aspsk, I just tried:
root@irevoire /home/irevoire/learning/src # ulimit -l
64
root@irevoire /home/irevoire/learning/src # ulimit -l unlimited
root@irevoire /home/irevoire/learning/src # ./xdp_loader -d enp4s0f0 -A
libbpf: load bpf program failed: Operation not permitted
libbpf: failed to load program 'xdp_router'
libbpf: failed to load object 'learning_kern.o'
ERR: loading BPF-OBJ file(learning_kern.o) (-22): Invalid argument
ERR: loading file: learning_kern.o
40 root@irevoire /home/irevoire/learning/src # ulimit -l
64
root@irevoire /home/irevoire/learning/src # ulimit -l 1024
root@irevoire /home/irevoire/learning/src # ulimit -l
1024
root@irevoire /home/irevoire/learning/src # ./xdp_loader -d enp4s0f0 -A
Success: Loaded BPF-object(learning_kern.o) and used section(xdp_router)
- XDP prog attached on device:enp4s0f0(ifindex:6)
- Pinning maps in /sys/fs/bpf/enp4s0f0/I don't know why the unlimited doesn't works for me, but the bug clearly come from ulimit :+1:
After a new test, obviously the forwarding still doesn't works.
irevoire
commented
5 years ago Hello back, I tried something else:
So for the second step I created a linux bridge and found out the forwarding didn't works because the default setup of iptable was dropping my paquets. After running the following command the forwarding was working:
# sysctl -w net.bridge.bridge-nf-call-iptables=0
# sysctl -w net.bridge.bridge-nf-call-arptables=0Then:
root@irevoire /home/irevoire/learning/src # ./xdp_loader -d enp4s0f0 -A --force
Success: Loaded BPF-object(learning_kern.o) and used section(xdp_router)
- XDP prog attached on device:enp4s0f0(ifindex:6)
- Unpinning (remove) prev maps in /sys/fs/bpf/enp4s0f0/
- Pinning maps in /sys/fs/bpf/enp4s0f0/
root@irevoire /home/irevoire/learning/src # ./xdp_loader -d enp4s0f1 -A --force
Success: Loaded BPF-object(learning_kern.o) and used section(xdp_router)
- XDP prog attached on device:enp4s0f1(ifindex:9)
- Unpinning (remove) prev maps in /sys/fs/bpf/enp4s0f1/
- Pinning maps in /sys/fs/bpf/enp4s0f1/
root@irevoire /home/irevoire/learning/src # cat /sys/kernel/debug/tracing/trace_pipe
<idle>-0 [006] ..s1 3976.005540: 0: bpf fib lookup: 4
<idle>-0 [001] ..s1 3976.005741: 0: bpf fib lookup: 4
<idle>-0 [003] ..s1 3976.102461: 0: bpf fib lookup: 4
<idle>-0 [003] ..s1 3976.204087: 0: bpf fib lookup: 4
<idle>-0 [003] .Ns1 3977.107971: 0: bpf fib lookup: 4
<idle>-0 [003] ..s1 3977.212499: 0: bpf fib lookup: 4
<idle>-0 [007] ..s1 3977.701876: 0: bpf fib lookup: 0
<idle>-0 [007] ..s1 3977.701881: 0: bpf fib lookup: 0
ksoftirqd/3-35 [003] ..s. 3977.701885: 0: bpf fib lookup: 4
ksoftirqd/3-35 [003] ..s. 3977.701905: 0: bpf fib lookup: 4
<idle>-0 [003] ..s1 3978.706821: 0: bpf fib lookup: 4
<idle>-0 [007] ..s1 3978.706823: 0: bpf fib lookup: 0
<idle>-0 [003] ..s1 3978.706929: 0: bpf fib lookup: 4
<idle>-0 [007] ..s1 3978.706930: 0: bpf fib lookup: 0
<idle>-0 [001] ..s1 3980.095364: 0: bpf fib lookup: 4
<idle>-0 [003] ..s1 3980.665609: 0: bpf fib lookup: 4
<idle>-0 [007] ..s1 3980.665611: 0: bpf fib lookup: 0
<idle>-0 [003] ..s1 3980.770075: 0: bpf fib lookup: 4
<idle>-0 [007] ..s1 3980.770076: 0: bpf fib lookup: 0
<idle>-0 [003] ..s1 3982.664414: 0: bpf fib lookup: 4
...It still does not works, but sometimes I got a 0 with bpf_fib_lookup.
I don't know if this help
aspsk
commented
5 years ago @irevoire Forwarding works indeed, as you are seeing packets for which bpf_fib_lookup returns 0. It is normal that not all packets which come from your interface are forwarded, some of them are to be delivered to your host. If you are in doubts, add more debug output (L3 type, src MAC, dst MAC, src IP, dest IP) for each packet, until you see exactly which packets are delivered and which are forwarded.
This is, however, out of the scope of this issue, so I think you should close it. (You still will be able to post comments here.)
irevoire
commented
5 years ago Well I mean... If I see some packets being forwarded when adding a print in the code but I'm still unable to do a simple ping we can still consider that the packet03 solution have a problem no?
aspsk
commented
5 years ago @irevoire You are running your own copy of code with a custom network setup, not the packet03 solution. This issue makes sense to be kept open if the published solution doesn't work with the setup described in the lesson.
For you case, if you are expecting a particular packet to be forwarded, but it is not received by the other side, then you should find its actual path and where it gets dropped. Before and after sending a packet which is expected to be forwarded look at 1) your debug output 2) output of xdp_stast for both interfaces 3) any additional debug printouts which you may put in. Check in advance that 1) forwarding works without attached XDP programs 2) redirect maps are configured properly for both interfaces after attaching XDP programs, etc.
Hello,
After following all the previous exercise I had a hard time to make the last one works. So I just tried to use the solution in the packet-solutions directory and it was still not working with this error when loading the code:
Here I've made a minimal directory with exactly the code I ran: https://github.com/irevoire/xdp-learning I don't know if I missed something or if it's a problem in my config.