search

mestery / xdp-skeleton

Example XDP code and a Vagrant environment to run it in
BSD 3-Clause "New" or "Revised" License
2 stars 0 forks source link

XDP program won't load on Ubuntu Jammy 64 #2

Open mestery opened 2 years ago

mestery commented 2 years ago

I'm trying to update the VM and docker containers to Ubuntu Jammy. This is working ok, and I can get things to compile, but when I try to load the XDP program, it fails and claims: kernel subsystem misconfigured verifier. The full error message is as follows:

vagrant@xdp-skeleton:/git/src$ sudo /git/scripts/start.sh
+ for i in /proc/sys/net/ipv4/conf/*/rp_filter
+ echo 0
+ for i in /proc/sys/net/ipv4/conf/*/rp_filter
+ echo 0
+ for i in /proc/sys/net/ipv4/conf/*/rp_filter
+ echo 0
+ for i in /proc/sys/net/ipv4/conf/*/rp_filter
+ echo 0
+ for i in /proc/sys/net/ipv4/conf/*/rp_filter
+ echo 0
+ ip netns add xdp
+ ip link add veth1 type veth peer name veth2
+ ip link set veth2 netns xdp
+ ip add add 10.2.2.1/24 dev veth1
+ ip link set dev veth1 up addr 52:54:27:27:27:25
+ ip netns exec xdp ip addr add 10.2.2.2/24 dev veth2
+ ip netns exec xdp ip link set dev veth2 up addr 52:54:27:27:27:26
+ ip link set dev veth1 mtu 9000
+ ip netns exec xdp ip link set dev veth2 mtu 9000
+ sysctl -w net.core.rmem_default=134217728
net.core.rmem_default = 134217728
+ sysctl -w net.core.rmem_max=268435456
net.core.rmem_max = 268435456
+ sysctl -w net.core.wmem_default=134217728
net.core.wmem_default = 134217728
+ sysctl -w net.core.wmem_max=268435456
net.core.wmem_max = 268435456
+ sysctl -w 'net.ipv4.tcp_rmem=134217728 134217728 134217728'
net.ipv4.tcp_rmem = 134217728 134217728 134217728
+ sysctl -w 'net.ipv4.tcp_wmem=134217728 134217728 134217728'
net.ipv4.tcp_wmem = 134217728 134217728 134217728
+ sysctl -w 'net.ipv4.tcp_mem=134217728 134217728 134217728'
net.ipv4.tcp_mem = 134217728 134217728 134217728
+ sysctl -w 'net.ipv4.udp_mem=134217728 134217728 134217728'
net.ipv4.udp_mem = 134217728 134217728 134217728
+ sysctl -w net.ipv4.udp_rmem_min=134217728
net.ipv4.udp_rmem_min = 134217728
+ sysctl -w 'net.ipv4.udp_wmem_min=134217728 134217728 134217728'
net.ipv4.udp_wmem_min = 134217728 134217728 134217728
+ ip netns exec xdp sysctl -w 'net.ipv4.tcp_rmem=134217728 134217728 134217728'
net.ipv4.tcp_rmem = 134217728 134217728 134217728
+ ip netns exec xdp sysctl -w 'net.ipv4.tcp_wmem=134217728 134217728 134217728'
net.ipv4.tcp_wmem = 134217728 134217728 134217728
+ ip netns exec xdp sysctl -w net.ipv4.udp_rmem_min=134217728
net.ipv4.udp_rmem_min = 134217728
+ ip netns exec xdp sysctl -w 'net.ipv4.udp_wmem_min=134217728 134217728 134217728'
net.ipv4.udp_wmem_min = 134217728 134217728 134217728
+ ethtool -K veth1 tso off
+ ip netns exec xdp ethtool -K veth2 tso off
+ ulimit -l unlimited
+ '[' '!' -d /sys/fs/bpf ']'
+ /git/src/user/xdploader -file /git/src/kern/xdp.elf -i veth1
Dumping info for ELF file /git/src/kern/xdp.elf
Maps:
    if_redirect: Device map, Fd 8
    jmp_map: Array of programs, Fd 9
    tail_call: Per-CPU array, Fd 10
    stats_map: Per-CPU array, Fd 7

Programs:
    xdp_skeleton_2: XDP, size 344, license "GPL"
    xdp_skeleton_1: XDP, size 1240, license "GPL"
    xdp_skeleton: XDP, size 736, license "GPL"

Loading tail call jumps
Loading program name xdp_skeleton_2
Looking at program slot (2)
Converting string (2)
Trying to insert FD (3) into position (2) in jmp_map
Loading program name xdp_skeleton_1
Looking at program slot (1)
Error loading program xdp_skeleton_1:
panic: ebpf_prog_load() failed: 0: (bf) r6 = r1
1: (85) call bpf_get_smp_processor_id#8
2: (63) *(u32 *)(r10 -4) = r0
3: (61) r9 = *(u32 *)(r6 +4)
4: (61) r8 = *(u32 *)(r6 +0)
5: (b7) r7 = 0
6: (73) *(u8 *)(r10 -30) = r7
last_idx 6 first_idx 0
regs=80 stack=0 before 5: (b7) r7 = 0
7: (b7) r1 = 2670
8: (6b) *(u16 *)(r10 -32) = r1
9: (18) r1 = 0x6f6974636e756620
11: (7b) *(u64 *)(r10 -40) = r1
12: (18) r1 = 0x676e697265746e45
14: (7b) *(u64 *)(r10 -48) = r1
15: (18) r1 = 0x203a315f6e6f7465
17: (7b) *(u64 *)(r10 -56) = r1
18: (18) r1 = 0x6c656b735f706478
20: (7b) *(u64 *)(r10 -64) = r1
21: (bf) r1 = r10
22: (07) r1 += -64
23: (b7) r2 = 35
24: (85) call bpf_trace_printk#6
last_idx 24 first_idx 0
regs=4 stack=0 before 23: (b7) r2 = 35
25: (bf) r1 = r8
26: (07) r1 += 14
27: (2d) if r1 > r9 goto pc+105
 R0=inv(id=0) R1_w=pkt(id=0,off=14,r=14,imm=0) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=14,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
28: (bf) r1 = r8
29: (07) r1 += 34
30: (2d) if r1 > r9 goto pc+102
 R0=inv(id=0) R1_w=pkt(id=0,off=34,r=34,imm=0) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
31: (71) r1 = *(u8 *)(r8 +12)
32: (71) r2 = *(u8 *)(r8 +13)
33: (67) r2 <<= 8
34: (4f) r2 |= r1
35: (55) if r2 != 0x8 goto pc+97
 R0=inv(id=0) R1=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2=inv8 R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
36: (bf) r2 = r10
37: (07) r2 += -4
38: (18) r1 = 0xffff9dbf88277a00
40: (85) call bpf_map_lookup_elem#1
41: (55) if r0 != 0x0 goto pc+10

from 41 to 52: R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
52: (71) r1 = *(u8 *)(r0 +0)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
53: (71) r2 = *(u8 *)(r0 +1)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
54: (67) r2 <<= 8
55: (4f) r2 |= r1
56: (dc) r2 = be16 r2
57: (71) r1 = *(u8 *)(r8 +23)
58: (5d) if r2 != r1 goto pc+54
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-32=?????mmm fp-40=mmmmmmmm fp-48=mmmmmmmm fp-56=mmmmmmmm fp-64=mmmmmmmm
59: (b7) r1 = 10
60: (6b) *(u16 *)(r10 -12) = r1
61: (b7) r1 = 2037148789
62: (63) *(u32 *)(r10 -16) = r1
63: (18) r1 = 0x6673736563637573
65: (7b) *(u64 *)(r10 -24) = r1
66: (18) r1 = 0x2029782528206c6f
68: (7b) *(u64 *)(r10 -32) = r1
69: (18) r1 = 0x636f746f72702064
71: (7b) *(u64 *)(r10 -40) = r1
72: (18) r1 = 0x65746164696c6156
74: (7b) *(u64 *)(r10 -48) = r1
75: (18) r1 = 0x203a315f6e6f7465
77: (7b) *(u64 *)(r10 -56) = r1
78: (18) r1 = 0x6c656b735f706478
80: (7b) *(u64 *)(r10 -64) = r1
81: (71) r2 = *(u8 *)(r0 +1)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv7810767272010605688 R2_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
82: (67) r2 <<= 8
83: (71) r1 = *(u8 *)(r0 +0)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv7810767272010605688 R2_w=inv(id=0,umax_value=65280,var_off=(0x0; 0xff00)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
84: (4f) r2 |= r1
85: (71) r1 = *(u8 *)(r0 +3)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R2_w=inv(id=0) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
86: (67) r1 <<= 8
87: (71) r3 = *(u8 *)(r0 +2)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0,umax_value=65280,var_off=(0x0; 0xff00)) R2_w=inv(id=0) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
88: (4f) r1 |= r3
89: (67) r1 <<= 16
90: (4f) r1 |= r2
91: (71) r2 = *(u8 *)(r0 +5)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0) R2_w=inv(id=0) R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
92: (67) r2 <<= 8
93: (71) r3 = *(u8 *)(r0 +4)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0) R2_w=inv(id=0,umax_value=65280,var_off=(0x0; 0xff00)) R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
94: (4f) r2 |= r3
95: (71) r4 = *(u8 *)(r0 +6)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0) R2_w=inv(id=0) R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
96: (71) r3 = *(u8 *)(r0 +7)
 R0_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) R1_w=inv(id=0) R2_w=inv(id=0) R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R6=ctx(id=0,off=0,imm=0) R7=invP0 R8=pkt(id=0,off=0,r=34,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-16=??mmmmmm fp-24_w=inv7382371093486728563 fp-32_w=inv2317515584241167471 fp-40_w=inv7165073554179891300 fp-48_w=inv7310575179022491990 fp-56_w=inv2322222843807364197 fp-64_w=inv7810767272010605688
97: (67) r3 <<= 8
98: (4f) r3 |= r4
99: (67) r3 <<= 16
100: (4f) r3 |= r2
101: (67) r3 <<= 32
102: (4f) r3 |= r1
103: (bf) r1 = r10
104: (07) r1 += -64
105: (b7) r2 = 54
106: (85) call bpf_trace_printk#6
last_idx 106 first_idx 35
regs=4 stack=0 before 105: (b7) r2 = 54
107: (bf) r1 = r6
108: (18) r2 = 0xffff9dbf88277a00
110: (b7) r3 = 2
111: (85) call bpf_tail_call#12
kernel subsystem misconfigured verifier
processed 114 insns (limit 1000000) max_states_per_insn 0 total_states 5 peak_states 5 mark_read 3

goroutine 1 [running]:
main.LoadJmpMap({0x534518, 0xc00004c6b0}, {0x5348b8, 0xc0001400e0})
    /opt/src/xdploader/user/main.go:92 +0x5b3
main.main()
    /opt/src/xdploader/user/main.go:165 +0x2a6
vagrant@xdp-skeleton:/git/src$

It's unclear to me if it's a problem with the Jammy 64 kernel, goebpf, or something else.

mestery commented 2 years ago

BTW: The new jammy64 code is in the jammy64 branch.

mestery commented 2 years ago

@bersoare Have you tried using goebpf with Ubuntu Jammy yet? Have you seen a verifier error like above?

mestery commented 2 years ago

BTW: This is with the following kernel version:

vagrant@xdp-skeleton:/git/src$ uname -a
Linux xdp-skeleton 5.15.0-37-generic #39-Ubuntu SMP Wed Jun 1 19:16:45 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
vagrant@xdp-skeleton:/git/src$
bersoare commented 2 years ago

hey @mestery ,

i have tried the main branch, it seems to work just fine (even with the modified xdp code). when i changed to jammy, however, i m having some cross compile issues...btw, the kernel i have at hand is 5.4, so maybe it wont be good enough to try reproducing this.

are you able to load the ebpf prog from the container itself? i think you should be able to do so as long as the container is in privileged mode. also, in the container, you can try to peek at the compiled elf with llvm-objdump-$version -S xdp.elf, it should show you the code path along with the instruction #, it can help spotting where the faulty line is. example:

00000000000007c8 xdp_skeleton_2:
; {
     249:   bf 16 00 00 00 00 00 00 r6 = r1
     250:   b7 01 00 00 6e 0a 00 00 r1 = 2670
;     bpf_printk("xdp_skeleton_2: Entering function\n");
     251:   6b 1a f8 ff 00 00 00 00 *(u16 *)(r10 - 8) = r1
     252:   18 01 00 00 20 66 75 6e 00 00 00 00 63 74 69 6f r1 = 8028075781168391712 ll
     254:   7b 1a f0 ff 00 00 00 00 *(u64 *)(r10 - 16) = r1
     255:   18 01 00 00 45 6e 74 65 00 00 00 00 72 69 6e 67 r1 = 7453010373394067013 ll
     257:   7b 1a e8 ff 00 00 00 00 *(u64 *)(r10 - 24) = r1
     258:   18 01 00 00 65 74 6f 6e 00 00 00 00 5f 32 3a 20 r1 = 2322223943318991973 ll
     260:   7b 1a e0 ff 00 00 00 00 *(u64 *)(r10 - 32) = r1
     261:   18 01 00 00 78 64 70 5f 00 00 00 00 73 6b 65 6c r1 = 7810767272010605688 ll
     263:   7b 1a d8 ff 00 00 00 00 *(u64 *)(r10 - 40) = r1
     264:   b7 07 00 00 00 00 00 00 r7 = 0
     265:   73 7a fa ff 00 00 00 00 *(u8 *)(r10 - 6) = r7
     266:   bf a1 00 00 00 00 00 00 r1 = r10
     267:   07 01 00 00 d8 ff ff ff r1 += -40
     268:   b7 02 00 00 23 00 00 00 r2 = 35
     269:   85 00 00 00 06 00 00 00 call 6
     270:   b7 01 00 00 02 00 00 00 r1 = 2
     271:   63 1a d8 ff 00 00 00 00 *(u32 *)(r10 - 40) = r1
     272:   bf a2 00 00 00 00 00 00 r2 = r10
     273:   07 02 00 00 d8 ff ff ff r2 += -40
;     struct datarec *rec = bpf_map_lookup_elem(&stats_map, &action);
     274:   18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0 ll
     276:   85 00 00 00 01 00 00 00 call 1
;     if (!rec)
     277:   15 00 0c 00 00 00 00 00 if r0 == 0 goto +12 <LBB2_2>
;     rec->rx_packets++;
     278:   79 01 00 00 00 00 00 00 r1 = *(u64 *)(r0 + 0)
     279:   07 01 00 00 01 00 00 00 r1 += 1
     280:   7b 10 00 00 00 00 00 00 *(u64 *)(r0 + 0) = r1
;     rec->rx_bytes += (ctx->data_end - ctx->data);
     281:   61 61 00 00 00 00 00 00 r1 = *(u32 *)(r6 + 0)
     282:   61 62 04 00 00 00 00 00 r2 = *(u32 *)(r6 + 4)
     283:   1f 12 00 00 00 00 00 00 r2 -= r1
     284:   67 02 00 00 20 00 00 00 r2 <<= 32
     285:   77 02 00 00 20 00 00 00 r2 >>= 32
     286:   79 01 08 00 00 00 00 00 r1 = *(u64 *)(r0 + 8)
     287:   0f 21 00 00 00 00 00 00 r1 += r2
     288:   7b 10 08 00 00 00 00 00 *(u64 *)(r0 + 8) = r1
;     return action;
     289:   61 a7 d8 ff 00 00 00 00 r7 = *(u32 *)(r10 - 40)

as there were minor changes in the actual ebpf code, it sounds to me that it could be some linking issue with a newer kernel or even the distro.

also, the kernel subsystem misconfigured verifier message is a little bit too generic, but most of the hits i could found were related to not being able to load the map (that is, bpf_map == NULL) https://github.com/torvalds/linux/blob/master/kernel/bpf/verifier.c

i'd start checking if all the maps (the prog arrays, i believe) are created just fine and show up in filesystem (if pinned) and in the kernel (with bpftool map)

i'll try to find some time to solve the compile issues i found, and will let you know if i find something else