ebpf无法加载，报failed to load kprobe-based eBPF

[hezhiye]

x86_64 linux内核 6.1 nettrace-1.2.6-1.tl3.x86_64.tar.bz2 程序 内核配置都是好的 CONFIG_KPROBES=y CONFIG_KPROBES_ON_FTRACE=y CONFIG_HAVE_KPROBES=y CONFIG_HAVE_KPROBES_ON_FTRACE=y CONFIG_KPROBE_EVENTS=y CONFIG_FTRACE=y CONFIG_DYNAMIC_FTRACE=y CONFIG_BPF=y CONFIG_HAVE_EBPF_JIT=y CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y CONFIG_DEBUG_INFO_BTF=y

执行nettrace -p icmp --detail --diag --date 最后有如下错误： 198: (07) r1 += -40 ; frame1: R1_w=fp-40 ; l4_proto = _C(sk, sk_protocol); 199: (b7) r2 = 2 ; frame1: R2_w=P2 200: (85) call bpf_probe_read_kernel#113 ; frame1: R0=Pscalar() fp-40=mmmmmmmm ; l4_proto = _C(sk, sk_protocol); 201: (69) r1 = (u16 )(r10 -40) ; frame1: R1_w=Pscalar(umax=65535,var_off=(0x0; 0xffff)) R10=fp0 202: (05) goto pc+127 ; if (l4_proto == IPPROTO_IP) 330: (bf) r2 = r1 ; frame1: R1_w=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) R2_w=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) 331: (57) r2 &= 255 ; frame1: R2_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 332: (b7) r4 = 6 ; frame1: R4_w=P6 333: (79) r3 = (u64 )(r10 -104) ; frame1: R3_w=P2048 R10=fp0 ; if (l4_proto == IPPROTO_IP) 334: (15) if r2 == 0x0 goto pc+1 ; frame1: R2_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 335: (bf) r4 = r1 ; frame1: R1=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) R4=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) ; if (FILTER_CHECK(ctx, l4_proto, l4_proto)) 336: (71) r1 = (u8 )(r9 +88) ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) R9=map_value(off=0,ks=4,vs=152,imm=0) 337: (15) if r1 == 0x0 goto pc+4 ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 338: (bf) r1 = r4 ; frame1: R1_w=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) R4=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) 339: (57) r1 &= 255 ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 340: (71) r2 = (u8 )(r9 +87) ; frame1: R2_w=Pscalar(umax=255,var_off=(0x0; 0xff)) R9=map_value(off=0,ks=4,vs=152,imm=0) ; if (FILTER_CHECK(ctx, l4_proto, l4_proto)) 341: (5d) if r2 != r1 goto pc+253 ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) R2_w=Pscalar(umax=255,var_off=(0x0; 0xff)) ; switch (l4_proto) { 342: (bf) r1 = r4 ; frame1: R1_w=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) R4=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) 343: (57) r1 &= 255 ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 344: (7b) (u64 )(r10 -96) = r9 ; frame1: R9=map_value(off=0,ks=4,vs=152,imm=0) R10=fp0 fp-96_w=map_value 345: (7b) (u64 )(r10 -112) = r4 ; frame1: R4=Pscalar(id=7,umax=65535,var_off=(0x0; 0xffff)) R10=fp0 fp-112=P 346: (15) if r1 == 0x11 goto pc+24 ; frame1: R1=Pscalar(umax=255,var_off=(0x0; 0xff)) 347: (55) if r1 != 0x6 goto pc+48 ; frame1: R1=P6 348: failed to resolve CO-RE relocation [1515] struct tcp_sock.packets_out (0:69 @ offset 1804) processed 208 insns (limit 1000000) max_states_per_insn 0 total_states 18 peak_states 18 mark_read 10 -- END PROG LOAD LOG -- libbpf: prog '__trace_napi_gro_receive_entry': failed to load: -22 libbpf: failed to load object 'kprobe' libbpf: failed to load BPF skeleton 'kprobe': -22 ERROR: failed to load kprobe-based eBPF ERROR: failed to load kprobe-based bpf

[menglongdong]

看报错信息，是CORE解析tcp_sock结构体的时候报错的。很奇怪，这个结构体以及对应的字段应该是存在的，除非你的内核不支持TCP协议。

我这里做了一点适配，你可以看一下这个编译的版本能不能解决问题： nettrace.zip

[hezhiye]

; ske->state = _C(skc, skc_state); 448: (b7) r2 = 1 ; frame1: R2_w=P1 449: (85) call bpf_probe_read_kernel#113 ; frame1: R0=Pscalar() fp-40=mmmmmmmm ; ske->state = _C(skc, skc_state); 450: (71) r1 = (u8 )(r10 -40) ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) R10=fp0 ; ske->state = _C(skc, skc_state); 451: (73) (u8 )(r9 +76) = r1 ; frame1: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) R9=fp-152 fp-80=fp 452: failed to resolve CO-RE relocation [1520] struct inet_connection_sock.icsk_retransmits (0:18 @ offset 1361) processed 288 insns (limit 1000000) max_states_per_insn 0 total_states 23 peak_states 23 mark_read 18 -- END PROG LOAD LOG -- libbpf: prog '__trace_napi_gro_receive_entry': failed to load: -22 libbpf: failed to load object 'kprobe' libbpf: failed to load BPF skeleton 'kprobe': -22 ERROR: failed to load kprobe-based eBPF ERROR: failed to load kprobe-based bpf

[menglongdong]

再试一下这个版本的： nettrace.zip

[hezhiye]

740: (15) if r1 == 0xffff goto pc+2 ; frame2: R1=Pscalar(umax=65535,var_off=(0x0; 0xffff)) ; l4 = ctx->data + ctx->trans_header; 741: (79) r4 = (u64 )(r6 +0) ; frame2: R4_w=Pscalar() R6=fp-40 ; l4 = ctx->data + ctx->trans_header; 742: (0f) r4 += r1 ; frame2: R1=Pscalar(umax=65535,var_off=(0x0; 0xffff)) R4=Pscalar() 743: (7b) (u64 )(r10 -24) = r4 ; frame2: R4=Pscalar() R10=fp0 fp-24_w=mmmmmmmm 744: (7b) (u64 )(r10 -32) = r3 ; frame2: R3=fp-152 R10=fp0 fp-32_w=fp ; if (pkt->proto_l3 == ETH_P_IPV6) { 745: (69) r1 = (u16 )(r3 +56) ; frame2: R1_w=Pscalar(umax=65535,var_off=(0x0; 0xffff)) R3=fp-152 ; if (pkt->proto_l3 == ETH_P_IPV6) { 746: (55) if r1 != 0x86dd goto pc+79 ; frame2: R1_w=P34525 ; if (FILTER_ITER_ENABLED(ctx, addr)) 747: (71) r1 = (u8 )(r6 +38) ; frame2: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) R6=fp-40 748: (15) if r1 == 0x0 goto pc+7 ; frame2: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 749: (79) r1 = (u64 )(r6 +8) ; frame2: R1_w=map_value(off=0,ks=4,vs=168,imm=0) R6=fp-40 750: (71) r2 = (u8 )(r1 +20) ; frame2: R1=map_value(off=0,ks=4,vs=168,imm=0) R2=Pscalar(umax=255,var_off=(0x0; 0xff)) 751: (55) if r2 != 0x0 goto pc+402 ; frame2: R2=P0 752: (71) r2 = (u8 )(r1 +4) ; frame2: R1=map_value(off=0,ks=4,vs=168,imm=0) R2_w=Pscalar(umax=255,var_off=(0x0; 0xff)) 753: (55) if r2 != 0x0 goto pc+400 ; frame2: R2_w=P0 754: (71) r1 = (u8 )(r1 +12) ; frame2: R1_w=Pscalar(umax=255,var_off=(0x0; 0xff)) ; if (FILTER_ITER_ENABLED(ctx, addr)) 755: (55) if r1 != 0x0 goto pc+398 ; frame2: R1_w=P0 756: failed to resolve CO-RE relocation [1573] struct ipv6hdr.saddr (0:6 @ offset 8) processed 1093 insns (limit 1000000) max_states_per_insn 1 total_states 86 peak_states 86 mark_read 28 -- END PROG LOAD LOG -- libbpf: prog '__trace_napi_gro_receive_entry': failed to load: -22 libbpf: failed to load object 'kprobe' libbpf: failed to load BPF skeleton 'kprobe': -22 ERROR: failed to load kprobe-based eBPF ERROR: failed to load kprobe-based bpf

[hezhiye]

这个工具是不是与内核版本，内核配置有关啊，原先5.1 内核现在升级到了6.1

[menglongdong]

理论上内核版本越高，越受支持。从你的报错信息来看，感觉你是把网络编译成了内核模块？

请确认一下你的版本是否支持DEBUG_INFO_BTF_MODULES内核配置，支持的话是否开启了。

[hezhiye]

哪个网络模块？DEBUG_INFO_BTF_MODULES是没配置

[hezhiye]

但zcat /proc/config.gz | grep DEBUG_INFO_BTF_MODULES 却为空

[hezhiye]

重新加了DEBUG_INFO_BTF_MODULES，可以了

[hezhiye]

用你最后给我的程序

[menglongdong]

嗯，看样子是网络相关的一些一般会编译到内核里的功能被编译成了内核模块导致的。