invalid access to map value, R3 min value is outside of the allowed memory range

[sbezverk]

Hello,

I am getting verifier failure despite of using suggested workarounds to make verifier happy. Here is the fragment of the code triggering verifier failure:

#define MAX_PACKET_LENGTH 1024

  // Use per CPU array map to be able to store 1k data buffer
   __u32 packet_data_map_id = 0;
    void *packet_data_buffer = NULL;
    packet_data_buffer = bpf_map_lookup_elem(&packet_data_map, &packet_data_map_id);
    if (!packet_data_buffer)
    {
        return;
    }
 // Calculate actual length of  L4+data portion 
   __u32 data_length = (__u32)(data_end - (data + sizeof(struct ethhdr) + sizeof(struct ipv6hdr)));
    // To make verifier happy Start block
    data_length &= 1023;
    int tmp = MAX_PACKET_LENGTH - data_length;
    if (tmp < 0)
    {
        return;
    }
    if (data_length <= 0)
    {
        return;
    }
    // To make verifier happy End block
    ret = bpf_skb_load_bytes(skb, sizeof(struct ethhdr) + sizeof(struct ipv6hdr), (void *)&packet_data_buffer[0], data_length);
    if (ret < 0)
    {
        return;
    }

Verifier error:

; ret = bpf_skb_load_bytes(skb, 0, &eth_copy, sizeof(eth_copy));
174: (bf) r1 = r6
175: (b4) w2 = 0
176: (b4) w4 = 14
177: (85) call bpf_skb_load_bytes#26
last_idx 177 first_idx 171
regs=10 stack=0 before 176: (b4) w4 = 14
; if (ret < 0)
178: (c6) if w0 s< 0x0 goto pc+56
 R0_w=inv(id=0,smax_value=9223372034707292159,umax_value=18446744071562067967,var_off=(0x0; 0xffffffff7fffffff),s32_min_value=0,u32_max_value=2147483647) R6=ctx(id=0,off=0,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8=map_value(id=0,off=0,ks=19,vs=256,imm=0) R9=pkt(id=0,off=54,r=54,imm=0) R10=fp0 fp-144=?????mmm fp-152=mmmmmmmm fp-160=mmmmmmmm fp-176=???????m fp-184=mmmmmmmm fp-192=mmmmmmmm fp-200=????mmmm
; __u32 packet_data_map_id = 0;
179: (b4) w1 = 0
180: (63) *(u32 *)(r10 -4) = r1
last_idx 180 first_idx 171
regs=2 stack=0 before 179: (b4) w1 = 0
181: (bf) r2 = r10
; 
182: (07) r2 += -4
; packet_data_buffer = bpf_map_lookup_elem(&packet_data_map, &packet_data_map_id);
183: (18) r1 = 0xffff9c5bf6bc4a00
185: (85) call bpf_map_lookup_elem#1
186: (7b) *(u64 *)(r10 -200) = r0
; if (!packet_data_buffer)
187: (15) if r0 == 0x0 goto pc+47
 R0=map_value(id=0,off=0,ks=4,vs=1024,imm=0) R6=ctx(id=0,off=0,imm=0) R7=pkt_end(id=0,off=0,imm=0) R8=map_value(id=0,off=0,ks=19,vs=256,imm=0) R9=pkt(id=0,off=54,r=54,imm=0) R10=fp0 fp-8=mmmm???? fp-144=?????mmm fp-152=mmmmmmmm fp-160=mmmmmmmm fp-176=???????m fp-184=mmmmmmmm fp-192=mmmmmmmm fp-200_w=map_value
; __u32 data_length = (__u32)(data_end - (data + sizeof(struct ethhdr) + sizeof(struct ipv6hdr)));
!
188: (1c) w7 -= w9               < ----  data_length is treated as a pointer
!
; data_length &= 1023;
189: (54) w7 &= 1023
; ret = bpf_skb_load_bytes(skb, sizeof(struct ethhdr) + sizeof(struct ipv6hdr), packet_data_buffer, data_length);
190: (bf) r1 = r6
191: (b4) w2 = 54
192: (79) r3 = *(u64 *)(r10 -200)
!
193: (bc) w4 = w7    < --- If I read it right then it should contain the u32 data length
!
194: (85) call bpf_skb_load_bytes#26
 R0=map_value(id=0,off=0,ks=4,vs=1024,imm=0) R1_w=ctx(id=0,off=0,imm=0) R2_w=inv54 R3_w=map_value(id=0,off=0,ks=4,vs=1024,imm=0) R4_w=inv(id=0,umax_value=1023,var_off=(0x0; 0x3ff)) R6=ctx(id=0,off=0,imm=0) R7_w=inv(id=0,umax_value=1023,var_off=(0x0; 0x3ff)) R8=map_value(id=0,off=0,ks=19,vs=256,imm=0) R9=pkt(id=0,off=54,r=54,imm=0) R10=fp0 fp-8=mmmm???? fp-144=?????mmm fp-152=mmmmmmmm fp-160=mmmmmmmm fp-176=???????m fp-184=mmmmmmmm fp-192=mmmmmmmm fp-200_w=map_value
invalid access to map value, value_size=1024 off=0 size=0
R3 min value is outside of the allowed memory range
processed 183 insns (limit 1000000) max_states_per_insn 0 total_states 12 peak_states 12 mark_read 10

When I use a constant instead of a data_length variable verifier passes:

; packet_data_buffer = bpf_map_lookup_elem(&packet_data_map, &packet_data_map_id);
183: (18) r1 = 0xffff9c59c3a34800
185: (85) call bpf_map_lookup_elem#1
186: (7b) *(u64 *)(r10 -200) = r0
; if (!packet_data_buffer)
187: (15) if r0 == 0x0 goto pc+50
 R0=map_value(id=0,off=0,ks=4,vs=1024,imm=0) R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=54,r=54,imm=0) R8=map_value(id=0,off=0,ks=19,vs=256,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-144=?????mmm fp-152=mmmmmmmm fp-160=mmmmmmmm fp-176=???????m fp-184=mmmmmmmm fp-192=mmmmmmmm fp-200_w=map_value
; ret = bpf_skb_load_bytes(skb, sizeof(struct ethhdr) + sizeof(struct ipv6hdr), packet_data_buffer, dl);
188: (bf) r1 = r6
189: (b4) w2 = 54
190: (79) r3 = *(u64 *)(r10 -200)
191: (b4) w4 = 64                                      < ----- parameter passed as length
192: (85) call bpf_skb_load_bytes#26
 R0=map_value(id=0,off=0,ks=4,vs=1024,imm=0) R1_w=ctx(id=0,off=0,imm=0) R2_w=inv54 R3_w=map_value(id=0,off=0,ks=4,vs=1024,imm=0) R4_w=inv64 R6=ctx(id=0,off=0,imm=0) R7=pkt(id=0,off=54,r=54,imm=0) R8=map_value(id=0,off=0,ks=19,vs=256,imm=0) R9=pkt_end(id=0,off=0,imm=0) R10=fp0 fp-8=mmmm???? fp-144=?????mmm fp-152=mmmmmmmm fp-160=mmmmmmmm fp-176=???????m fp-184=mmmmmmmm fp-192=mmmmmmmm fp-200_w=map_value
last_idx 192 first_idx 186
regs=10 stack=0 before 191: (b4) w4 = 64
; if (ret < 0)
193: (c6) if w0 s< 0x0 goto pc+44

In both working and non working cases R3 looks exactly the same: R3_w=map_value(id=0,off=0,ks=4,vs=1024,imm=0) the difference is how length parameter is viewed:

• non working case R4_w get value from R7=invP(id=0,umax_value=1023,var_off=(0x0; 0x3ff)) which make sense and
• working case R4_w=inv64

I would appreciate if somebody could suggest what else can be done to make verifier happy.

[sbezverk]

@yonghong-song Appreciate if you could check, I found some of your suggestion to work around similar issues, unfortunately in my case they do not work. Thanks a lot!

[yonghong-song]

Could you construct a self-contained example so I can take a look? Which kernel version did you use? I am asking since recent kernel improved alu32 tracking (32-bit register tracking like w7 above). For example, for

R4_w=inv(id=0,umax_value=1023,var_off=(0x0; 0x3ff))

I did not see update for {s, u}32_max/min_value, s{min, max}_value etc. Maybe some verifier improvements are needed.

[sbezverk]

@yonghong-song It is a bit hard to build self contained example,

here is my kernel version: Linux dev-ubuntu-1 5.10.10-051010-generic #202101231639 SMP Sat Jan 23 17:16:37 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

If you could think of a way to improve verifier, please let me know, I can easily test it.

[yonghong-song]

maybe you could post your bpf program? It may not work out of box, but people may be able to tweak your program to reproduce the issue?

[stgraham2000]

@sbezverk did you ever figure this one out? I'm having the same problem using Rust + Aya. Even out of the box examples are failing. I also find that if I make the length a constant then the problem goes away.