search

loxilb-io / loxilb

eBPF based cloud-native load-balancer for Kubernetes|Edge|Telco|IoT|XaaS.
https://www.loxilb.io
Apache License 2.0
1.39k stars 120 forks source link

BPFireOS: cpumap libbpf: map 'cpu_map': failed to create: Argument list too long #661

Closed vincentmli closed 5 months ago

vincentmli commented 5 months ago

Describe the bug I am following https://loxilb-io.github.io/loxilbdocs/loxilbebpf/ to use ntc to manually load and attach program

ntc filter add dev red0 ingress bpf da obj /opt/loxilb/llb_ebpf_main.o sec tc_packet_hook0 verbos

verifier complains error below, is it because the map is not loaded?

BTF debug data section '.BTF' rejected: Invalid argument (22)!
 - Length:       73740
Verifier analysis:

magic: 0xeb9f
version: 1
flags: 0x0
hdr_len: 24
type_off: 0
type_len: 12812
str_off: 12812
str_len: 60904
btf_total_size: 73740
[1] PTR (anon) type_id=3
[2] INT int size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
[3] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=6
[4] INT __ARRAY_SIZE_TYPE__ size=4 bits_offset=0 nr_bits=32 encoding=(none)
[5] PTR (anon) type_id=2
[6] PTR (anon) type_id=7
[7] STRUCT xfi size=360 vlen=10
    fm type_id=8 bits_offset=0
    l2m type_id=13 bits_offset=128
    l34m type_id=20 bits_offset=320
    il2m type_id=13 bits_offset=704
    il34m type_id=20 bits_offset=896
    tm type_id=22 bits_offset=1280
    nm type_id=24 bits_offset=1472
    km type_id=25 bits_offset=2048
    qm type_id=27 bits_offset=2176
    pm type_id=28 bits_offset=2240
[8] STRUCT dp_fr_mdi size=16 vlen=3
    dat type_id=9 bits_offset=0
    dat_end type_id=9 bits_offset=32
    tstamp type_id=11 bits_offset=64
[9] TYPEDEF __u32 type_id=10
[10] INT unsigned int size=4 bits_offset=0 nr_bits=32 encoding=(none)
[11] TYPEDEF __u64 type_id=12
[12] INT unsigned long long size=8 bits_offset=0 nr_bits=64 encoding=(none)
[13] STRUCT dp_l2_mdi size=24 vlen=7
    vlan type_id=16 bits_offset=0
    dl_type type_id=14 bits_offset=48
    dl_dst type_id=19 bits_offset=64
    dl_src type_id=19 bits_offset=112
    vlan_pcp type_id=17 bits_offset=160
    valid type_id=17 bits_offset=168
    ssnid type_id=14 bits_offset=176
[14] TYPEDEF __u16 type_id=15
[15] INT unsigned short size=2 bits_offset=0 nr_bits=16 encoding=(none)
[16] ARRAY (anon) type_id=14 index_type_id=4 nr_elems=3
[17] TYPEDEF __u8 type_id=18
[18] INT unsigned char size=1 bits_offset=0 nr_bits=8 encoding=(none)
[19] ARRAY (anon) type_id=17 index_type_id=4 nr_elems=6
[20] STRUCT dp_l34_mdi size=48 vlen=10
    tos type_id=17 bits_offset=0
    nw_proto type_id=17 bits_offset=8
    valid type_id=17 bits_offset=16
    frg type_id=17 bits_offset=24
    source type_id=14 bits_offset=32
    dest type_id=14 bits_offset=48
    seq type_id=9 bits_offset=64
    ack type_id=9 bits_offset=96
    saddr type_id=21 bits_offset=128
    daddr type_id=21 bits_offset=256
[21] ARRAY (anon) type_id=9 index_type_id=4 nr_elems=4
[22] STRUCT dp_tun_mdi size=24 vlen=7
    new_tunnel_id type_id=9 bits_offset=0
    tun_encap type_id=14 bits_offset=32
    tun_decap type_id=14 bits_offset=48
    tunnel_id type_id=23 bits_offset=64
    tun_type type_id=9 bits_offset=96
    tun_rip type_id=23 bits_offset=128
    tun_sip type_id=23 bits_offset=160
[23] TYPEDEF __le32 type_id=9
[24] STRUCT dp_nat_mdi size=72 vlen=12
    nxip type_id=21 bits_offset=0
    nrip type_id=21 bits_offset=128
    pmhh type_id=21 bits_offset=256
    npmhh type_id=11 bits_offset=384
    nxport type_id=14 bits_offset=448
    ct_sts type_id=17 bits_offset=464
    sel_aid type_id=17 bits_offset=472
    nv6 type_id=17 bits_offset=480
    xlate_proto type_id=17 bits_offset=488
    dsr type_id=17 bits_offset=496
    cdis type_id=17 bits_offset=504
    ito type_id=11 bits_offset=512
[25] STRUCT dp_key_mdi size=16 vlen=1
    skey type_id=26 bits_offset=0
[26] ARRAY (anon) type_id=17 index_type_id=4 nr_elems=16
[27] STRUCT dp_qos_mdi size=8 vlen=6
    tc type_id=17 bits_offset=0
    icol type_id=17 bits_offset=8
    ocol type_id=17 bits_offset=16
    qfi type_id=17 bits_offset=24
    ipolid type_id=14 bits_offset=32
    opolid type_id=14 bits_offset=48
[28] STRUCT dp_pi_mdi size=80 vlen=43
    bd type_id=14 bitfield_size=0 bits_offset=0
    py_bytes type_id=14 bitfield_size=0 bits_offset=16
    pipe_act type_id=17 bitfield_size=0 bits_offset=32
    l3_off type_id=17 bitfield_size=0 bits_offset=40
    phit type_id=14 bitfield_size=0 bits_offset=48
    nh_num type_id=14 bitfield_size=0 bits_offset=64
    qos_id type_id=14 bitfield_size=0 bits_offset=80
    rcode type_id=9 bitfield_size=0 bits_offset=96
    tc type_id=17 bitfield_size=0 bits_offset=128
    pprop type_id=17 bitfield_size=0 bits_offset=136
    lkup_dmac type_id=19 bitfield_size=0 bits_offset=144
    iport type_id=14 bitfield_size=0 bits_offset=192
    oport type_id=14 bitfield_size=0 bits_offset=208
    zone type_id=14 bitfield_size=0 bits_offset=224
    l4_off type_id=17 bitfield_size=0 bits_offset=240
    table_id type_id=17 bitfield_size=0 bits_offset=248
    sseid type_id=11 bitfield_size=0 bits_offset=256
    dseid type_id=11 bitfield_size=0 bits_offset=320
    mirr type_id=14 bitfield_size=0 bits_offset=384
    tcp_flags type_id=17 bitfield_size=0 bits_offset=400
    nf type_id=17 bitfield_size=0 bits_offset=408
    rule_id type_id=14 bitfield_size=0 bits_offset=416
    l3_adj type_id=29 bitfield_size=0 bits_offset=432
    il3_off type_id=17 bitfield_size=0 bits_offset=448
    il4_off type_id=17 bitfield_size=0 bits_offset=456
    itcp_flags type_id=17 bitfield_size=0 bits_offset=464
    l4fin type_id=17 bitfield_size=1 bits_offset=472
    dbg type_id=17 bitfield_size=1 bits_offset=473
    goct type_id=17 bitfield_size=1 bits_offset=474
    nfc type_id=17 bitfield_size=1 bits_offset=475
    pten type_id=17 bitfield_size=2 bits_offset=476
    il4fin type_id=17 bitfield_size=1 bits_offset=478
    dir type_id=17 bitfield_size=1 bits_offset=479
    l3_len type_id=14 bitfield_size=0 bits_offset=480
    l3_plen type_id=14 bitfield_size=0 bits_offset=496
    il3_len type_id=14 bitfield_size=0 bits_offset=512
    il3_plen type_id=14 bitfield_size=0 bits_offset=528
    dp_mark type_id=14 bitfield_size=0 bits_offset=544
    dp_rec type_id=14 bitfield_size=0 bits_offset=560
    tun_off type_id=14 bitfield_size=0 bits_offset=576
    fw_mid type_id=14 bitfield_size=0 bits_offset=592
    fw_lid type_id=14 bitfield_size=0 bits_offset=608
    fw_rid type_id=14 bitfield_size=0 bits_offset=624
[29] TYPEDEF __s16 type_id=30
[30] INT short size=2 bits_offset=0 nr_bits=16 encoding=SIGNED
[31] PTR (anon) type_id=32
[32] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=1
[33] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=5 bits_offset=64
    value type_id=6 bits_offset=128
    max_entries type_id=31 bits_offset=192
[34] VAR xfis type_id=33 linkage=1
[35] PTR (anon) type_id=36
[36] STRUCT intf_key size=8 vlen=3
    ifindex type_id=9 bits_offset=0
    ing_vid type_id=14 bits_offset=32
    pad type_id=14 bits_offset=48
[37] PTR (anon) type_id=38
[38] STRUCT dp_intf_tact size=32 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=40 bits_offset=128
[39] STRUCT dp_cmn_act size=16 vlen=7
    act_type type_id=17 bits_offset=0
    ftrap type_id=17 bits_offset=8
    oaux type_id=14 bits_offset=16
    cidx type_id=9 bits_offset=32
    fwrid type_id=9 bits_offset=64
    mark type_id=14 bits_offset=96
    record type_id=14 bits_offset=112
[40] UNION (anon) size=16 vlen=1
    set_ifi type_id=41 bits_offset=0
[41] STRUCT dp_intf_tact_set_ifi size=16 vlen=8
    xdp_ifidx type_id=14 bits_offset=0
    zone type_id=14 bits_offset=16
    bd type_id=14 bits_offset=32
    mirr type_id=14 bits_offset=48
    polid type_id=14 bits_offset=64
    pprop type_id=17 bits_offset=80
    pten type_id=17 bits_offset=88
    r type_id=42 bits_offset=96
[42] ARRAY (anon) type_id=17 index_type_id=4 nr_elems=4
[43] PTR (anon) type_id=44
[44] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=512
[45] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=35 bits_offset=64
    value type_id=37 bits_offset=128
    max_entries type_id=43 bits_offset=192
[46] VAR intf_map type_id=45 linkage=1
[47] PTR (anon) type_id=9
[48] PTR (anon) type_id=49
[49] STRUCT dp_pb_stats size=16 vlen=2
    bytes type_id=50 bits_offset=0
    packets type_id=50 bits_offset=64
[50] TYPEDEF uint64_t type_id=51
[51] TYPEDEF __uint64_t type_id=12
[52] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=43 bits_offset=192
[53] VAR intf_stats_map type_id=52 linkage=1
[54] PTR (anon) type_id=55
[55] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=6144
[56] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=54 bits_offset=192
[57] VAR bd_stats_map type_id=56 linkage=1
[58] PTR (anon) type_id=59
[59] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=4
[60] PTR (anon) type_id=61
[61] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=128
[62] STRUCT (anon) size=32 vlen=4
    type type_id=58 bits_offset=0
    key type_id=5 bits_offset=64
    value type_id=47 bits_offset=128
    max_entries type_id=60 bits_offset=192
[63] VAR pkt_ring type_id=62 linkage=1
[64] STRUCT (anon) size=32 vlen=4
    type type_id=58 bits_offset=0
    key type_id=5 bits_offset=64
    value type_id=47 bits_offset=128
    max_entries type_id=60 bits_offset=192
[65] VAR cp_ring type_id=64 linkage=1
[66] PTR (anon) type_id=67
[67] STRUCT ll_dp_pmdi size=24 vlen=9
    ifindex type_id=9 bits_offset=0
    dp_inport type_id=14 bits_offset=32
    dp_oport type_id=14 bits_offset=48
    rcode type_id=9 bits_offset=64
    table_id type_id=14 bits_offset=96
    phit type_id=14 bits_offset=112
    pkt_len type_id=9 bits_offset=128
    resolve_ip type_id=9 bits_offset=160
    data type_id=70 bits_offset=192
[68] TYPEDEF uint8_t type_id=69
[69] TYPEDEF __uint8_t type_id=18
[70] ARRAY (anon) type_id=68 index_type_id=4 nr_elems=0
[71] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=66 bits_offset=128
    max_entries type_id=31 bits_offset=192
[72] VAR pkts type_id=71 linkage=1
[73] PTR (anon) type_id=74
[74] STRUCT dp_fc_tacts size=480 vlen=6
    ca type_id=39 bits_offset=0
    its type_id=11 bits_offset=128
    zone type_id=9 bits_offset=192
    pad type_id=14 bits_offset=224
    pten type_id=14 bits_offset=240
    fcta type_id=84 bits_offset=256
[75] STRUCT dp_fc_tact size=64 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=76 bits_offset=128
[76] UNION (anon) size=48 vlen=6
    port_act type_id=77 bits_offset=0
    nh_act type_id=78 bits_offset=0
    nat_act type_id=80 bits_offset=0
    nl2 type_id=79 bits_offset=0
    ntun type_id=81 bits_offset=0
    l2ov type_id=83 bits_offset=0
[77] STRUCT dp_rdr_act size=4 vlen=2
    oport type_id=14 bits_offset=0
    fr type_id=14 bits_offset=16
[78] STRUCT dp_rt_nh_act size=24 vlen=4
    nh_num type_id=14 bits_offset=0
    bd type_id=14 bits_offset=16
    tid type_id=9 bits_offset=32
    l2nh type_id=79 bits_offset=64
[79] STRUCT dp_rt_l2nh_act size=16 vlen=4
    dmac type_id=19 bits_offset=0
    smac type_id=19 bits_offset=48
    bd type_id=14 bits_offset=96
    rnh_num type_id=14 bits_offset=112
[80] STRUCT dp_nat_act size=48 vlen=11
    xip type_id=21 bits_offset=0
    rip type_id=21 bits_offset=128
    xport type_id=14 bits_offset=256
    fr type_id=17 bits_offset=272
    doct type_id=17 bits_offset=280
    rid type_id=9 bits_offset=288
    aid type_id=9 bits_offset=320
    nv6 type_id=17 bits_offset=352
    dsr type_id=17 bits_offset=360
    cdis type_id=17 bits_offset=368
    nmh type_id=17 bits_offset=376
[81] STRUCT dp_rt_tunnh_act size=32 vlen=2
    l3t type_id=82 bits_offset=0
    l2nh type_id=79 bits_offset=128
[82] STRUCT dp_rt_l3tun_act size=16 vlen=4
    rip type_id=9 bits_offset=0
    sip type_id=9 bits_offset=32
    tid type_id=9 bits_offset=64
    aux type_id=9 bits_offset=96
[83] STRUCT dp_l2vlan_act size=4 vlen=2
    vlan type_id=14 bits_offset=0
    oport type_id=14 bits_offset=16
[84] ARRAY (anon) type_id=75 index_type_id=4 nr_elems=7
[85] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=73 bits_offset=128
    max_entries type_id=31 bits_offset=192
[86] VAR fcas type_id=85 linkage=1
[87] PTR (anon) type_id=88
[88] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=14
[89] STRUCT (anon) size=32 vlen=4
    type type_id=87 bits_offset=0
    key type_id=5 bits_offset=64
    value type_id=5 bits_offset=128
    max_entries type_id=43 bits_offset=192
[90] VAR tx_intf_map type_id=89 linkage=1
[91] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=54 bits_offset=192
[92] VAR tx_intf_stats_map type_id=91 linkage=1
[93] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=54 bits_offset=192
[94] VAR tx_bd_stats_map type_id=93 linkage=1
[95] PTR (anon) type_id=96
[96] STRUCT dp_smac_key size=8 vlen=2
    smac type_id=19 bits_offset=0
    bd type_id=14 bits_offset=48
[97] PTR (anon) type_id=98
[98] STRUCT dp_smac_tact size=16 vlen=1
    ca type_id=39 bits_offset=0
[99] PTR (anon) type_id=100
[100] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=8192
[101] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=95 bits_offset=64
    value type_id=97 bits_offset=128
    max_entries type_id=99 bits_offset=192
[102] VAR smac_map type_id=101 linkage=1
[103] PTR (anon) type_id=104
[104] STRUCT dp_dmac_key size=8 vlen=2
    dmac type_id=19 bits_offset=0
    bd type_id=14 bits_offset=48
[105] PTR (anon) type_id=106
[106] STRUCT dp_dmac_tact size=20 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=107 bits_offset=128
[107] UNION (anon) size=4 vlen=2
    vlan_act type_id=83 bits_offset=0
    port_act type_id=77 bits_offset=0
[108] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=103 bits_offset=64
    value type_id=105 bits_offset=128
    max_entries type_id=99 bits_offset=192
[109] VAR dmac_map type_id=108 linkage=1
[110] PTR (anon) type_id=111
[111] STRUCT dp_tmac_key size=12 vlen=4
    mac type_id=19 bits_offset=0
    tun_type type_id=17 bits_offset=48
    pad type_id=17 bits_offset=56
    tunnel_id type_id=9 bits_offset=64
[112] PTR (anon) type_id=113
[113] STRUCT dp_tmac_tact size=40 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=114 bits_offset=128
[114] UNION (anon) size=24 vlen=1
    rt_nh type_id=78 bits_offset=0
[115] PTR (anon) type_id=116
[116] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=2048
[117] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=110 bits_offset=64
    value type_id=112 bits_offset=128
    max_entries type_id=115 bits_offset=192
[118] VAR tmac_map type_id=117 linkage=1
[119] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=115 bits_offset=192
[120] VAR tmac_stats_map type_id=119 linkage=1
[121] PTR (anon) type_id=122
[122] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=2
[123] PTR (anon) type_id=124
[124] STRUCT dp_nh_key size=4 vlen=1
    nh_num type_id=9 bits_offset=0
[125] PTR (anon) type_id=126
[126] STRUCT dp_nh_tact size=48 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=127 bits_offset=128
[127] UNION (anon) size=32 vlen=2
    rt_l2nh type_id=79 bits_offset=0
    rt_tnh type_id=81 bits_offset=0
[128] PTR (anon) type_id=129
[129] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=4096
[130] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=123 bits_offset=64
    value type_id=125 bits_offset=128
    max_entries type_id=128 bits_offset=192
[131] VAR nh_map type_id=130 linkage=1
[132] PTR (anon) type_id=133
[133] STRUCT dp_ct_key size=40 vlen=7
    daddr type_id=21 bits_offset=0
    saddr type_id=21 bits_offset=128
    sport type_id=14 bits_offset=256
    dport type_id=14 bits_offset=272
    zone type_id=14 bits_offset=288
    l4proto type_id=17 bits_offset=304
    v6 type_id=17 bits_offset=312
[134] PTR (anon) type_id=135
[135] STRUCT dp_ct_tact size=264 vlen=6
    ca type_id=39 bits_offset=0
    lock type_id=136 bits_offset=128
    ctd type_id=137 bits_offset=192
    ito type_id=11 bits_offset=1600
    lts type_id=11 bits_offset=1664
    (anon) type_id=177 bits_offset=1728
[136] STRUCT bpf_spin_lock size=4 vlen=1
    val type_id=9 bits_offset=0
[137] STRUCT dp_ct_dat size=176 vlen=8
    rid type_id=14 bits_offset=0
    aid type_id=14 bits_offset=16
    nid type_id=9 bits_offset=32
    pi type_id=138 bits_offset=64
    dir type_id=145 bits_offset=864
    smr type_id=171 bits_offset=896
    xi type_id=173 bits_offset=928
    pb type_id=176 bits_offset=1280
[138] TYPEDEF ct_pinf_t type_id=139
[139] STRUCT (anon) size=100 vlen=5
    (anon) type_id=140 bits_offset=0
    frag type_id=14 bits_offset=608
    npmhh type_id=14 bits_offset=624
    pmhh type_id=21 bits_offset=640
    l3i type_id=167 bits_offset=768
[140] UNION (anon) size=76 vlen=4
    t type_id=141 bits_offset=0
    u type_id=151 bits_offset=0
    i type_id=153 bits_offset=0
    s type_id=157 bits_offset=0
[141] TYPEDEF ct_tcp_pinf_t type_id=142
[142] STRUCT (anon) size=40 vlen=3
    state type_id=143 bits_offset=0
    fndir type_id=145 bits_offset=32
    tcp_cts type_id=150 bits_offset=64
[143] TYPEDEF ct_tcp_state_t type_id=144
[144] ENUM (anon) size=4 vlen=9
    CT_TCP_CLOSED val=0
    CT_TCP_SS val=1
    CT_TCP_SA val=2
    CT_TCP_EST val=4
    CT_TCP_FINI val=16
    CT_TCP_FINI2 val=32
    CT_TCP_FINI3 val=64
    CT_TCP_CW val=128
    CT_TCP_ERR val=256
[145] TYPEDEF ct_dir_t type_id=146
[146] ENUM (anon) size=4 vlen=3
    CT_DIR_IN val=0
    CT_DIR_OUT val=1
    CT_DIR_MAX val=2
[147] TYPEDEF ct_tcp_pinfd_t type_id=148
[148] STRUCT (anon) size=16 vlen=5
    hstate type_id=14 bits_offset=0
    init_acks type_id=14 bits_offset=16
    seq type_id=9 bits_offset=32
    pack type_id=149 bits_offset=64
    pseq type_id=149 bits_offset=96
[149] TYPEDEF __be32 type_id=9
[150] ARRAY (anon) type_id=147 index_type_id=4 nr_elems=2
[151] TYPEDEF ct_udp_pinf_t type_id=152
[152] STRUCT (anon) size=12 vlen=4
    state type_id=14 bits_offset=0
    pkts_seen type_id=14 bits_offset=16
    rpkts_seen type_id=14 bits_offset=32
    fndir type_id=145 bits_offset=64
[153] TYPEDEF ct_icmp_pinf_t type_id=154
[154] STRUCT (anon) size=4 vlen=3
    state type_id=68 bits_offset=0
    errs type_id=68 bits_offset=8
    lseq type_id=155 bits_offset=16
[155] TYPEDEF uint16_t type_id=156
[156] TYPEDEF __uint16_t type_id=15
[157] TYPEDEF ct_sctp_pinf_t type_id=158
[158] STRUCT (anon) size=76 vlen=6
    state type_id=159 bits_offset=0
    fndir type_id=145 bits_offset=32
    itag type_id=161 bits_offset=64
    otag type_id=161 bits_offset=96
    cookie type_id=161 bits_offset=128
    sctp_cts type_id=166 bits_offset=160
[159] TYPEDEF ct_sctp_state_t type_id=160
[160] ENUM (anon) size=4 vlen=12
    CT_SCTP_CLOSED val=0
    CT_SCTP_INIT val=1
    CT_SCTP_INITA val=2
    CT_SCTP_COOKIE val=4
    CT_SCTP_COOKIEA val=16
    CT_SCTP_PRE_EST val=32
    CT_SCTP_EST val=64
    CT_SCTP_SHUT val=128
    CT_SCTP_SHUTA val=256
    CT_SCTP_SHUTC val=512
    CT_SCTP_ERR val=1024
    CT_SCTP_ABRT val=2048
[161] TYPEDEF uint32_t type_id=162
[162] TYPEDEF __uint32_t type_id=10
[163] TYPEDEF ct_sctp_pinfd_t type_id=164
[164] STRUCT (anon) size=28 vlen=4
    nh type_id=9 bits_offset=0
    odst type_id=9 bits_offset=32
    osrc type_id=9 bits_offset=64
    mh_host type_id=165 bits_offset=96
[165] ARRAY (anon) type_id=149 index_type_id=4 nr_elems=4
[166] ARRAY (anon) type_id=163 index_type_id=4 nr_elems=2
[167] TYPEDEF ct_l3inf_t type_id=168
[168] STRUCT (anon) size=4 vlen=1
    state type_id=169 bits_offset=0
[169] TYPEDEF ct_state_t type_id=170
[170] ENUM (anon) size=4 vlen=6
    CT_STATE_NONE val=0
    CT_STATE_REQ val=1
    CT_STATE_REP val=2
    CT_STATE_EST val=4
    CT_STATE_FIN val=8
    CT_STATE_DOR val=16
[171] TYPEDEF ct_smr_t type_id=172
[172] ENUM (anon) size=4 vlen=8
    CT_SMR_ERR val=-1
    CT_SMR_INPROG val=0
    CT_SMR_EST val=1
    CT_SMR_UEST val=2
    CT_SMR_FIN val=3
    CT_SMR_CTD val=4
    CT_SMR_UNT val=100
    CT_SMR_INIT val=200
[173] TYPEDEF nxfrm_inf_t type_id=174
[174] STRUCT mf_xfrm_inf size=44 vlen=12
    nat_flags type_id=68 bitfield_size=0 bits_offset=0
    inactive type_id=68 bitfield_size=0 bits_offset=8
    wprio type_id=68 bitfield_size=0 bits_offset=16
    nv6 type_id=68 bitfield_size=0 bits_offset=24
    dsr type_id=68 bitfield_size=0 bits_offset=32
    mhon type_id=68 bitfield_size=4 bits_offset=40
    mhs type_id=68 bitfield_size=4 bits_offset=44
    nat_xport type_id=155 bitfield_size=0 bits_offset=48
    nat_xip type_id=175 bitfield_size=0 bits_offset=64
    nat_rip type_id=175 bitfield_size=0 bits_offset=192
    osp type_id=155 bitfield_size=0 bits_offset=320
    odp type_id=155 bitfield_size=0 bits_offset=336
[175] ARRAY (anon) type_id=161 index_type_id=4 nr_elems=4
[176] TYPEDEF dp_pb_stats_t type_id=49
[177] UNION (anon) size=48 vlen=4
    port_act type_id=77 bits_offset=0
    pdr_sess_act type_id=178 bits_offset=0
    rt_nh type_id=78 bits_offset=0
    nat_act type_id=80 bits_offset=0
[178] STRUCT dp_sess_act size=4 vlen=1
    sess_id type_id=9 bits_offset=0
[179] PTR (anon) type_id=180
[180] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=524288
[181] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=132 bits_offset=64
    value type_id=134 bits_offset=128
    max_entries type_id=179 bits_offset=192
[182] VAR ct_map type_id=181 linkage=1
[183] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=179 bits_offset=192
[184] VAR ct_stats_map type_id=183 linkage=1
[185] PTR (anon) type_id=186
[186] STRUCT dp_nat_key size=24 vlen=6
    daddr type_id=21 bits_offset=0
    dport type_id=14 bits_offset=128
    zone type_id=14 bits_offset=144
    mark type_id=14 bits_offset=160
    l4proto type_id=17 bits_offset=176
    v6 type_id=17 bits_offset=184
[187] PTR (anon) type_id=188
[188] STRUCT dp_nat_tacts size=776 vlen=13
    ca type_id=39 bits_offset=0
    ito type_id=50 bits_offset=128
    pto type_id=50 bits_offset=192
    lock type_id=136 bits_offset=256
    nxfrm type_id=155 bits_offset=288
    cdis type_id=68 bits_offset=304
    npmhh type_id=68 bits_offset=312
    sel_hint type_id=155 bits_offset=320
    sel_type type_id=155 bits_offset=336
    pmhh type_id=189 bits_offset=352
    nxfrms type_id=190 bits_offset=448
    lts type_id=50 bits_offset=6080
    base_to type_id=50 bits_offset=6144
[189] ARRAY (anon) type_id=161 index_type_id=4 nr_elems=3
[190] ARRAY (anon) type_id=174 index_type_id=4 nr_elems=16
[191] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=185 bits_offset=64
    value type_id=187 bits_offset=128
    max_entries type_id=128 bits_offset=192
[192] VAR nat_map type_id=191 linkage=1
[193] PTR (anon) type_id=194
[194] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=65536
[195] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=193 bits_offset=192
[196] VAR nat_stats_map type_id=195 linkage=1
[197] PTR (anon) type_id=198
[198] STRUCT dp_nat_epacts size=84 vlen=3
    ca type_id=39 bits_offset=0
    lock type_id=136 bits_offset=128
    active_sess type_id=199 bits_offset=160
[199] ARRAY (anon) type_id=161 index_type_id=4 nr_elems=16
[200] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=197 bits_offset=128
    max_entries type_id=128 bits_offset=192
[201] VAR nat_ep_map type_id=200 linkage=1
[202] PTR (anon) type_id=203
[203] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=11
[204] PTR (anon) type_id=205
[205] STRUCT dp_rtv4_key size=12 vlen=2
    l type_id=206 bits_offset=0
    (anon) type_id=208 bits_offset=32
[206] STRUCT bpf_lpm_trie_key size=4 vlen=2
    prefixlen type_id=9 bits_offset=0
    data type_id=207 bits_offset=32
[207] ARRAY (anon) type_id=17 index_type_id=4 nr_elems=0
[208] UNION (anon) size=8 vlen=2
    v4k type_id=19 bits_offset=0
    addr type_id=9 bits_offset=0
[209] PTR (anon) type_id=210
[210] STRUCT dp_rt_tact size=40 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=211 bits_offset=128
[211] UNION (anon) size=24 vlen=2
    port_act type_id=77 bits_offset=0
    rt_nh type_id=78 bits_offset=0
[212] PTR (anon) type_id=213
[213] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=32768
[214] STRUCT (anon) size=40 vlen=5
    type type_id=202 bits_offset=0
    key type_id=204 bits_offset=64
    value type_id=209 bits_offset=128
    map_flags type_id=31 bits_offset=192
    max_entries type_id=212 bits_offset=256
[215] VAR rt_v4_map type_id=214 linkage=1
[216] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=212 bits_offset=192
[217] VAR rt_v4_stats_map type_id=216 linkage=1
[218] PTR (anon) type_id=219
[219] STRUCT dp_rtv6_key size=20 vlen=2
    l type_id=206 bits_offset=0
    (anon) type_id=220 bits_offset=32
[220] UNION (anon) size=16 vlen=1
    addr type_id=21 bits_offset=0
[221] STRUCT (anon) size=40 vlen=5
    type type_id=202 bits_offset=0
    key type_id=218 bits_offset=64
    value type_id=209 bits_offset=128
    map_flags type_id=31 bits_offset=192
    max_entries type_id=115 bits_offset=256
[222] VAR rt_v6_map type_id=221 linkage=1
[223] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=115 bits_offset=192
[224] VAR rt_v6_stats_map type_id=223 linkage=1
[225] PTR (anon) type_id=226
[226] STRUCT dp_mirr_tact size=48 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=227 bits_offset=128
[227] UNION (anon) size=32 vlen=3
    rt_tnh type_id=81 bits_offset=0
    vlan_act type_id=83 bits_offset=0
    port_act type_id=77 bits_offset=0
[228] PTR (anon) type_id=229
[229] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=32
[230] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=225 bits_offset=128
    max_entries type_id=228 bits_offset=192
[231] VAR mirr_map type_id=230 linkage=1
[232] PTR (anon) type_id=233
[233] STRUCT dp_sess4_key size=16 vlen=4
    daddr type_id=9 bits_offset=0
    saddr type_id=9 bits_offset=32
    teid type_id=9 bits_offset=64
    r type_id=9 bits_offset=96
[234] PTR (anon) type_id=235
[235] STRUCT dp_sess_tact size=32 vlen=7
    ca type_id=39 bits_offset=0
    qfi type_id=68 bits_offset=128
    r1 type_id=68 bits_offset=136
    r2 type_id=155 bits_offset=144
    rip type_id=161 bits_offset=160
    sip type_id=161 bits_offset=192
    teid type_id=161 bits_offset=224
[236] PTR (anon) type_id=237
[237] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=20480
[238] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=232 bits_offset=64
    value type_id=234 bits_offset=128
    max_entries type_id=236 bits_offset=192
[239] VAR sess_v4_map type_id=238 linkage=1
[240] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=236 bits_offset=192
[241] VAR sess_v4_stats_map type_id=240 linkage=1
[242] PTR (anon) type_id=243
[243] STRUCT dp_fcv4_key size=16 vlen=7
    daddr type_id=9 bits_offset=0
    saddr type_id=9 bits_offset=32
    sport type_id=14 bits_offset=64
    dport type_id=14 bits_offset=80
    l4proto type_id=17 bits_offset=96
    pad type_id=17 bits_offset=104
    in_port type_id=14 bits_offset=112
[244] STRUCT (anon) size=32 vlen=4
    type type_id=31 bits_offset=0
    key type_id=242 bits_offset=64
    value type_id=73 bits_offset=128
    max_entries type_id=179 bits_offset=192
[245] VAR fc_v4_map type_id=244 linkage=1
[246] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=179 bits_offset=192
[247] VAR fc_v4_stats_map type_id=246 linkage=1
[248] PTR (anon) type_id=249
[249] STRUCT dp_fwv4_ent size=112 vlen=2
    k type_id=250 bits_offset=0
    fwa type_id=262 bits_offset=384
[250] STRUCT pdi_key size=48 vlen=9
    dest type_id=251 bits_offset=0
    source type_id=251 bits_offset=64
    dport type_id=253 bits_offset=128
    sport type_id=253 bits_offset=192
    inport type_id=258 bits_offset=256
    zone type_id=258 bits_offset=288
    bd type_id=258 bits_offset=320
    protocol type_id=260 bits_offset=352
    nr type_id=260 bits_offset=368
[251] TYPEDEF pdi_tup32_t type_id=252
[252] STRUCT pdi_tup32_ size=8 vlen=2
    val type_id=9 bits_offset=0
    valid type_id=9 bits_offset=32
[253] TYPEDEF pdi_tupr16_t type_id=254
[254] STRUCT pdi_tupwr16_ size=8 vlen=2
    has_range type_id=161 bits_offset=0
    u type_id=255 bits_offset=32
[255] UNION (anon) size=4 vlen=2
    r type_id=256 bits_offset=0
    v type_id=257 bits_offset=0
[256] STRUCT pdi_tup16r_ size=4 vlen=2
    min type_id=14 bits_offset=0
    max type_id=14 bits_offset=16
[257] STRUCT pdi_tup16v_ size=4 vlen=2
    val type_id=14 bits_offset=0
    valid type_id=14 bits_offset=16
[258] TYPEDEF pdi_tup16_t type_id=259
[259] STRUCT pdi_tup16_ size=4 vlen=2
    val type_id=14 bits_offset=0
    valid type_id=14 bits_offset=16
[260] TYPEDEF pdi_tup8_t type_id=261
[261] STRUCT pdi_tup8_ size=2 vlen=2
    val type_id=17 bits_offset=0
    valid type_id=17 bits_offset=8
[262] STRUCT dp_fwv4_tact size=64 vlen=2
    ca type_id=39 bits_offset=0
    (anon) type_id=263 bits_offset=128
[263] UNION (anon) size=48 vlen=2
    port_act type_id=77 bits_offset=0
    nat_act type_id=80 bits_offset=0
[264] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=248 bits_offset=128
    max_entries type_id=99 bits_offset=192
[265] VAR fw_v4_map type_id=264 linkage=1
[266] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=99 bits_offset=192
[267] VAR fw_v4_stats_map type_id=266 linkage=1
[268] PTR (anon) type_id=269
[269] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=3
[270] PTR (anon) type_id=271
[271] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=8
[272] STRUCT (anon) size=32 vlen=4
    type type_id=268 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=47 bits_offset=128
    max_entries type_id=270 bits_offset=192
[273] VAR pgm_tbl type_id=272 linkage=1
[274] PTR (anon) type_id=275
[275] STRUCT dp_pol_tact size=96 vlen=3
    ca type_id=39 bits_offset=0
    lock type_id=136 bits_offset=128
    (anon) type_id=276 bits_offset=192
[276] UNION (anon) size=72 vlen=1
    pol type_id=277 bits_offset=0
[277] STRUCT dp_policer_act size=72 vlen=13
    trtcm type_id=17 bits_offset=0
    color_aware type_id=17 bits_offset=8
    drop_prio type_id=14 bits_offset=16
    pad type_id=9 bits_offset=32
    cbs type_id=9 bits_offset=64
    ebs type_id=9 bits_offset=96
    tok_c type_id=9 bits_offset=128
    tok_e type_id=9 bits_offset=160
    toksc_pus type_id=11 bits_offset=192
    tokse_pus type_id=11 bits_offset=256
    lastc_uts type_id=11 bits_offset=320
    laste_uts type_id=11 bits_offset=384
    ps type_id=278 bits_offset=448
[278] STRUCT dp_pol_stats size=16 vlen=2
    drop_packets type_id=50 bits_offset=0
    pass_packets type_id=50 bits_offset=64
[279] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=274 bits_offset=128
    max_entries type_id=99 bits_offset=192
[280] VAR polx_map type_id=279 linkage=1
[281] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=5 bits_offset=64
    value type_id=242 bits_offset=128
    max_entries type_id=31 bits_offset=192
[282] VAR xfck type_id=281 linkage=1
[283] PTR (anon) type_id=284
[284] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=256
[285] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=47 bits_offset=128
    max_entries type_id=283 bits_offset=192
[286] VAR crc32c_map type_id=285 linkage=1
[287] PTR (anon) type_id=288
[288] ARRAY (anon) type_id=2 index_type_id=4 nr_elems=16
[289] STRUCT (anon) size=32 vlen=4
    type type_id=287 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=47 bits_offset=128
    max_entries type_id=60 bits_offset=192
[290] VAR cpu_map type_id=289 linkage=1
[291] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=47 bits_offset=128
    max_entries type_id=60 bits_offset=192
[292] VAR live_cpu_map type_id=291 linkage=1
[293] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=48 bits_offset=128
    max_entries type_id=115 bits_offset=192
[294] VAR pplat_map type_id=293 linkage=1
[295] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=134 bits_offset=128
    max_entries type_id=121 bits_offset=192
[296] VAR xctk type_id=295 linkage=1
[297] PTR (anon) type_id=298
[298] STRUCT gtp_parser size=64 vlen=8
    gh type_id=299 bits_offset=0
    geh type_id=300 bits_offset=64
    nh type_id=301 bits_offset=128
    gtp_next type_id=301 bits_offset=192
    hlen type_id=17 bits_offset=256
    nhl type_id=302 bits_offset=320
    neh type_id=302 bits_offset=384
    elen type_id=17 bits_offset=448
[299] PTR (anon) type_id=344
[300] PTR (anon) type_id=343
[301] PTR (anon) type_id=0
[302] PTR (anon) type_id=17
[303] STRUCT (anon) size=32 vlen=4
    type type_id=1 bits_offset=0
    key type_id=5 bits_offset=64
    value type_id=297 bits_offset=128
    max_entries type_id=31 bits_offset=192
[304] VAR gparser type_id=303 linkage=1
[305] PTR (anon) type_id=306
[306] STRUCT dp_ct_ctrtact size=32 vlen=5
    ca type_id=39 bits_offset=0
    lock type_id=136 bits_offset=128
    start type_id=9 bits_offset=160
    counter type_id=9 bits_offset=192
    entries type_id=9 bits_offset=224
[307] STRUCT (anon) size=32 vlen=4
    type type_id=121 bits_offset=0
    key type_id=47 bits_offset=64
    value type_id=305 bits_offset=128
    max_entries type_id=31 bits_offset=192
[308] VAR ct_ctr type_id=307 linkage=1
[309] PTR (anon) type_id=310
[310] STRUCT __sk_buff size=176 vlen=31
    len type_id=9 bits_offset=0
    pkt_type type_id=9 bits_offset=32
    mark type_id=9 bits_offset=64
    queue_mapping type_id=9 bits_offset=96
    protocol type_id=9 bits_offset=128
    vlan_present type_id=9 bits_offset=160
    vlan_tci type_id=9 bits_offset=192
    vlan_proto type_id=9 bits_offset=224
    priority type_id=9 bits_offset=256
    ingress_ifindex type_id=9 bits_offset=288
    ifindex type_id=9 bits_offset=320
    tc_index type_id=9 bits_offset=352
    cb type_id=311 bits_offset=384
    hash type_id=9 bits_offset=544
    tc_classid type_id=9 bits_offset=576
    data type_id=9 bits_offset=608
    data_end type_id=9 bits_offset=640
    napi_id type_id=9 bits_offset=672
    family type_id=9 bits_offset=704
    remote_ip4 type_id=9 bits_offset=736
    local_ip4 type_id=9 bits_offset=768
    remote_ip6 type_id=21 bits_offset=800
    local_ip6 type_id=21 bits_offset=928
    remote_port type_id=9 bits_offset=1056
    local_port type_id=9 bits_offset=1088
    data_meta type_id=9 bits_offset=1120
    (anon) type_id=312 bits_offset=1152
    tstamp type_id=11 bits_offset=1216
    wire_len type_id=9 bits_offset=1280
    gso_segs type_id=9 bits_offset=1312
    (anon) type_id=314 bits_offset=1344
[311] ARRAY (anon) type_id=9 index_type_id=4 nr_elems=5
[312] UNION (anon) size=8 vlen=1
    flow_keys type_id=313 bits_offset=0
[313] PTR (anon) type_id=341
[314] UNION (anon) size=8 vlen=1
    sk type_id=315 bits_offset=0
[315] PTR (anon) type_id=342
[316] FUNC_PROTO (anon) return=2 args=(309 md)
[317] FUNC tc_packet_func_fast type_id=316
[318] FUNC_PROTO (anon) return=2 args=(309 md)
[319] FUNC tc_packet_func type_id=318
[320] FUNC_PROTO (anon) return=2 args=(309 md)
[321] FUNC tc_packet_func_slow type_id=320
[322] FUNC_PROTO (anon) return=2 args=(309 md)
[323] FUNC tc_packet_func_fw type_id=322
[324] FUNC_PROTO (anon) return=2 args=(309 md)
[325] FUNC tc_csum_func1 type_id=324
[326] FUNC_PROTO (anon) return=2 args=(309 md)
[327] FUNC tc_csum_func2 type_id=326
[328] FUNC_PROTO (anon) return=2 args=(309 md)
[329] FUNC tc_slow_unp_func type_id=328
[330] INT char size=1 bits_offset=0 nr_bits=8 encoding=SIGNED
[331] ARRAY (anon) type_id=330 index_type_id=4 nr_elems=13
[332] VAR _license type_id=331 linkage=1
[333] CONST (anon) type_id=330
[334] ARRAY (anon) type_id=333 index_type_id=4 nr_elems=13
[335] VAR dp_ing_ct_main.____fmt type_id=334 linkage=0
[336] ARRAY (anon) type_id=333 index_type_id=4 nr_elems=16
[337] VAR dp_sctp_csum.____fmt type_id=336 linkage=0
[338] DATASEC .maps size=0 vlen=42 size == 0

Prog section 'tc_packet_hook0' rejected: Permission denied (13)!
 - Type:         3
 - Instructions: 3399 (0 over limit)
 - License:      Dual BSD/GPL

Verifier analysis:

0: R1=ctx(off=0,imm=0) R10=fp0
0: (bf) r6 = r1                       ; R1=ctx(off=0,imm=0) R6_w=ctx(off=0,imm=0)
1: (b7) r0 = 0                        ; R0_w=0
2: (61) r1 = *(u32 *)(r6 +60)         ; R1_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6_w=ctx(off=0,imm=0)
3: (18) r2 = 0xab1ef01d               ; R2_w=2870931485
5: (1d) if r1 == r2 goto pc+1134      ; R1_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R2_w=2870931485
6: (18) r1 = 0xf01dab1e               ; R1_w=4028476190
8: (63) *(u32 *)(r6 +60) = r1         ; R1_w=4028476190 R6_w=ctx(off=0,imm=0)
9: (b7) r1 = 0                        ; R1_w=0
10: (7b) *(u64 *)(r10 -80) = r1       ; R1_w=0 R10=fp0 fp-80_w=00000000
11: (7b) *(u64 *)(r10 -88) = r1       ; R1_w=0 R10=fp0 fp-88_w=00000000
12: (7b) *(u64 *)(r10 -96) = r1       ; R1_w=0 R10=fp0 fp-96_w=00000000
13: (7b) *(u64 *)(r10 -104) = r1      ; R1_w=0 R10=fp0 fp-104_w=00000000
14: (7b) *(u64 *)(r10 -112) = r1      ; R1_w=0 R10=fp0 fp-112_w=00000000
15: (7b) *(u64 *)(r10 -120) = r1      ; R1_w=0 R10=fp0 fp-120_w=00000000
16: (7b) *(u64 *)(r10 -128) = r1      ; R1_w=0 R10=fp0 fp-128_w=00000000
17: (7b) *(u64 *)(r10 -136) = r1      ; R1_w=0 R10=fp0 fp-136_w=00000000
18: (7b) *(u64 *)(r10 -144) = r1      ; R1_w=0 R10=fp0 fp-144_w=00000000
19: (7b) *(u64 *)(r10 -152) = r1      ; R1_w=0 R10=fp0 fp-152_w=00000000
20: (7b) *(u64 *)(r10 -160) = r1      ; R1_w=0 R10=fp0 fp-160_w=00000000
21: (7b) *(u64 *)(r10 -168) = r1      ; R1_w=0 R10=fp0 fp-168_w=00000000
22: (7b) *(u64 *)(r10 -176) = r1      ; R1_w=0 R10=fp0 fp-176_w=00000000
23: (7b) *(u64 *)(r10 -184) = r1      ; R1_w=0 R10=fp0 fp-184_w=00000000
24: (7b) *(u64 *)(r10 -192) = r1      ; R1_w=0 R10=fp0 fp-192_w=00000000
25: (7b) *(u64 *)(r10 -200) = r1      ; R1_w=0 R10=fp0 fp-200_w=00000000
26: (7b) *(u64 *)(r10 -208) = r1      ; R1_w=0 R10=fp0 fp-208_w=00000000
27: (7b) *(u64 *)(r10 -216) = r1      ; R1_w=0 R10=fp0 fp-216_w=00000000
28: (7b) *(u64 *)(r10 -224) = r1      ; R1_w=0 R10=fp0 fp-224_w=00000000
29: (7b) *(u64 *)(r10 -232) = r1      ; R1_w=0 R10=fp0 fp-232_w=00000000
30: (7b) *(u64 *)(r10 -240) = r1      ; R1_w=0 R10=fp0 fp-240_w=00000000
31: (7b) *(u64 *)(r10 -248) = r1      ; R1_w=0 R10=fp0 fp-248_w=00000000
32: (7b) *(u64 *)(r10 -256) = r1      ; R1_w=0 R10=fp0 fp-256_w=00000000
33: (7b) *(u64 *)(r10 -264) = r1      ; R1_w=0 R10=fp0 fp-264_w=00000000
34: (7b) *(u64 *)(r10 -272) = r1      ; R1_w=0 R10=fp0 fp-272_w=00000000
35: (7b) *(u64 *)(r10 -280) = r1      ; R1_w=0 R10=fp0 fp-280_w=00000000
36: (7b) *(u64 *)(r10 -288) = r1      ; R1_w=0 R10=fp0 fp-288_w=00000000
37: (7b) *(u64 *)(r10 -296) = r1      ; R1_w=0 R10=fp0 fp-296_w=00000000
38: (7b) *(u64 *)(r10 -304) = r1      ; R1_w=0 R10=fp0 fp-304_w=00000000
39: (7b) *(u64 *)(r10 -312) = r1      ; R1_w=0 R10=fp0 fp-312_w=00000000
40: (7b) *(u64 *)(r10 -320) = r1      ; R1_w=0 R10=fp0 fp-320_w=00000000
41: (7b) *(u64 *)(r10 -328) = r1      ; R1_w=0 R10=fp0 fp-328_w=00000000
42: (7b) *(u64 *)(r10 -336) = r1      ; R1_w=0 R10=fp0 fp-336_w=00000000
43: (7b) *(u64 *)(r10 -344) = r1      ; R1_w=0 R10=fp0 fp-344_w=00000000
44: (7b) *(u64 *)(r10 -352) = r1      ; R1_w=0 R10=fp0 fp-352_w=00000000
45: (7b) *(u64 *)(r10 -360) = r1      ; R1_w=0 R10=fp0 fp-360_w=00000000
46: (7b) *(u64 *)(r10 -368) = r1      ; R1_w=0 R10=fp0 fp-368_w=00000000
47: (7b) *(u64 *)(r10 -376) = r1      ; R1_w=0 R10=fp0 fp-376_w=00000000
48: (7b) *(u64 *)(r10 -384) = r1      ; R1_w=0 R10=fp0 fp-384_w=00000000
49: (7b) *(u64 *)(r10 -392) = r1      ; R1_w=0 R10=fp0 fp-392_w=00000000
50: (7b) *(u64 *)(r10 -400) = r1      ; R1_w=0 R10=fp0 fp-400_w=00000000
51: (7b) *(u64 *)(r10 -408) = r1      ; R1_w=0 R10=fp0 fp-408_w=00000000
52: (7b) *(u64 *)(r10 -416) = r1      ; R1_w=0 R10=fp0 fp-416_w=00000000
53: (7b) *(u64 *)(r10 -424) = r1      ; R1_w=0 R10=fp0 fp-424_w=00000000
54: (7b) *(u64 *)(r10 -432) = r1      ; R1_w=0 R10=fp0 fp-432_w=00000000
55: (61) r8 = *(u32 *)(r6 +76)        ; R6_w=ctx(off=0,imm=0) R8_w=pkt(off=0,r=0,imm=0)
56: (61) r7 = *(u32 *)(r6 +80)        ; R6_w=ctx(off=0,imm=0) R7_w=pkt_end(off=0,imm=0)
57: (bf) r1 = r7                      ; R1_w=pkt_end(off=0,imm=0) R7_w=pkt_end(off=0,imm=0)
58: (1f) r1 -= r8                     ; R1_w=scalar() R8_w=pkt(off=0,r=0,imm=0)
59: (6b) *(u16 *)(r10 -150) = r1      ; R1_w=scalar() R10=fp0 fp-152_w=0000mm00
60: (bf) r9 = r8                      ; R8_w=pkt(off=0,r=0,imm=0) R9_w=pkt(off=0,r=0,imm=0)
61: (07) r9 += 14                     ; R9_w=pkt(off=14,r=0,imm=0)
62: (18) r2 = 0xffffffff              ; R2_w=4294967295
64: (2d) if r9 > r7 goto pc+116       ; R7_w=pkt_end(off=0,imm=0) R9_w=pkt(off=14,r=14,imm=0)
65: (b7) r1 = 1                       ; R1_w=1
66: (73) *(u8 *)(r10 -395) = r1       ; R1_w=1 R10=fp0 fp-400_w=00m00000
67: (71) r1 = *(u8 *)(r8 +9)          ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
68: (67) r1 <<= 8                     ; R1_w=scalar(umax=65280,var_off=(0x0; 0xff00))
69: (71) r2 = *(u8 *)(r8 +8)          ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
70: (4f) r1 |= r2                     ; R1_w=scalar() R2_w=scalar(umax=255,var_off=(0x0; 0xff))
71: (71) r3 = *(u8 *)(r8 +10)         ; R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
72: (67) r3 <<= 16                    ; R3_w=scalar(umax=16711680,var_off=(0x0; 0xff0000))
73: (71) r2 = *(u8 *)(r8 +11)         ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
74: (67) r2 <<= 24                    ; R2_w=scalar(umax=4278190080,var_off=(0x0; 0xff000000),s32_max=2130706432)
75: (4f) r2 |= r3                     ; R2_w=scalar() R3_w=scalar(umax=16711680,var_off=(0x0; 0xff0000))
76: (4f) r2 |= r1                     ; R1_w=scalar() R2_w=scalar()
77: (71) r3 = *(u8 *)(r8 +1)          ; R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
78: (67) r3 <<= 8                     ; R3_w=scalar(umax=65280,var_off=(0x0; 0xff00))
79: (71) r1 = *(u8 *)(r8 +0)          ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
80: (4f) r3 |= r1                     ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R3_w=scalar()
81: (71) r4 = *(u8 *)(r8 +2)          ; R4_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
82: (67) r4 <<= 16                    ; R4_w=scalar(umax=16711680,var_off=(0x0; 0xff0000))
83: (71) r1 = *(u8 *)(r8 +3)          ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
84: (67) r1 <<= 24                    ; R1_w=scalar(umax=4278190080,var_off=(0x0; 0xff000000),s32_max=2130706432)
85: (4f) r1 |= r4                     ; R1_w=scalar() R4_w=scalar(umax=16711680,var_off=(0x0; 0xff0000))
86: (63) *(u32 *)(r10 -400) = r2      ; R2_w=scalar() R10=fp0 fp-400_w=00m0mmmm
87: (4f) r1 |= r3                     ; R1_w=scalar() R3_w=scalar()
88: (71) r2 = *(u8 *)(r8 +5)          ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
89: (67) r2 <<= 8                     ; R2_w=scalar(umax=65280,var_off=(0x0; 0xff00))
90: (71) r3 = *(u8 *)(r8 +4)          ; R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
91: (4f) r2 |= r3                     ; R2_w=scalar() R3_w=scalar(umax=255,var_off=(0x0; 0xff))
92: (71) r3 = *(u8 *)(r8 +6)          ; R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
93: (67) r3 <<= 16                    ; R3_w=scalar(umax=16711680,var_off=(0x0; 0xff0000))
94: (71) r4 = *(u8 *)(r8 +7)          ; R4_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
95: (67) r4 <<= 24                    ; R4_w=scalar(umax=4278190080,var_off=(0x0; 0xff000000),s32_max=2130706432)
96: (4f) r4 |= r3                     ; R3_w=scalar(umax=16711680,var_off=(0x0; 0xff0000)) R4_w=scalar()
97: (4f) r4 |= r2                     ; R2_w=scalar() R4_w=scalar()
98: (67) r4 <<= 32                    ; R4_w=scalar(smax=9223372032559808512,umax=18446744069414584320,var_off=(0x0; 0xffffffff00000000),s32_min=0,s32_max=0,u32_max=0)
99: (4f) r4 |= r1                     ; R1_w=scalar() R4_w=scalar()
100: (7b) *(u64 *)(r10 -408) = r4     ; R4_w=scalar() R10=fp0 fp-408_w=mmmmmmmm
101: (71) r1 = *(u8 *)(r8 +5)         ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
102: (67) r1 <<= 8                    ; R1_w=scalar(umax=65280,var_off=(0x0; 0xff00))
103: (71) r2 = *(u8 *)(r8 +4)         ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
104: (4f) r1 |= r2                    ; R1_w=scalar() R2_w=scalar(umax=255,var_off=(0x0; 0xff))
105: (71) r2 = *(u8 *)(r8 +1)         ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
106: (67) r2 <<= 8                    ; R2_w=scalar(umax=65280,var_off=(0x0; 0xff00))
107: (71) r3 = *(u8 *)(r8 +0)         ; R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
108: (4f) r2 |= r3                    ; R2_w=scalar() R3_w=scalar(umax=255,var_off=(0x0; 0xff))
109: (6b) *(u16 *)(r10 -134) = r2     ; R2_w=scalar() R10=fp0 fp-136_w=0000mm00
110: (6b) *(u16 *)(r10 -130) = r1     ; R1_w=scalar() R10=fp0 fp-136_w=mm00mm00
111: (71) r1 = *(u8 *)(r8 +2)         ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
112: (67) r1 <<= 16                   ; R1_w=scalar(umax=16711680,var_off=(0x0; 0xff0000))
113: (71) r2 = *(u8 *)(r8 +3)         ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
114: (67) r2 <<= 24                   ; R2_w=scalar(umax=4278190080,var_off=(0x0; 0xff000000),s32_max=2130706432)
115: (4f) r2 |= r1                    ; R1_w=scalar(umax=16711680,var_off=(0x0; 0xff0000)) R2_w=scalar()
116: (77) r2 >>= 16                   ; R2_w=scalar(umax=281474976710655,var_off=(0x0; 0xffffffffffff))
117: (6b) *(u16 *)(r10 -132) = r2     ; R2_w=scalar(umax=281474976710655,var_off=(0x0; 0xffffffffffff)) R10=fp0 fp-136_w=mmmmmm00
118: (71) r2 = *(u8 *)(r8 +12)        ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
119: (71) r1 = *(u8 *)(r8 +13)        ; R1_w=scalar(umax=255,var_off=(0x0; 0xff)) R8_w=pkt(off=0,r=14,imm=0)
120: (67) r1 <<= 8                    ; R1_w=scalar(umax=65280,var_off=(0x0; 0xff00))
121: (4f) r1 |= r2                    ; R1_w=scalar() R2_w=scalar(umax=255,var_off=(0x0; 0xff))
122: (6b) *(u16 *)(r10 -410) = r1     ; R1_w=scalar() R10=fp0 fp-416_w=mm000000
123: (b7) r3 = 6                      ; R3_w=6
124: (b7) r2 = 2                      ; R2=2
125: (2d) if r3 > r1 goto pc+55       ; R1=scalar(umin=6) R3=6
126: (61) r2 = *(u32 *)(r6 +48)       ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(off=0,imm=0)
127: (18) r3 = 0xdeadbeef             ; R3_w=3735928559
129: (5d) if r2 != r3 goto pc+2       ; R2_w=3735928559 R3_w=3735928559
130: (61) r2 = *(u32 *)(r6 +52)       ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(off=0,imm=0)
131: (6b) *(u16 *)(r10 -104) = r2     ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-104=
132: (61) r2 = *(u32 *)(r6 +20)       ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(off=0,imm=0)
133: (15) if r2 == 0x0 goto pc+3      ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
134: (61) r2 = *(u32 *)(r6 +24)       ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(off=0,imm=0)
135: (dc) r2 = be16 r2                ; R2_w=scalar()
136: (6b) *(u16 *)(r10 -416) = r2     ; R2=scalar() R10=fp0 fp-416=mm0000mm
137: (bf) r2 = r9                     ; R2_w=pkt(off=14,r=14,imm=0) R9=pkt(off=14,r=14,imm=0)
138: (1f) r2 -= r8                    ; R2_w=scalar() R8=pkt(off=0,r=14,imm=0)
139: (73) *(u8 *)(r10 -147) = r2      ; R2_w=scalar() R10=fp0 fp-152=00m0mm00
140: (65) if r1 s> 0x9998 goto pc+893         ; R1=scalar(smax=39320,umin=6)
141: (15) if r1 == 0x8 goto pc+112    ; R1=scalar(smax=39320,umin=6)
142: (15) if r1 == 0x608 goto pc+1    ; R1=scalar(smax=39320,umin=6)
143: (05) goto pc+894
1038: (b7) r1 = 524288                ; R1_w=524288
1039: (b7) r3 = 8                     ; R3_w=8
1040: (61) r2 = *(u32 *)(r6 +4)       ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(off=0,imm=0)
1041: (07) r2 += -3                   ; R2_w=scalar(smin=-3,smax=4294967292)
1042: (67) r2 <<= 32                  ; R2_w=scalar(smax=9223372032559808512,umax=18446744069414584320,var_off=(0x0; 0xffffffff00000000),s32_min=0,s32_max=0,u32_max=0)
1043: (77) r2 >>= 32                  ; R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
1044: (18) r4 = 0xfffffffe            ; R4=4294967294
1046: (2d) if r4 > r2 goto pc-853     ; R2=scalar(umin=4294967294,umax=4294967295,var_off=(0xfffffffe; 0x1),s32_min=-2,s32_max=-1) R4=4294967294
1047: (05) goto pc-860
188: (71) r2 = *(u8 *)(r10 -148)      ; R2_w=P0 R10=fp0 fp-152=00m0mm00
189: (4f) r2 |= r3                    ; R2_w=P8 R3=8
190: (73) *(u8 *)(r10 -148) = r2      ; R2_w=P8 R10=fp0 fp-152=00mmmm00
191: (61) r2 = *(u32 *)(r10 -140)     ; R2_w=P0 R10=fp0 fp-144=00000000
192: (4f) r2 |= r1                    ; R1=524288 R2_w=P524288
193: (63) *(u32 *)(r10 -140) = r2     ; R2=524288 R10=fp0 fp-144=mmmm0000
194: (b7) r1 = 0                      ; R1_w=0
195: (63) *(u32 *)(r10 -72) = r1      ; R1_w=0 R10=fp0 fp-72=????0000
196: (71) r2 = *(u8 *)(r10 -148)      ; R2_w=scalar(umax=255,var_off=(0x0; 0xff)) R10=fp0 fp-152=00mmmm00
197: (55) if r2 != 0x0 goto pc+856    ; R2_w=0
198: (69) r1 = *(u16 *)(r10 -410)     ; R1_w=scalar(umax=65535,var_off=(0x0; 0xffff)) R10=fp0 fp-416=mm0000mm
199: (55) if r1 != 0x8 goto pc+883    ; R1_w=8
200: (61) r1 = *(u32 *)(r10 -360)     ; R1_w=P0 R10=fp0 fp-360=00000000
201: (63) *(u32 *)(r10 -64) = r1      ; R1_w=P0 R10=fp0 fp-64=????0000
202: (61) r1 = *(u32 *)(r10 -376)     ; R1_w=P0 R10=fp0 fp-376=00000000
203: (63) *(u32 *)(r10 -60) = r1      ; R1_w=P0 R10=fp0 fp-64=00000000
204: (69) r1 = *(u16 *)(r10 -388)     ; R1_w=P0 R10=fp0 fp-392=00000000
205: (6b) *(u16 *)(r10 -56) = r1      ; R1_w=P0 R10=fp0 fp-56=??????00
206: (69) r1 = *(u16 *)(r10 -386)     ; R1_w=P0 R10=fp0 fp-392=00000000
207: (6b) *(u16 *)(r10 -54) = r1      ; R1_w=P0 R10=fp0 fp-56=????0000
208: (71) r1 = *(u8 *)(r10 -391)      ; R1_w=P0 R10=fp0 fp-392=00000000
209: (73) *(u8 *)(r10 -52) = r1       ; R1_w=P0 R10=fp0 fp-56=???00000
210: (b7) r1 = 19                     ; R1_w=19
211: (73) *(u8 *)(r10 -121) = r1      ; R1_w=19 R10=fp0 fp-128=m0000000
212: (b7) r1 = 0                      ; R1_w=0
213: (6b) *(u16 *)(r10 -50) = r1      ; R1_w=0 R10=fp0 fp-56=00?00000
214: (73) *(u8 *)(r10 -51) = r1       ; R1_w=0 R10=fp0 fp-56=00000000
215: (63) *(u32 *)(r10 -68) = r1      ; R1_w=0 R10=fp0 fp-72=00000000
216: (bf) r2 = r10                    ; R2_w=fp0 R10=fp0
217: (07) r2 += -64                   ; R2_w=fp-64
218: (18) r1 = 0x0                    ; R1_w=0
220: (85) call bpf_map_lookup_elem#1
R1 type=scalar expected=map_ptr
processed 181 insns (limit 1000000) max_states_per_insn 0 total_states 4 peak_states 4 mark_read 4

Error fetching program/map!
Unable to load program

To Reproduce

  1. compile ntc on ubuntu 22.04 and copy ntc to BPFire OS (fork of IPFire)
  2. compile loxilb and copy /opt/loxilb/*.o to BPFire OS /opt/loxilb
  3. run ntc filter add dev red0 ingress bpf da obj /opt/loxilb/llb_ebpf_main.o sec tc_packet_hook0 verbos

Expected behavior

expect program load and attach

Screenshots If applicable, add screenshots to help explain your problem.

Environment (please complete the following information):

Additional context

trying to port loxilb to open source firewall BPFire OS https://github.com/vincentmli/BPFire/issues/12

vincentmli commented 5 months ago

I first tried to start loxilb, but I get error below, that is why I tried ntc manually, my attempt is to attach to TC hook, it appears loxilb trying XDP hook first?

[root@bpfire-3 ~]# ./loxilb 
loxilb start
INFO: xdp_link_detach() no curr XDP prog on ifindex:6
09:38:04 DEBUG loxilb_libdp.c:2682: exec: ntc filter del dev llb0 ingress 2>&1 > /dev/null
Error: Parent Qdisc doesn't exists.
We have an error talking to the kernel
09:38:04 DEBUG loxilb_libdp.c:2743: /opt/loxilb/llb_xdp_main.o: nr 0 psection xdp_packet_hook
libbpf: map 'cpu_map': failed to create: Argument list too long(-7)
libbpf: failed to load object '/opt/loxilb/llb_xdp_main.o'
ERR: loading BPF-OBJ file(/opt/loxilb/llb_xdp_main.o) (-7): Argument list too long
ERR: loading file: /opt/loxilb/llb_xdp_main.o
UltraInstinct14 commented 5 months ago

Yes you are right. First it loads its XDP program which also creates/pins its map. The same maps are then shared across loxilb's ebpf programs as well.

vincentmli commented 5 months ago

thanks for the quick response, is there option to allow loxilb load and attach in TC, skip XDP? I already have other XDP program attached to the interface by xdp-tools xdp-loader, I prefer loxilb attach program to TC

UltraInstinct14 commented 5 months ago

loxilb only loads its xdp in its own interface "llb0" and does not use other system interfaces. If you can prevent loading other xdp programs in llb0, probably sudo ./loxilb would work.

vincentmli commented 5 months ago

I see, I am not loading other xdp program in llb0 since I don't know that llb0 even exists :). I already have other xdp program load and attached to my red0 interface which is a real system interface in BPFire by xdp-loader. I read somewhere that it says loxilb will try to load program to each system interfaces and cause conflict? maybe I read it wrong. anyway any clue on https://github.com/loxilb-io/loxilb/issues/661#issuecomment-2090974974 error ? I think it is nice still for loxilb to have options to load the program to specific system interface and on specific hook like TC, that is flexible in my opinion :)

UltraInstinct14 commented 5 months ago

No it is not true that loxilb will try to load in all interfaces. loxilb has an option "--blacklist=" and it will not load in any interface in this blacklist.

This is a close to manual way of loading loxilb xdp/ebpf 

loxilb-ebpf/utils/mkllb_bpffs.sh
sudo ip link add tap0 type veth peer name tap1
sudo ifconfig tap0 up
sudo ifconfig tap1 up
sudo mkdir -p /opt/fs/bpf
sudo bpftool -d prog  load  /opt/loxilb/llb_xdp_main.o /opt/fs/bpf/xdp_packet_hook type xdp pinmaps /opt/loxilb/dp/bpf
sudo ntc filter add dev tap0 egress bpf da obj /opt/loxilb/llb_ebpf_main.o sec tc_packet_hook0

Last step will fail because some maps are not initialized fully. Those maps can be filled using bpftool as well but need to check more.

vincentmli commented 5 months ago

ok, I got it, do you want me to repeat above manual steps to figure out the missing maps ? and that is the possible reason to cause verifier complains ?

UltraInstinct14 commented 5 months ago

ok, I got it, do you want me to repeat above manual steps to figure out the missing maps ? and that is the possible reason to cause verifier complains ?

Yes kindly try the manual steps once.

UltraInstinct14 commented 5 months ago

And to remove xdp loading you can use the following patch:

diff --git a/common/common_user_bpf_xdp.c b/common/common_user_bpf_xdp.c
index 7e7e74e..496554f 100644
--- a/common/common_user_bpf_xdp.c
+++ b/common/common_user_bpf_xdp.c
@@ -283,6 +283,7 @@ struct bpf_object *load_bpf_and_xdp_attach(struct config *cfg)
     exit(EXIT_FAIL_BPF);
   }

+#if 0
   /* At this point: BPF-progs are (only) loaded by the kernel, and prog_fd
    * is our select file-descriptor handle. Next step is attaching this FD
    * to a kernel hook point, in this case XDP net_device link-level hook.
@@ -293,6 +294,7 @@ struct bpf_object *load_bpf_and_xdp_attach(struct config *cfg)
     /* Consider this soft error */
     /* exit(err); */
   }
+#endif

   return bpf_obj;
 }

Kindly recompile and recheck !!

vincentmli commented 5 months ago

bpftool -d prog load /opt/loxilb/llb_xdp_main.o /opt/fs/bpf/xdp_packet_hook type xdp pinmaps /opt/loxilb/dp/bpf

ok, here is the manual step output from bpftool, it looks the problem is in [ ](libbpf: map 'cpu_map': failed to create: Argument list too long(-7))

libbpf: map 'ct_ctr': found max_entries = 1.
libbpf: prog 'xdp_packet_func': unrecognized ELF section name 'xdp_packet_hook'
libbpf: prog 'xdp_pass_func': unrecognized ELF section name 'xdp_pass'
libbpf: sec '.relxdp_packet_hook': collecting relocation for section(3) 'xdp_packet_hook'
libbpf: sec '.relxdp_packet_hook': relo #0: insn #5 against 'xfis'
libbpf: prog 'xdp_packet_func': found map 0 (xfis, sec 6, off 0) for insn #5
libbpf: sec '.relxdp_packet_hook': relo #1: insn #572 against 'live_cpu_map'
libbpf: prog 'xdp_packet_func': found map 1 (live_cpu_map, sec 6, off 32) for insn #572
libbpf: sec '.relxdp_packet_hook': relo #2: insn #596 against 'cpu_map'
libbpf: prog 'xdp_packet_func': found map 2 (cpu_map, sec 6, off 64) for insn #596
libbpf: sec '.relxdp_packet_hook': relo #3: insn #1060 against 'gparser'
libbpf: prog 'xdp_packet_func': found map 40 (gparser, sec 6, off 1296) for insn #1060
libbpf: map 'xfis': created successfully, fd=4
libbpf: map 'live_cpu_map': created successfully, fd=5
libbpf: map 'cpu_map': failed to create: Argument list too long(-7)
libbpf: failed to load object '/opt/loxilb/llb_xdp_main.o'
Error: failed to load object file
vincentmli commented 5 months ago

just fyi, I have libbpf 1.3.0 installed in BPFire system

UltraInstinct14 commented 5 months ago

Got it. It seems libbpf version mismatch. By the way latest loxilb does not use the shared libbpf any more. You can check -

ldd ./loxilb
ldd /usr/local/sbin/ntc
vincentmli commented 5 months ago

yes, loxilb is compiled with static libbpf 0.8

[root@bpfire-3 ~]# ldd loxilb 
    linux-vdso.so.1 (0x00007fffd0b79000)
    libelf.so.1 => /usr/lib/libelf.so.1 (0x00007fd6f41b2000)
    libz.so.1 => /lib64/libz.so.1 (0x00007fd6f4198000)
    libc.so.6 => /lib64/libc.so.6 (0x00007fd6f3fb7000)
    libzstd.so.1 => /usr/lib/libzstd.so.1 (0x00007fd6f3ef5000)
    liblzma.so.5 => /usr/lib/liblzma.so.5 (0x00007fd6f3ec5000)
    libbz2.so.1.0 => /lib/libbz2.so.1.0 (0x00007fd6f3eb2000)
    /lib64/ld-linux-x86-64.so.2 (0x00007fd6f41e3000)
[root@bpfire-3 ~]# ldd /usr/local/bin/ntc 
    linux-vdso.so.1 (0x00007ffed4590000)
    libelf.so.1 => /usr/lib/libelf.so.1 (0x00007fd8a1d53000)
    libmnl.so.0 => /usr/lib/libmnl.so.0 (0x00007fd8a1d4c000)
    libm.so.6 => /lib/libm.so.6 (0x00007fd8a1c60000)
    libc.so.6 => /lib/libc.so.6 (0x00007fd8a1a7f000)
    libzstd.so.1 => /usr/lib/libzstd.so.1 (0x00007fd8a19bd000)
    liblzma.so.5 => /usr/lib/liblzma.so.5 (0x00007fd8a198d000)
    libbz2.so.1.0 => /lib/libbz2.so.1.0 (0x00007fd8a1978000)
    /lib64/ld-linux-x86-64.so.2 (0x00007fd8a1e1c000)

my bpftool version

[root@bpfire-3 ~]# bpftool -V
bpftool v7.3.0
using libbpf v1.3
features: llvm, skeletons

[root@bpfire-3 ~]# ldd /usr/sbin/bpftool 
    linux-vdso.so.1 (0x00007ffe74b5e000)
    libelf.so.1 => /usr/lib/libelf.so.1 (0x00007237c3688000)
    libcap.so.2 => /usr/lib/libcap.so.2 (0x00007237c367c000)
    libm.so.6 => /lib/libm.so.6 (0x00007237c3590000)
    libzstd.so.1 => /usr/lib/libzstd.so.1 (0x00007237c34ce000)
    libncursesw.so.6 => /usr/lib/libncursesw.so.6 (0x00007237c3455000)
    libxml2.so.2 => /usr/lib/libxml2.so.2 (0x00007237c32da000)
    libstdc++.so.6 => /usr/lib/libstdc++.so.6 (0x00007237c3000000)
    libgcc_s.so.1 => /usr/lib/libgcc_s.so.1 (0x00007237c32b3000)
    libc.so.6 => /lib/libc.so.6 (0x00007237c2e1f000)
    /lib64/ld-linux-x86-64.so.2 (0x00007237c36b9000)
    liblzma.so.5 => /usr/lib/liblzma.so.5 (0x00007237c2def000)
    libbz2.so.1.0 => /lib/libbz2.so.1.0 (0x00007237c32a0000)
vincentmli commented 5 months ago

@UltraInstinct14 I have tried to build a few bpftool versions and all failed with same error with bpftool -d prog load /opt/loxilb/llb_xdp_main.o ..., could you let me know what bpftool version that has been working for you?

vincentmli commented 5 months ago

@UltraInstinct14 I have tried to build a few bpftool versions and all failed with same error with bpftool -d prog load /opt/loxilb/llb_xdp_main.o ..., could you let me know what bpftool version that has been working for you?

now I am not sure if it bpftool or libbpf version issue since I build bpftool v6.8.0 with libbpf v0.8, the issue is still the same

[root@bpfire-2 /]# /tmp/bpftool -V
/tmp/bpftool v6.8.0
using libbpf v0.8
features: libbpf_strict, skeletons

libbpf: map 'live_cpu_map': created successfully, fd=5
libbpf: map 'cpu_map': failed to create: Argument list too long(-7)
libbpf: failed to load object '/opt/loxilb/llb_xdp_main.o'
Error: failed to load object file
Warning: bpftool is now running in libbpf strict mode and has more stringent requirements about BPF programs.
If it used to work for this object file but now doesn't, see --legacy option for more details.
vincentmli commented 5 months ago

I wonder why live_cpu_map succeeded, but not cpu_map, the difference is the map type is BPF_MAP_TYPE_ARRAY vs BPF_MAP_TYPE_CPUMAP

struct {
              __uint(type,        BPF_MAP_TYPE_CPUMAP);
              __type(key,         __u32);
              __type(value,       __u32);
              __uint(max_entries, MAX_CPUS);
} cpu_map SEC(".maps");

struct {
              __uint(type,        BPF_MAP_TYPE_ARRAY);
              __type(key,         __u32);
              __type(value,       __u32);
              __uint(max_entries, MAX_CPUS);
} live_cpu_map SEC(".maps");
vincentmli commented 5 months ago

I wonder why live_cpu_map succeeded, but not cpu_map, the difference is the map type is BPF_MAP_TYPE_ARRAY vs BPF_MAP_TYPE_CPUMAP

struct {
              __uint(type,        BPF_MAP_TYPE_CPUMAP);
              __type(key,         __u32);
              __type(value,       __u32);
              __uint(max_entries, MAX_CPUS);
} cpu_map SEC(".maps");

struct {
              __uint(type,        BPF_MAP_TYPE_ARRAY);
              __type(key,         __u32);
              __type(value,       __u32);
              __uint(max_entries, MAX_CPUS);
} live_cpu_map SEC(".maps");

so I hacked the code to change cpu_map to BPF_MAP_TYPE_ARRAY type, libbpf get passed ok

libbpf: map 'live_cpu_map': created successfully, fd=5
libbpf: map 'cpu_map': created successfully, fd=6

but of course I get error below since cpu_map type does not match

599: (b7) r3 = 0                      ; R3_w=0
600: (85) call bpf_redirect_map#51
cannot pass map_type 2 into func bpf_redirect_map#51
verification time 2872 usec
stack depth 40

the problem seems related to BPF_MAP_TYPE_CPUMAP and libbpf, some sort of limitation there.

vincentmli commented 5 months ago

ok I made progress, if I change cpu_map below from MAX_CPUS to 2, it seems the map get pinned ok

struct {
              __uint(type,        BPF_MAP_TYPE_CPUMAP);
              __type(key,         __u32);
              __type(value,       __u32);
              __uint(max_entries, 2);
} cpu_map SEC(".maps");

bpftool -d prog load /opt/loxilb/llb_xdp_main.o /opt/fs/bpf/xdp_packet_hook type xdp pinmaps /opt/loxilb/dp/bpf

libbpf: prog 'xdp_pass_func': -- BEGIN PROG LOAD LOG --
func#0 @0
0: R1=ctx(off=0,imm=0) R10=fp0
; return dp_ing_pass_main(ctx);
0: (b7) r0 = 2                        ; R0_w=2
1: (95) exit
verification time 51 usec
stack depth 0
processed 2 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0
-- END PROG LOAD LOG --
libbpf: pinned map '/opt/loxilb/dp/bpf/xfis'
libbpf: pinned map '/opt/loxilb/dp/bpf/live_cpu_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/cpu_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/intf_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/intf_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/bd_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/pkt_ring'
libbpf: pinned map '/opt/loxilb/dp/bpf/cp_ring'
libbpf: pinned map '/opt/loxilb/dp/bpf/pkts'
libbpf: pinned map '/opt/loxilb/dp/bpf/fcas'
libbpf: pinned map '/opt/loxilb/dp/bpf/tx_intf_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/tx_intf_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/tx_bd_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/smac_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/dmac_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/tmac_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/tmac_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/nh_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/ct_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/ct_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/nat_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/nat_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/nat_ep_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/rt_v4_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/rt_v4_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/rt_v6_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/rt_v6_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/mirr_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/sess_v4_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/sess_v4_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/fc_v4_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/fc_v4_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/fw_v4_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/fw_v4_stats_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/pgm_tbl'
libbpf: pinned map '/opt/loxilb/dp/bpf/polx_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/xfck'
libbpf: pinned map '/opt/loxilb/dp/bpf/crc32c_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/pplat_map'
libbpf: pinned map '/opt/loxilb/dp/bpf/xctk'
libbpf: pinned map '/opt/loxilb/dp/bpf/gparser'
libbpf: pinned map '/opt/loxilb/dp/bpf/ct_ctr'
vincentmli commented 5 months ago

ok, I think I found the problem, loxilb has MAX_CPUS set to 128 that is greater than BPFire NR_CPUS which is 64, I can adjust BPFire NR_CPUS to values bigger than 128

kernel/bpf/cpumap.c:

static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
{
        u32 value_size = attr->value_size;
        struct bpf_cpu_map *cmap;

        /* check sanity of attributes */
        if (attr->max_entries == 0 || attr->key_size != 4 ||
            (value_size != offsetofend(struct bpf_cpumap_val, qsize) &&
             value_size != offsetofend(struct bpf_cpumap_val, bpf_prog.fd)) ||
            attr->map_flags & ~BPF_F_NUMA_NODE)
                return ERR_PTR(-EINVAL);

        /* Pre-limit array size based on NR_CPUS, not final CPU check */
        if (attr->max_entries > NR_CPUS)
                return ERR_PTR(-E2BIG);  <======

include/linux/threads.h

/* Places which use this should consider cpumask_var_t. */
#define NR_CPUS         CONFIG_NR_CPUS

and BPFire/IPFire config/kernel/kernel.config.x86_64-ipfire has

CONFIG_NR_CPUS_RANGE_BEGIN=2
CONFIG_NR_CPUS_RANGE_END=512
CONFIG_NR_CPUS_DEFAULT=64
CONFIG_NR_CPUS=64 <====
TrekkieCoder commented 5 months ago

Hi @vincentmli Great find !!