search

ovn-org / ovn

Open Virtual Network
Apache License 2.0
521 stars 254 forks source link

sNAT Hardware offload cannot be implemented #207

Closed wanghuiheze closed 10 months ago

wanghuiheze commented 1 year ago

Test the sNAT hardware offload Experimental environment:

central 10.185.11.71/24 - OVN Central
gateway01 172.31.5.23/24 - OVN Gateway Node
node1 172.31.1.21/24 – will serve as an OVN Host
client 172.31.6.24/24 - physical network node
#We use namespace instead of VM
namespace 10.199.100.10 - in node1 Node

tunnel:geneve

#OVS version
[root@gateway01 ~]# ovs-vsctl -V
ovs-vsctl (Open vSwitch) 2.17.7-e054917
DB Schema 8.3.1
[root@node01 ~]# ovs-vsctl -V
ovs-vsctl (Open vSwitch) 2.17.6-c5d234f
DB Schema 8.3.0

#OVN version
[root@newcentral ~]# ovn-nbctl -V
ovn-nbctl 23.03.0
Open vSwitch Library 3.1.0
DB Schema 7.0.0
[root@newcentral ~]# ovn-sbctl -V
ovn-sbctl 23.03.0
Open vSwitch Library 3.1.0
DB Schema 20.27.0

#Kernal version
[root@gateway01 ~]# uname -a
Linux gateway01 5.14.0-284.11.1.el9_2.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 9 05:49:00 EDT 2023 x86_64 x86_64 x86_64 GNU/Linux
[root@gateway01 ~]# cat /etc/redhat-release 
AlmaLinux release 9.2 (Turquoise Kodkod)

[root@node01 ~]# uname -a
Linux node01 4.18.0-425.3.1.el8.x86_64 #1 SMP Tue Nov 8 14:08:25 EST 2022 x86_64 x86_64 x86_64 GNU/Linux
[root@node01 ~]# cat /etc/redhat-release 
AlmaLinux release 8.7 (Stone Smilodon)

                   ____ ____ 
                  |  client | 172.31.4.0/24 Physical Network
                   ---------
                       |
                   ____|____ 
                  |  switch | localSW1
                   ---------          
                   ____|____          snat 123.123.123.123   10.199.100.0/24
                  |  router | GR1     port 'GR1-localSW1': 172.31.5.230/24
                   ---------          port 'GR1-joinSW':    172.16.255.3/24
                   ____|____ 
                  |  switch | joinSW       172.16.255.0/24
                   ---------          
                       |
                   ____|____ 
                  |  router | dr      port 'dr1-joinSW':  172.16.255.2/24
                   ---------          port 'dr-sw1': 10.199.100.1/24
                   /       \          port 'dr-sw2': 10.199.200.1/24
           _______/_       _\_______  
          |  sw1    |     |   sw2   | sw1:10.199.100.0/24
           ---------       ---------  sw2:10.199.200.0/24
            / 
    _______/_ 
   | sw1-vm1 |
    --------- 
10.199.100.10

View the ovn database:

[root@newcentral ~]# ovn-nbctl show 
switch 9b507f28-6635-43d5-a782-f8561d01a6ff (subnet_04b62bbe1f147000)
    port subnet_04b62bbe1f147000-router
        type: router
        router-port: vpc_049e374424847000-subnet_04b62bbe1f147000
    port i_04b636e391c47000
        addresses: ["00:00:19:92:00:20 10.199.200.20/24"]
switch 84b95eb4-8244-4fd4-b39a-1555e952a0b4 (vpc_04a7d57200347000-vbr)
    port ri_04af32c445847000
        type: router
        router-port: vbr_04a0be3b4b247000-vpc_04a7d57200347000
    port ri_04aacfb978547000
        type: router
        router-port: vbr_04a0be3427447000-vpc_04a7d57200347000
    port vpc_04a7d57200347000-join-port
        type: router
        router-port: vpc_04a7d57200347000-vbr-port
switch 939b6ec0-be19-49b4-aa82-dbfa626d2022 (vbr_04a0be3b4b247000-local)
    port vbr_04a0be3b4b247000-local-network
        type: localnet
        addresses: ["unknown"]
    port vbr_04a0be3b4b247000-local-router
        type: router
        router-port: vbr_04a0be3b4b247000-route-local
switch 80573916-6550-4a34-8570-bc67caad05cf (vpc_04afbc45e9847000-vbr)
    port vpc_04afbc45e9847000-join-port
        type: router
        router-port: vpc_04afbc45e9847000-vbr-port
switch 74a43bfe-79ea-4400-b02e-3bd7043e6dc0 (subnet_049e37f43f047000)
    port subnet_049e37f43f047000-router
        type: router
        router-port: vpc_049e374424847000-subnet_049e37f43f047000
    port i_04b0bdce8661c000
        addresses: ["AE:91:91:13:D9:FE 10.199.100.12/24"]
    port i_04b0bdcbade1c000
        addresses: ["AE:BE:BE:6B:0B:83 10.199.100.11/24"]
    port i_04aacda5ceb47000
        addresses: ["00:00:19:91:00:10 10.199.100.10/24"]
switch 5832950f-ec29-42e9-ad37-09dd194638ce (vbr_04a0be3427447000-local)
    port vbr_04a0be3427447000-local-network
        type: localnet
        addresses: ["unknown"]
    port vbr_04a0be3427447000-local-router
        type: router
        router-port: vbr_04a0be3427447000-route-local
switch 9e5e34e3-259f-4078-a833-6efa09ed6d15 (vpc_049e374424847000-vbr)
    port ri_04a0be3cb4347000
        type: router
        router-port: vbr_04a0be3427447000-vpc_049e374424847000
    port vpc_049e374424847000-join-port
        type: router
        router-port: vpc_049e374424847000-vbr-port
    port ri_04a0be4962547000
        type: router
        router-port: vbr_04a0be3b4b247000-vpc_049e374424847000
router bd40b34c-8cf1-46a1-8dd9-9d5ef42e07a2 (vpc_04a7d57200347000)
    port vpc_04a7d57200347000-pcc_04aacd2929547000
        mac: "AE:56:56:A2:49:B2"
        networks: ["10.31.0.2/24"]
        route-table: vtb_04a7d57200347000
    port vpc_04a7d57200347000-vbr-port
        mac: "AE:EF:EF:DB:2F:F3"
        networks: ["172.16.255.2/24"]
        route-table: vtb_04a7d57200347000
router 1ea16498-bf2b-4182-90e2-64bab54bd1fb (vbr_04a0be3427447000)
    port vbr_04a0be3427447000-vpc_04a7d57200347000
        mac: "AE:DD:DD:E4:9E:58"
        networks: ["172.16.255.3/24"]
    port vbr_04a0be3427447000-route-local
        mac: "AE:A9:A9:0C:6F:C8"
        networks: ["172.31.5.230/24"]
    port vbr_04a0be3427447000-vpc_049e374424847000
        mac: "AE:CD:CD:04:D7:D4"
        networks: ["172.16.255.3/24"]
    nat 9bd37dcd-2a01-4ff6-9089-bce2558fc5b9
        external ip: "123.123.123.123"
        logical ip: "10.199.100.0/24"
        type: "snat"
router 0ef0e9c1-4352-4ba0-9002-772f6cc7366b (vpc_049e374424847000)
    port vpc_049e374424847000-pcc_04aacd2929547000
        mac: "AE:56:56:A2:49:B1"
        networks: ["10.31.0.1/24"]
        route-table: vtb_049e374424847000
    port vpc_049e374424847000-subnet_049e37f43f047000
        mac: "AE:95:95:65:27:A6"
        networks: ["10.199.100.1/24"]
        route-table: vtb_049e374424847000
    port vpc_049e374424847000-subnet_04b62bbe1f147000
        mac: "AE:8D:8D:55:BA:26"
        networks: ["10.199.200.1/24"]
        route-table: vtb_049e374424847000
    port vpc_049e374424847000-vbr-port
        mac: "AE:03:03:17:EB:DF"
        networks: ["172.16.255.2/24"]
        route-table: vtb_049e374424847000
router db24641e-7597-47fc-81d9-de2ec15ca016 (vpc_04afbc45e9847000)
    port vpc_04afbc45e9847000-vbr-port
        mac: "AE:BC:BC:4F:98:7C"
        networks: ["172.16.255.2/24"]

[root@newcentral ~]# ovn-nbctl show vbr_04a0be3427447000
router 1ea16498-bf2b-4182-90e2-64bab54bd1fb (vbr_04a0be3427447000)
    port vbr_04a0be3427447000-vpc_04a7d57200347000
        mac: "AE:DD:DD:E4:9E:58"
        networks: ["172.16.255.3/24"]
    port vbr_04a0be3427447000-route-local
        mac: "AE:A9:A9:0C:6F:C8"
        networks: ["172.31.5.230/24"]
    port vbr_04a0be3427447000-vpc_049e374424847000
        mac: "AE:CD:CD:04:D7:D4"
        networks: ["172.16.255.3/24"]
    nat 9bd37dcd-2a01-4ff6-9089-bce2558fc5b9
        external ip: "123.123.123.123"
        logical ip: "10.199.100.0/24"
        type: "snat"

#NAT table
[root@newcentral ~]# ovn-nbctl list nat
_uuid               : 9bd37dcd-2a01-4ff6-9089-bce2558fc5b9
allowed_ext_ips     : []
exempted_ext_ips    : []
external_ids        : {}
external_ip         : "123.123.123.123"
external_mac        : []
external_port_range : ""
gateway_port        : []
logical_ip          : "10.199.100.0/24"
logical_port        : []
options             : {stateless="false"}
type                : snat

#Logical_router table
[root@newcentral ~]# ovn-nbctl find logical_router name="vbr_04a0be3427447000"
_uuid               : 1ea16498-bf2b-4182-90e2-64bab54bd1fb
copp                : []
enabled             : []
external_ids        : {vendor=sina}
load_balancer       : []
load_balancer_group : []
name                : vbr_04a0be3427447000
nat                 : [9bd37dcd-2a01-4ff6-9089-bce2558fc5b9]
options             : {chassis=gateway01}
policies            : []
ports               : [52a9cb30-1a9f-4855-95a7-f7bc5d741d82, 5bcf8982-f784-405c-8ada-ef73bd57c4a6, 77b73cdb-8c91-4221-8fac-4253ac5a0e2d]
static_routes       : [3919f68b-cc08-46c2-95c0-93b4d5305e44, 3a71b83b-1333-447f-a3d3-84ec0483fc96, 437f7eaa-4fe5-41f4-817e-30e09ca8228a, 90322d85-0f43-4cc5-9916-60000d620da5]

[root@newcentral ~]# ovn-sbctl show 
Chassis gateway01
    hostname: gateway01
    Encap geneve
        ip: "172.31.5.23"
        options: {csum="true"}
    Port_Binding vbr_04a0be3427447000-vpc_04a7d57200347000
    Port_Binding vbr_04a0be3427447000-route-local
    Port_Binding vbr_04a0be3427447000-vpc_049e374424847000
    Port_Binding ri_04aacfb978547000
    Port_Binding vbr_04a0be3427447000-local-router
    Port_Binding ri_04a0be3cb4347000
Chassis node01
    hostname: node01
    Encap geneve
        ip: "172.31.1.21"
        options: {csum="true"}
    Port_Binding i_04aacda5ceb47000

Some information about the gateway01 chassis is listed below

[root@gateway01 ~]# ovs-vsctl show 
ad701a9e-a75c-486f-870d-1d58605d28df
    Bridge br-ex
        Port ens1f0
            Interface ens1f0
        Port patch-vbr_04a0be3427447000-local-network-to-br-int
            Interface patch-vbr_04a0be3427447000-local-network-to-br-int
                type: patch
                options: {peer=patch-br-int-to-vbr_04a0be3427447000-local-network}
        Port br-ex
            Interface br-ex
                type: internal
    Bridge br-int
        fail_mode: secure
        datapath_type: system
        Port ovn-node01-0
            Interface ovn-node01-0
                type: geneve
                options: {csum="true", key=flow, remote_ip="172.31.1.21"}
        Port patch-br-int-to-vbr_04a0be3427447000-local-network
            Interface patch-br-int-to-vbr_04a0be3427447000-local-network
                type: patch
                options: {peer=patch-vbr_04a0be3427447000-local-network-to-br-int}
        Port br-int
            Interface br-int
                type: internal
    ovs_version: "2.17.7-e054917"

[root@gateway01 ~]# ovs-vsctl list openvswitch
ovs-vsctl: unknown table "openvswitch"
[root@gateway01 ~]# ovs-vsctl list open_vswitch
_uuid               : ad701a9e-a75c-486f-870d-1d58605d28df
bridges             : [03f31940-ceab-4cb5-a2a9-b25071509d2c, 6e0703d0-192c-4178-af00-c2aff8e7ea55]
cur_cfg             : 17
datapath_types      : [netdev, system]
datapaths           : {system=f9d52158-c861-4cb4-8cfa-acfd26440534}
db_version          : "8.3.1"
doca_initialized    : false
doca_version        : none
dpdk_initialized    : false
dpdk_version        : "MLNX_DPDK 22.11.1.4.2"
external_ids        : {hostname=gateway01, ovn-bridge-mappings="physnet1:br-ex", ovn-encap-ip="172.31.5.23", ovn-encap-type=geneve, ovn-remote="tcp:10.185.11.71:6642", rundir="/var/run/openvswitch", system-id=gateway01}
iface_types         : [bareudp, erspan, geneve, gre, gtpu, internal, ip6erspan, ip6gre, lisp, patch, stt, system, tap, vxlan]
manager_options     : []
next_cfg            : 17
other_config        : {hw-offload="true", ovn-chassis-idx-gateway01="", vlan-limit="0"}
ovs_version         : "2.17.7-e054917"
ssl                 : []
statistics          : {}
system_type         : almalinux
system_version      : "9.2"

[root@gateway01 ~]# ethtool -i ens1f0
driver: mlx5_core
version: 23.04-1.1.3
firmware-version: 22.37.1014 (MT_0000000359)
expansion-rom-version: 
bus-info: 0000:02:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes

[root@gateway01 ~]# ethtool -k ens1f0 | grep offload
tcp-segmentation-offload: on
generic-segmentation-offload: on
generic-receive-offload: on
large-receive-offload: off
rx-vlan-offload: off [requested on]
tx-vlan-offload: on
l2-fwd-offload: off [fixed]
hw-tc-offload: on
esp-hw-offload: off [fixed]
esp-tx-csum-hw-offload: off [fixed]
rx-udp_tunnel-port-offload: on
tls-hw-tx-offload: off [fixed]
tls-hw-rx-offload: off [fixed]
macsec-hw-offload: off [fixed]
hsr-tag-ins-offload: off [fixed]
hsr-tag-rm-offload: off [fixed]
hsr-fwd-offload: off [fixed]
hsr-dup-offload: off [fixed]

10.199.100.10 -> 172.31.6.24 (snat(src=123.123.123.123))

1.With the iperf3 test, there seems to be no problem, all flows are unloaded normally, and the traffic is as expected.

#iperf3 test
[root@gateway02 ~]# iperf3 -s
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
Accepted connection from 123.123.123.123, port 56014
[  5] local 172.31.6.24 port 5201 connected to 123.123.123.123 port 56022
[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec  2.48 GBytes  21.3 Gbits/sec                  
[  5]   1.00-2.00   sec  2.61 GBytes  22.4 Gbits/sec                  
[  5]   2.00-3.00   sec  2.61 GBytes  22.4 Gbits/sec                  
[  5]   3.00-4.00   sec  2.61 GBytes  22.4 Gbits/sec                  
[  5]   4.00-5.00   sec  2.61 GBytes  22.5 Gbits/sec                  
[  5]   5.00-6.00   sec  2.61 GBytes  22.5 Gbits/sec                  
[  5]   6.00-7.00   sec  2.61 GBytes  22.5 Gbits/sec                  
[  5]   7.00-8.00   sec  2.61 GBytes  22.5 Gbits/sec                  
[  5]   8.00-9.00   sec  2.61 GBytes  22.4 Gbits/sec                  
[  5]   9.00-10.00  sec  2.61 GBytes  22.4 Gbits/sec         

[root@node01 ~]# ip netns exec sw1-vm1 iperf3 -c 172.31.6.24 -i 1 -t 1000
Connecting to host 172.31.6.24, port 5201
[  5] local 10.199.100.10 port 56022 connected to 172.31.6.24 port 5201
[ ID] Interval           Transfer     Bitrate         Retr  Cwnd
[  5]   0.00-1.00   sec  2.59 GBytes  22.2 Gbits/sec   59   1.00 MBytes       
[  5]   1.00-2.00   sec  2.61 GBytes  22.4 Gbits/sec  103    994 KBytes       
[  5]   2.00-3.00   sec  2.61 GBytes  22.4 Gbits/sec   59    937 KBytes       
[  5]   3.00-4.00   sec  2.61 GBytes  22.4 Gbits/sec   69    957 KBytes       
[  5]   4.00-5.00   sec  2.61 GBytes  22.5 Gbits/sec   29   1002 KBytes       
[  5]   5.00-6.00   sec  2.61 GBytes  22.4 Gbits/sec   40   1.04 MBytes       
[  5]   6.00-7.00   sec  2.61 GBytes  22.5 Gbits/sec   18    756 KBytes      
#View the flow of hardware offload in ovs:
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows type=offloaded  
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:8546650, bytes:12084949738, used:0.980s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x7d)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0x7d),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:8546646, bytes:12084948109, used:0.980s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x7e)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0x7e),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:8546645, bytes:12084946695, used:0.980s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:1615469836, bytes:2384348244165, used:0.980s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:159918, bytes:11197872, used:0.980s, actions:ct(zone=16,nat),recirc(0x7f)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0x7f),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:159918, bytes:11197872, used:0.980s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x80)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0x80),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:159918, bytes:11197872, used:0.980s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:37541599, bytes:4654672765, used:0.980s, actions:2

#View details
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows -m
ufid:5ac63871-658b-4b25-8e56-217768575a16, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),skb_mark(0/0),ct_state(0/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:25590231, bytes:36184573272, used:0.180s, offloaded:yes, dp:tc, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x7d)
ufid:af46cfe3-bc62-4f74-ab05-d449c2e50bfa, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x7d),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:25590227, bytes:36184571643, used:0.180s, offloaded:yes, dp:tc, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x7e)
ufid:f219f29b-222c-4f58-a1cf-a6bcf43af76f, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x7e),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:25590227, bytes:36184571643, used:0.180s, offloaded:yes, dp:tc, actions:ct_clear,ens1f0
ufid:664eb157-1281-450f-8aa0-9ab469591e34, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:1632513421, bytes:2409504570083, used:0.180s, offloaded:yes, dp:tc, actions:br-ex
ufid:a25c1957-775d-4fcf-91d4-06a2675e4a9c, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x21),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:480973, bytes:33677870, used:0.180s, offloaded:yes, dp:tc, actions:ct(zone=16,nat),recirc(0x7f)
ufid:87c65051-3be0-40da-9f68-b8df8102abe6, skb_priority(0/0),skb_mark(0/0),ct_state(0x20/0x30),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x7f),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=10.199.100.0/255.255.255.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:480973, bytes:33677870, used:0.180s, offloaded:yes, dp:tc, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x80)
ufid:ee212541-e807-4efc-8119-1c0f4b0b9671, skb_priority(0/0),skb_mark(0/0),ct_state(0x28/0x39),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x80),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no),tcp(src=0/0,dst=0/0), packets:480973, bytes:33677870, used:0.180s, offloaded:yes, dp:tc, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),genev_sys_6081
ufid:01d285c1-2635-4fd8-8738-f8df0173163f, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:37862656, bytes:4694489941, used:0.180s, offloaded:yes, dp:tc, actions:ens1f0
ufid:34a47ff4-1e36-41cd-953e-909b51e5bf79, recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(ens1f0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:51895, bytes:6175505, used:0.233s, dp:ovs, actions:drop

#View the flow of fastpath：
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows 
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:29841401, bytes:42195727652, used:0.000s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x7d)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0x7d),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:29841397, bytes:42195726023, used:0.000s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x7e)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0x7e),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:29841397, bytes:42195726023, used:0.000s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:1636764592, bytes:2415779298479, used:0.000s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:561852, bytes:39342064, used:0.000s, actions:ct(zone=16,nat),recirc(0x7f)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0x7f),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:561852, bytes:39342064, used:0.000s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x80)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0x80),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:561852, bytes:39342064, used:0.000s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:37943535, bytes:4704521601, used:0.000s, actions:2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:51896, bytes:6175624, used:0.100s, actions:drop
#View the conntrack：
[root@gateway01 ~]$ ovs-appctl dpctl/dump-conntrack | wc -l
4
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack 
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=54346,dport=5201),reply=(src=172.31.6.24,dst=10.199.100.10,sport=5201,dport=54346),zone=6
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=50604,dport=5201),reply=(src=172.31.6.24,dst=123.123.123.123,sport=5201,dport=50604),zone=16
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=50604,dport=5201),reply=(src=172.31.6.24,dst=10.199.100.10,sport=5201,dport=50604),zone=6
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=54346,dport=5201),reply=(src=172.31.6.24,dst=123.123.123.123,sport=5201,dport=54346),zone=16
#View the traffic:
[root@gateway01 ~]# sar -n DEV 1
Linux 5.14.0-284.11.1.el9_2.x86_64 (gateway01)  08/04/2023      _x86_64_        (48 CPU)

11:56:54 PM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s   %ifutil
11:56:55 PM        lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM  enp8s0f0      1.00      1.00      0.06      0.18      0.00      0.00      0.00      0.00
11:56:55 PM  enp8s0f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM    ens2f0      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM  enp8s0f2      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM    ens2f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM  enp8s0f3      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM    ens1f0 1394058.00 1394071.00 1973751.61 1897756.30      0.00      0.00      0.00     16.17
11:56:55 PM    ens1f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM ovs-system      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM     br-ex      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM    br-int      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
11:56:55 PM genev_sys_6081      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00

result:Use iperf3 to test that everything is OK.

2.Using dperf to test udp packets, it was found that some flows were not offloaded, and the traffic did not meet the expected value. The CPU occupied 100% of the ksoftirqd process

#dperf test
[root@node01 ~]# ip netns exec sw1-vm1 ./dperf.x -c dperf.conf

seconds 18                 cpuUsage 7   7   7   7   7   7   
pktRx   0                  pktTx    5,000,000          bitsRx   0                  bitsTx  4,240,000,000      dropTx  0                 
tcpRx   0                  tcpTx    0                  udpRx    0                  udpTx   5,000,000         
arpRx   0                  arpTx    0                  icmpRx   0                  icmpTx  0                 
tosRx   0                  otherRx  0                  badRx    0                 
udpRt   5,000,000          udpDrop  0                  tcpDrop  0                 
skOpen  0                  skClose  0                  skCon    5,000              skErr   0                 
ierrors 0                  oerrors  0                  imissed  0                 

seconds 19                 cpuUsage 7   7   7   7   7   7   
pktRx   0                  pktTx    5,000,084          bitsRx   0                  bitsTx  4,240,071,232      dropTx  0                 
tcpRx   0                  tcpTx    0                  udpRx    0                  udpTx   5,000,084         
arpRx   0                  arpTx    0                  icmpRx   0                  icmpTx  0                 
tosRx   0                  otherRx  0                  badRx    0                 
udpRt   5,000,084          udpDrop  0                  tcpDrop  0                 
skOpen  0                  skClose  0                  skCon    5,000              skErr   0                
ierrors 0                  oerrors  0                  imissed  0                 

#View the traffic:
[root@gateway01 ~]# sar -n DEV 1
Linux 5.14.0-284.11.1.el9_2.x86_64 (gateway01)  08/05/2023      _x86_64_        (48 CPU)

12:07:23 AM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s   %ifutil
12:07:24 AM        lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM  enp8s0f0      2.00      1.00      0.12      0.18      0.00      0.00      0.00      0.00
12:07:24 AM  enp8s0f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM    ens2f0      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM  enp8s0f2      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM    ens2f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM  enp8s0f3      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM    ens1f0 4920514.00 167186.00 807275.69  17959.43      0.00      0.00      1.00      6.61
12:07:24 AM    ens1f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM ovs-system      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM     br-ex      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM    br-int      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:24 AM genev_sys_6081      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00

12:07:24 AM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s   %ifutil
12:07:25 AM        lo      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM  enp8s0f0      6.00     13.00      0.38      7.21      0.00      0.00      0.00      0.01
12:07:25 AM  enp8s0f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM    ens2f0      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM  enp8s0f2      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM    ens2f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM  enp8s0f3      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM    ens1f0 5001917.00 170087.00 820623.04  18270.96      0.00      0.00      0.00      6.72
12:07:25 AM    ens1f1      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM ovs-system      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM     br-ex      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM    br-int      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
12:07:25 AM genev_sys_6081      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00

#View NIC traffic
[root@gateway01 ~]# ifconfig ens1f0
ens1f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        ether e8:eb:d3:3a:d7:f8  txqueuelen 1000  (Ethernet)
        RX packets 8797857529  bytes 5322858965370 (4.8 TiB)
        RX errors 0  dropped 1219084890  overruns 0  frame 0
        TX packets 3279243046  bytes 4213255342903 (3.8 TiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@gateway01 ~]# ifconfig ens1f0
ens1f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        ether e8:eb:d3:3a:d7:f8  txqueuelen 1000  (Ethernet)
        RX packets 8806548992  bytes 5324319131049 (4.8 TiB)
        RX errors 0  dropped 1227480644  overruns 0  frame 0
        TX packets 3279538578  bytes 4213287851313 (3.8 TiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

#View process
[root@gateway01 ~]# top
top - 00:14:38 up 2 days,  1:49,  1 user,  load average: 0.49, 0.29, 0.18
Tasks: 515 total,   2 running, 513 sleeping,   0 stopped,   0 zombie
%Cpu(s):  0.0 us,  0.0 sy,  0.0 ni, 97.9 id,  0.0 wa,  0.0 hi,  2.1 si,  0.0 st
MiB Mem : 128272.0 total, 125688.7 free,   2726.4 used,    818.1 buff/cache
MiB Swap:   4096.0 total,   4096.0 free,      0.0 used. 125545.6 avail Mem 

    PID USER      PR  NI    VIRT    RES    SHR S  %CPU  %MEM     TIME+ COMMAND                                                 
    166 root      20   0       0      0      0 R 100.0   0.0  22:53.12 ksoftirqd/29                                            
  33054 openvsw+  10 -10 4350096 291584  25892 S   1.0   0.2  23:26.05 ovs-vswitchd                                            
  37613 root      20   0   10912   4540   3376 R   1.0   0.0   0:00.08 top                                                     
      1 root      20   0  174004  18332  10924 S   0.0   0.0   0:06.86 systemd                                                 
      2 root      20   0       0      0      0 S   0.0   0.0   0:00.12 kthreadd                                                
      3 root       0 -20       0      0      0 I   0.0   0.0   0:00.00 rcu_gp                                                  
      4 root       0 -20       0      0      0 I   0.0   0.0   0:00.00 rcu_par_gp

#View the flow of hardware offload in ovs:
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows type=offloaded  
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=17,ttl=63,frag=no), packets:16764463, bytes:1542330596, used:0.000s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x85)
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:2491914653, bytes:3074457645572, used:0.640s, actions:1
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:5, bytes:350, used:0.730s, actions:2

#View details
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows -m
ufid:8ec2774d-8efe-4b14-a8b5-c53fef16e920, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),skb_mark(0/0),ct_state(0/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=17,tos=0/0,ttl=63,frag=no),udp(src=0/0,dst=0/0), packets:17616859, bytes:1620751028, used:0.000s, offloaded:yes, dp:tc, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x85)
ufid:56740c84-edb0-4037-91de-7216907857ec, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x85),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:17616825, bytes:1620747900, used:0.000s, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x89)
ufid:9ee0daa2-d3eb-4dfe-9231-15bbbac87ded, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x89),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=17,tos=0/0,ttl=0/0,frag=no),udp(src=0/0,dst=0/0), packets:17616828, bytes:1620748176, used:0.000s, dp:tc, actions:ct_clear,ens1f0
ufid:664eb157-1281-450f-8aa0-9ab469591e34, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:2519113731, bytes:3079027090586, used:0.200s, offloaded:yes, dp:tc, actions:br-ex
ufid:32a63a03-7bc5-4f97-b418-5a593875f363, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:5, bytes:350, used:5.730s, offloaded:yes, dp:tc, actions:ens1f0
ufid:4232db89-e385-416f-bdfa-adf82ea5a5dc, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, offloaded:yes, dp:tc, actions:drop
ufid:34a47ff4-1e36-41cd-953e-909b51e5bf79, recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(ens1f0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:0, bytes:0, used:never, dp:ovs, actions:drop

#View the flow of fastpath：
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows 
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=17,ttl=63,frag=no), packets:17894519, bytes:1646295748, used:0.000s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x85)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-rpl+trk),recirc_id(0x85),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:17894486, bytes:1646292712, used:0.000s, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x89)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-inv+trk),ct_mark(0/0x2),recirc_id(0x89),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=17,frag=no), packets:17894488, bytes:1646292896, used:0.000s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:2524233471, bytes:3079887206906, used:0.810s, actions:1
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:5, bytes:350, used:7.360s, actions:2
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, actions:drop
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:0, bytes:0, used:never, actions:drop
#View the conntrack：
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack  | wc -l
10000
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack  | wc -l
10000
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack  | wc -l
10000
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | less
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2391,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=2391),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3512,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3512),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3351,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3351),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5266,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5266),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5273,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5273),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3268,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=3268),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6768,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=6768),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5026,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5026),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3743,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3743),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6191,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6191),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4161,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=4161),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4525,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4525),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5846,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5846),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4898,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4898),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3361,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3361),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5677,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5677),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5285,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5285),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2536,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=2536),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6255,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6255),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2391,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=2391),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3512,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3512),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3351,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3351),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5266,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5266),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5273,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5273),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3268,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=3268),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6768,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=6768),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5026,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5026),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3743,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3743),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6191,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6191),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4161,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=4161),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4525,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4525),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5846,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5846),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4898,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4898),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3361,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3361),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5677,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5677),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5285,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5285),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2536,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=2536),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6255,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6255),zone=16
…………………………

#View vswitchd log
[root@gateway01 ~]# tail -f /var/log/openvswitch/ovs-vswitchd.log
2023-08-04T16:56:12.633Z|00051|bridge|INFO|bridge br-int: added interface br-int on port 65534
2023-08-04T16:56:12.633Z|00052|bridge|INFO|bridge br-ex: using datapath ID 0000e8ebd33ad7f8
2023-08-04T16:56:12.634Z|00053|connmgr|INFO|br-ex: added service controller "punix:/var/run/openvswitch/br-ex.mgmt"
2023-08-04T16:56:12.634Z|00054|bridge|INFO|bridge br-int: using datapath ID 0000e2027b129345
2023-08-04T16:56:12.634Z|00055|connmgr|INFO|br-int: added service controller "punix:/var/run/openvswitch/br-int.mgmt"
2023-08-04T16:56:12.634Z|00056|dpif|WARN|system@ovs-system: failed to query port patch-vbr_04a0be3427447000-local-network-to-br-int: Invalid argument
2023-08-04T16:56:12.636Z|00057|bridge|INFO|ovs-vswitchd (Open vSwitch) 2.17.7-e054917
2023-08-04T16:56:12.727Z|00058|netdev_offload_tc|INFO|added ingress qdisc to ovn-node01-0
2023-08-04T16:56:12.727Z|00059|netdev_offload|INFO|ovn-node01-0: Assigned flow API 'linux_tc'.
2023-08-04T16:56:12.727Z|00060|bridge|INFO|bridge br-int: added interface ovn-node01-0 on port 1
2023-08-04T16:56:22.639Z|00061|memory|INFO|285644 kB peak resident set size after 10.1 seconds
2023-08-04T16:56:22.639Z|00062|memory|INFO|handlers:48 idl-cells:413 ofconns:3 ports:6 revalidators:13 rules:1154 udpif keys:3
2023-08-04T16:56:22.755Z|00063|connmgr|INFO|br-int<->unix#1: 1148 flow_mods 10 s ago (1146 adds, 1 deletes, 1 modifications)

result:Using dperf to send udp packets indicates that there is a hardware offload problem on the gateway.There's nothing unusual about the vswitch log(The log level is info),I adjusted the log level to dbg,still no exception logs were found.

According to the flow table, the two flows are not offloaded.But I don't know why.

ufid:56740c84-edb0-4037-91de-7216907857ec, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x85),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:17616825, bytes:1620747900, used:0.000s, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x89)
ufid:9ee0daa2-d3eb-4dfe-9231-15bbbac87ded, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x89),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=17,tos=0/0,ttl=0/0,frag=no),udp(src=0/0,dst=0/0), packets:17616828, bytes:1620748176, used:0.000s, dp:tc, actions:ct_clear,ens1f0

3.Testing tcp packets with telnet found that some streams were not offloaded.

#telnet test
[root@node01 ~]# ip netns exec sw1-vm1 telnet  172.31.6.1 22
Trying 172.31.6.1...
Connected to 172.31.6.1.
Escape character is '^]'.
SSH-2.0-Comware-7.1.070

#View the flow of hardware offload in ovs:
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows type=offloaded  
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:1, bytes:52, used:0.830s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd0)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0xd0),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:1, bytes:52, used:0.820s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0xd1),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:1, bytes:52, used:0.820s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:3201892454, bytes:3193734234632, used:1.270s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:0.820s, actions:ct(zone=16,nat),recirc(0xd2)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0xd2),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:0.820s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd3)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0xd3),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:1, bytes:73, used:0.820s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:153, bytes:23326, used:0.820s, actions:2

#View details
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows -m
ufid:679aa52d-f823-4bea-9a72-751ee9625071, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),skb_mark(0/0),ct_state(0/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:2, bytes:118, used:2.780s, offloaded:yes, dp:tc, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd0)
ufid:52c65ea7-05f1-4f4c-bc3e-c2586fb06ead, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0xd0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:0, bytes:0, used:never, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
ufid:25b73b7b-054f-4287-9592-b89bd8275e1c, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0xd0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:1, bytes:52, used:3.750s, offloaded:yes, dp:tc, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
ufid:2d494a14-795b-469e-9854-df1d902c5b58, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd1),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:0, bytes:0, used:never, dp:tc, actions:ct_clear,ens1f0
ufid:1b87eaad-5b01-4b96-b46f-ad4bcc0507d0, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd1),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:52, used:3.750s, offloaded:yes, dp:tc, actions:ct_clear,ens1f0
ufid:664eb157-1281-450f-8aa0-9ab469591e34, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:3201892457, bytes:3193734235024, used:2.780s, offloaded:yes, dp:tc, actions:br-ex
ufid:25d2cb6b-d5be-4342-8a99-187d4e191122, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x21),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:73, used:3.750s, offloaded:yes, dp:tc, actions:ct(zone=16,nat),recirc(0xd2)
ufid:66c9fa76-be97-49eb-8851-09acd554a600, skb_priority(0/0),skb_mark(0/0),ct_state(0x20/0x30),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd2),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=10.199.100.0/255.255.255.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:73, used:3.750s, offloaded:yes, dp:tc, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd3)
ufid:be3e928d-e9c2-499f-a73b-a5251d27e861, skb_priority(0/0),skb_mark(0/0),ct_state(0x28/0x39),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd3),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:73, used:3.750s, offloaded:yes, dp:tc, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),genev_sys_6081
ufid:17c5fac7-b52f-4d48-ad30-f10a9fe6f028, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:153, bytes:23326, used:3.750s, offloaded:yes, dp:tc, actions:ens1f0
ufid:4232db89-e385-416f-bdfa-adf82ea5a5dc, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, offloaded:yes, dp:tc, actions:drop
ufid:34a47ff4-1e36-41cd-953e-909b51e5bf79, recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(ens1f0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:200, bytes:23800, used:0.276s, dp:ovs, actions:drop

#View the flow of fastpath：
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows 
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:2, bytes:118, used:4.430s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd0)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-rpl+trk),recirc_id(0xd0),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:0, bytes:0, used:never, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0xd0),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:1, bytes:52, used:5.390s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-inv+trk),ct_mark(0/0x2),recirc_id(0xd1),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:0, bytes:0, used:never, actions:ct_clear,2
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0xd1),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:1, bytes:52, used:5.390s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:3201892459, bytes:3193734235205, used:1.370s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:5.390s, actions:ct(zone=16,nat),recirc(0xd2)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0xd2),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:5.390s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd3)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0xd3),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:1, bytes:73, used:5.390s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:154, bytes:23430, used:1.560s, actions:2
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, actions:drop
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:200, bytes:23800, used:1.920s, actions:drop
#View the conntrack：
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack 
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=47190,dport=22),reply=(src=172.31.6.24,dst=10.199.100.10,sport=22,dport=47190),zone=6
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=47190,dport=22),reply=(src=172.31.6.24,dst=123.123.123.123,sport=22,dport=47190),zone=16

[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | wc -l
2

result:Testing with telnet,I think hardware offload is normal.But there are two flow that are never used and not offloaded.

According to the flow table, the two flow are not offloaded.But these two flow do not seem to be used(used:never),I don't know if this access is a normal offload,Just from the flow table of type=offload the offload is normal.

ufid:52c65ea7-05f1-4f4c-bc3e-c2586fb06ead, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0xd0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:0, bytes:0, used:never, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)

ufid:2d494a14-795b-469e-9854-df1d902c5b58, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd1),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:0, bytes:0, used:never, dp:tc, actions:ct_clear,ens1f0

I did not test the case of dperf sending tcp packets, I will add that after the subsequent tests. The question now is why are there some flows that cannot be offload in the second and third cases.What do I need to do to offload these flows.

I really hope to get your help. If you need any information, I will supplement it.The attachment is the result of the ovs-ofctl dump-flows br-int command

slowpath.txt

wanghuiheze commented 1 year ago

@marceloleitner Do you have any good suggestions or opinions,Thank you!

dceara commented 10 months ago

@wanghuiheze Sorry for the delayed reply but all flows that are not offloaded have ct_state(0x21/0x29). That translates to ct_state=+trk+new:

#define OVS_CS_F_NEW               0x01 /* Beginning of a new connection. */
#define OVS_CS_F_ESTABLISHED       0x02 /* Part of an existing connection. */
#define OVS_CS_F_RELATED           0x04 /* Related to an established
                     * connection. */

I think it's expected that they're not offloaded.

marceloleitner commented 10 months ago

Yup, sorry the delay as well. I can confirm Dumitru's expectations: +trk+new are NOT offloadable.

dceara commented 10 months ago

Thanks for the confirmation @marceloleitner! Closing this for now.