Mellanox cx5 网卡，FULLNAT 不通

[autumn0207]

现有一块 mellanox cx5 单网口的网卡，使用 dpvs 作 fullnat 不通，命令如下：

# add vlan sub interface
./dpip vlan add link dpdk0 id 3

# add VIP to WAN interface
./dpip addr add 10.251.20.9/32 dev dpdk0

# route for WAN/LAN access
# add routes for other network or default route if needed.
./dpip route add 10.251.20.9/22 dev dpdk0
./dpip route add 10.251.223.5/17 dev dpdk0.3

# add service <VIP:vport> to forwarding, scheduling mode is RR.
./ipvsadm -A -t 10.251.20.9:80 -s rr

# add two RS for service, forwarding mode is FNAT (-b)
./ipvsadm -a -t 10.251.20.9:80 -r 10.251.128.63 -b
./ipvsadm -a -t 10.251.20.9:80 -r 10.251.223.189 -b

# add at least one Local-IP (LIP) for FNAT on LAN interface
./ipvsadm --add-laddr -z 10.251.223.5 -t 10.251.20.9:80 -F dpdk0.3

rs 上收到了并回复了 SYN包，但是 dpvs 似乎没处理，会不停地 SYN_SENT，dpvs.conf 如下：

! This is dpvs default configuration file.
!
! The attribute "<init>" denotes the configuration item at initialization stage. Item of
! this type is configured oneshoot and not reloadable. If invalid value configured in the
! file, dpvs would use its default value.
!
! Note that dpvs configuration file supports the following comment type:
!   * line comment: using '#" or '!'
!   * inline range comment: using '<' and '>', put comment in between
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

! global config
global_defs {
    log_level   DEBUG
    log_file    /var/log/dpvs.log
    ! log_async_mode    on
}

! netif config
netif_defs {
    <init> pktpool_size     524287
    <init> pktpool_cache    256

    <init> device dpdk0 {
        rx {
            queue_number        8
            descriptor_number   1024
            rss                 all
        }
        tx {
            queue_number        8
            descriptor_number   1024
        }
        fdir {
            mode                perfect
            pballoc             64k
            status              matched
        }
        ! promisc_mode
        kni_name                dpdk0.kni
    }
}

! worker config (lcores)
worker_defs {
    <init> worker cpu0 {
        type    master
        cpu_id  0
    }

    <init> worker cpu1 {
        type    slave
        cpu_id  1
        port    dpdk0 {
            rx_queue_ids     0
            tx_queue_ids     0
            ! isol_rx_cpu_ids  9
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker cpu2 {
        type    slave
        cpu_id  2
        port    dpdk0 {
            rx_queue_ids     1
            tx_queue_ids     1
            ! isol_rx_cpu_ids  10
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker cpu3 {
        type        slave
        cpu_id      3
        port        dpdk0 {
            rx_queue_ids     2
            tx_queue_ids     2
            ! isol_rx_cpu_ids  11
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker   cpu4 {
        type        slave
        cpu_id      4
        port        dpdk0 {
            rx_queue_ids     3
            tx_queue_ids     3
            ! isol_rx_cpu_ids  12
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker   cpu5 {
        type        slave
        cpu_id      5
        port        dpdk0 {
            rx_queue_ids     4
            tx_queue_ids     4
            ! isol_rx_cpu_ids  13
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker   cpu6 {
        type        slave
        cpu_id      6
        port        dpdk0 {
            rx_queue_ids     5
            tx_queue_ids     5
            ! isol_rx_cpu_ids  14
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker   cpu7 {
        type        slave
        cpu_id      7
        port        dpdk0 {
            rx_queue_ids     6
            tx_queue_ids     6
            ! isol_rx_cpu_ids  15
            ! isol_rxq_ring_sz 1048576
        }
    }

    <init> worker   cpu8 {
        type        slave
        cpu_id      8
        port        dpdk0 {
            rx_queue_ids     7
            tx_queue_ids     7
            ! isol_rx_cpu_ids  16
            ! isol_rxq_ring_sz 1048576
        }
    }

}

! timer config
timer_defs {
    # cpu job loops to schedule dpdk timer management
    schedule_interval    500
}

! dpvs neighbor config
neigh_defs {
    <init> unres_queue_length  128
    timeout                    60
}

! dpvs ipv4 config
ipv4_defs {
    forwarding                 off
    <init> default_ttl         64
    fragment {
        <init> bucket_number   4096
        <init> bucket_entries  16
        <init> max_entries     4096
        <init> ttl             1
    }
}

! dpvs ipv6 config
ipv6_defs {
    disable                     off
    forwarding                  off
    route6 {
        <init> method           hlist
        recycle_time            10
    }
}

! control plane config
ctrl_defs {
    lcore_msg {
        <init> ring_size                4096
        sync_msg_timeout_us             20000
        priority_level                  low
    }
    ipc_msg {
        <init> unix_domain /var/run/dpvs_ctrl
    }
}

! ipvs config
ipvs_defs {
    conn {
        <init> conn_pool_size       2097152
        <init> conn_pool_cache      256
        conn_init_timeout           3
        ! expire_quiescent_template
        ! fast_xmit_close
        ! <init> redirect           off
    }

    udp {
        ! defence_udp_drop
        uoa_mode        opp
        uoa_max_trail   3
        timeout {
            normal      300
            last        3
        }
    }

    tcp {
        ! defence_tcp_drop
        timeout {
            none        2
            established 90
            syn_sent    3
            syn_recv    30
            fin_wait    7
            time_wait   7
            close       3
            close_wait  7
            last_ack    7
            listen      120
            synack      30
            last        2
        }
        synproxy {
            synack_options {
                mss             1452
                ttl             63
                sack
                ! wscale
                ! timestamp
            }
            ! defer_rs_syn
            rs_syn_max_retry    3
            ack_storm_thresh    10
            max_ack_saved       3
            conn_reuse_state {
                close
                time_wait
                ! fin_wait
                ! close_wait
                ! last_ack
           }
        }
    }
}

! sa_pool config
sa_pool {
    pool_hash_size   16
}

部分 log 日志：

IPVS: conn lookup: [1] TCP 10.251.18.34/50290 -> 10.251.20.9/80 hit
IPVS: conn lookup: [7] TCP 10.251.223.189/80 -> 10.251.223.5/1040 miss
IPVS: tcp_conn_sched: [7] try sched non-SYN packet: [..AR] 10.251.223.189/80->10.251.223.5/1040
IPVS: conn lookup: [6] TCP 10.251.18.34/50292 -> 10.251.20.9/80 miss
IPVS: new conn:  [6] TCP 10.251.18.34/50292 10.251.20.9/80 10.251.223.5/1029 10.251.128.63/80 refs 2
IPVS: state trans: TCP in [S...] 10.251.18.34:50292->10.251.128.63:80  state NONE->SYN_RECV conn.refcnt 2
IPVS: conn lookup: [6] TCP 10.251.128.63/80 -> 10.251.223.5/1029 hit
IPVS: state trans: TCP out [..AR] 10.251.18.34:50292->10.251.128.63:80  state SYN_RECV->CLOSE conn.refcnt 2

[taylor3768]

加vlan命令建议使用：dpip vlan add dpdk0.227 link dpdk0 proto vlan id 227

[autumn0207]

@taylor3768 cx5 网卡不支持 flow director 吗？

[mscbg]

dpdk-stable-18.11.2版本以上，dpvs 1.8版本以上试试。参考另外一张卡试试，https://github.com/iqiyi/dpvs/pull/451

[legend050709]

dpdk-stable-18.11.2版本以上，dpvs 1.8版本以上试试。参考另外一张卡试试，#451

dpdk-stable-18.11.2版本以下，dpvs 1.8版本以上吧。17.11.2是适配mlx5的吧？