fstack nginx tcp stream lower throughput than official nginx

[vincentmli]

Hi,

I have two Dell servers with Intel 82599 dual 10G port ( say server S1, server S2), they are connected directly with cable, I run f-stack nginx on S2, iperf client and server mode on S1, the packet flow as below, I am not sure where to start to tune to make the fstack nginx with better throughput with iperf, is tuning needed in dpdk driver layer or in the fstack tcp layer ? any clue would be appreciated!

iperf client (S1) <--->port1 fstack(S2)<---> port0 fstack(S2)<---->iperf server(S1)

f-stack config:

cat /usr/local/nginx_fstack/conf/f-stack.conf

[dpdk]
## Hexadecimal bitmask of cores to run on.
lcore_mask=ffff
channel=4
promiscuous=1
numa_on=1
## TCP segment offload, default: disabled.
tso=0
## HW vlan strip, default: enabled.
vlan_strip=1

# enabled port list
#
# EBNF grammar:
#
#    exp      ::= num_list {"," num_list}
#    num_list ::=  | 
#    range    ::= "-"
#    num      ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
#
# examples
#    0-3       ports 0, 1,2,3 are enabled
#    1-3,4,7   ports 1,2,3,4,7 are enabled
port_list=0,1

## Port config section
## Correspond to dpdk.port_list's index: port0, port1...
[port0]
addr=10.8.8.2
netmask=255.255.0.0
broadcast=10.8.255.255
gateway=10.8.8.8

lcore_list= 0-7 

[port1]
addr=10.9.9.2
netmask=255.255.0.0
broadcast=10.9.255.255
gateway=10.9.9.3
#
### lcore list used to handle this port
### the format is same as port_list
lcore_list= 0-7 
## FreeBSD network performance tuning configurations.
## Most native FreeBSD configurations are supported.
[freebsd.boot]
hz=100

## Block out a range of descriptors to avoid overlap
## with the kernel's descriptor space.
## You can increase this value according to your app.
fd_reserve=1024

kern.ipc.maxsockets=262144

net.inet.tcp.syncache.hashsize=4096
net.inet.tcp.syncache.bucketlimit=100

net.inet.tcp.tcbhashsize=65536

kern.ncallout=262144

[freebsd.sysctl]
kern.ipc.somaxconn=32768
kern.ipc.maxsockbuf=16777216

net.link.ether.inet.maxhold=5

net.inet.tcp.fast_finwait2_recycle=1
net.inet.tcp.sendspace=16384
net.inet.tcp.recvspace=8192
net.inet.tcp.nolocaltimewait=1
net.inet.tcp.cc.algorithm=cubic
net.inet.tcp.sendbuf_max=16777216
net.inet.tcp.recvbuf_max=16777216
net.inet.tcp.sendbuf_auto=1
net.inet.tcp.recvbuf_auto=1
net.inet.tcp.sendbuf_inc=16384
net.inet.tcp.recvbuf_inc=524288
net.inet.tcp.sack.enable=1
net.inet.tcp.blackhole=1
net.inet.tcp.msl=2000
net.inet.tcp.delayed_ack=0

net.inet.udp.blackhole=1
net.inet.ip.redirect=0

cat /usr/local/nginx_fstack/conf/nginx.conf

# root account is necessary.
user  root;
# should be equal to the lcore count of `dpdk.lcore_mask` in f-stack.conf.
worker_processes  8;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

# path of f-stack configuration file, default: $NGX_PREFIX/conf/f-stack.conf.
fstack_conf /usr/local/nginx_fstack/conf/f-stack.conf;

events {
    worker_connections  102400;
    use kqueue;
}

stream {
    upstream iperf_backends {
          server 10.8.8.8:5001;
    }
    server {
          listen 5001;
          proxy_pass iperf_backends;
    }
}

/home/dpdk/iperf -c 10.9.9.2 -P 8

------------------------------------------------------------
Client connecting to 10.9.9.2, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 10] local 10.9.9.3 port 54230 connected with 10.9.9.2 port 5001
[  3] local 10.9.9.3 port 54216 connected with 10.9.9.2 port 5001
[  4] local 10.9.9.3 port 54218 connected with 10.9.9.2 port 5001
[  5] local 10.9.9.3 port 54220 connected with 10.9.9.2 port 5001
[  6] local 10.9.9.3 port 54222 connected with 10.9.9.2 port 5001
[  8] local 10.9.9.3 port 54226 connected with 10.9.9.2 port 5001
[  7] local 10.9.9.3 port 54224 connected with 10.9.9.2 port 5001
[  9] local 10.9.9.3 port 54228 connected with 10.9.9.2 port 5001
[ ID] Interval       Transfer     Bandwidth
[ 10]  0.0-10.0 sec   858 MBytes   720 Mbits/sec
[  4]  0.0-10.0 sec   919 MBytes   770 Mbits/sec
[  5]  0.0-10.0 sec   868 MBytes   727 Mbits/sec
[  6]  0.0-10.0 sec   859 MBytes   720 Mbits/sec
[  7]  0.0-10.0 sec   895 MBytes   751 Mbits/sec
[  3]  0.0-10.0 sec   863 MBytes   723 Mbits/sec
[  8]  0.0-10.0 sec   888 MBytes   743 Mbits/sec
[  9]  0.0-10.0 sec   882 MBytes   739 Mbits/sec
[SUM]  0.0-10.0 sec  6.87 GBytes  5.89 Gbits/sec -------5.89G

I run the same nginx 1.11.10 from nginx download site and same nginx config, I can get much better throughput

/home/dpdk/iperf -c 10.9.9.2 -P 8

------------------------------------------------------------
Client connecting to 10.9.9.2, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 10] local 10.9.9.3 port 54336 connected with 10.9.9.2 port 5001
[  3] local 10.9.9.3 port 54322 connected with 10.9.9.2 port 5001
[  4] local 10.9.9.3 port 54324 connected with 10.9.9.2 port 5001
[  5] local 10.9.9.3 port 54326 connected with 10.9.9.2 port 5001
[  9] local 10.9.9.3 port 54334 connected with 10.9.9.2 port 5001
[  7] local 10.9.9.3 port 54330 connected with 10.9.9.2 port 5001
[  8] local 10.9.9.3 port 54332 connected with 10.9.9.2 port 5001
[  6] local 10.9.9.3 port 54328 connected with 10.9.9.2 port 5001
[ ID] Interval       Transfer     Bandwidth
[  3]  0.0-10.0 sec  1.13 GBytes   967 Mbits/sec
[  4]  0.0-10.0 sec  1.18 GBytes  1.01 Gbits/sec
[  5]  0.0-10.0 sec  1.67 GBytes  1.44 Gbits/sec
[  7]  0.0-10.0 sec  1.72 GBytes  1.48 Gbits/sec
[  8]  0.0-10.0 sec  1.23 GBytes  1.05 Gbits/sec
[  6]  0.0-10.0 sec  1.23 GBytes  1.06 Gbits/sec
[ 10]  0.0-10.0 sec  1.19 GBytes  1.02 Gbits/sec
[  9]  0.0-10.0 sec  1.63 GBytes  1.40 Gbits/sec
[SUM]  0.0-10.0 sec  11.0 GBytes  9.42 bits/sec -------much better throughput 9.42G

[jfb8856606]

see 59bb71f

[mulhome-com]

@vincentmli Hi, could you explain which iperf can be recognized by fstack nginx?