tcp slowdown when recv large stream

[up0617]

I'm trying to recv files via DPDK TCP sock. I found that DPDK ANS is faster at low size file but slower at big size than kernel, as below.

kernel	dpdk-ans	Size
0.014478	0.018714	9.2M
0.007788	0.009212	3.8M
0.004526	0.003432	1.3M
0.002268	0.001319	212K

(*sec)

I just wonder if It is a known issue.

I tried some settings to optimize performance(cpu affinity, pinning, numa, tcp_nodelay, change BURST_TX_DRAIN_US, MAX_TX_BURST and etc...) but these make no difference.

Below is my send/recv logic.

#define SEND_MAX_SIZE (8192)

#senderside code
int sendData(int sockfd, FILE * fp, char *path, int fileSize){
    int readLen;
    char buf[SEND_MAX_SIZE];  
    while ((readLen = fread(buf, sizeof(char), SEND_MAX_SIZE, fp))>0) {
        int sendLen = readLen;
        int sent = 0;
        int totalsend = 0;
        while(sendLen){
    #ifdef DPDK_Enabled
            sent = anssock_send(sockfd, buf+totalsend, sendLen, 0);
    #else
            sent = send(sockfd, buf+totalsend, sendLen, 0);
    #endif
            if (sent < 0)
            {
                sent = 0;
                if ( errno == EAGAIN){
                    continue;
                }
                else {
                    return -1;
                }
            }
            sendLen -= sent;
            totalsend += sent;
        }
    }
    return 0;
}

#recv side code
void recvFile(struct epoll_event event, char* filename, int fileSize)
{
    int totalRecv = 0;

    char *recvBuf = Files[fileidx].buf;

    while (fileSize)
    {
        int recvsize = SEND_MAX_SIZE > fileSize ? fileSize : SEND_MAX_SIZE;
#ifdef DPDK_Enabled
        int recv_len = anssock_recv(event.data.fd, recvBuf+totalRecv, recvsize, 0);
#else
        int recv_len = recv(event.data.fd, recvBuf+totalRecv, recvsize, 0);
#endif
        if (recv_len<=0){
            if ( errno == EAGAIN){
                continue;
            }
            close(event.data.fd);
            exit(EXIT_FAILURE);
        }
        fileSize -= recv_len;
        totalRecv += recv_len;
    }
    return;
}