epoll() seems stop working with zc mode in some case

[QbsuranAlang]

English is not my first language.

Here is the two interfaces with configured 8 RSS.

$ ethtool -l ens2f0
Current hardware settings:
RX:     0
TX:     0
Other:      1
Combined:   8

$ ethtool -l ens2f1
Current hardware settings:
RX:     0
TX:     0
Other:      1
Combined:   8

I open the two interfaces and make it in inline mode, and I found an issues(maybe).

1. Start the traffic generator
2. Start the inline program(it works, forwarding)
3. Stop the traffic generator(make sure no packets incoming)
4. Start the traffic generator again
5. Inline program is not working

or

1. Make sure no packets incoming
2. Start the inline program
3. Start the traffic generator
4. Inline program is not working

Here is some information.

$ cat /proc/net/pf_ring/info 
PF_RING Version          : 8.0.0 (8.0.0-stable:58e764f619bb4711a66b021f7a475401d99ba371)
Total rings              : 0

Standard (non ZC) Options
Ring slots               : 65536
Slot version             : 18
Capture TX               : Yes [RX+TX]
IP Defragment            : No
Socket Mode              : Standard
Cluster Fragment Queue   : 0

$ sudo pf_ringcfg --list-interfaces    
Name: ens2f0               Driver: ixgbe      RSS:     8    [Running ZC]
Name: ens2f1               Driver: ixgbe      RSS:     8    [Running ZC]
Name: enp1s0f0             Driver: igb        RSS:     8    [Supported by ZC]
Name: enp1s0f1             Driver: igb        RSS:     8    [Supported by ZC]

$ uname -a
Linux ubuntu 5.4.0-050400-generic #201911242031 SMP Mon Nov 25 01:35:10 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux

And the full code:

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif /* _GNU_SOURCE */

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <getopt.h>
#include <string.h>
#include <errno.h>
#include <signal.h>
#include <net/if.h>
#include <sys/epoll.h>
#include <pfring.h>
#include <pfring_zc.h>
#include <sched.h>

int stop = 0;

void handler(int sig) {
    stop = 1;
}//end handler

void stick_cpu_core(int cpu) {
    cpu_set_t cpuset;

    CPU_ZERO(&cpuset);
    CPU_SET(cpu, &cpuset);

    sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
}//end stick_cpu_core

pfring *open_ring(const char *dev) {
    pfring *r;

    printf("open %s\n", dev);
    r = pfring_open(dev, 1536, PF_RING_PROMISC);
    if(!r) {
        perror("pfring_open()");
        exit(1);
    }//end if

    pfring_enable_ring(r);

    return r;
}//end open_ring

static void epoll_ctl_add(int epfd, int fd, uint32_t events) {
    struct epoll_event ev;

    ev.events = events;
    ev.data.fd = fd;
    if(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev) == -1) {
        perror("epoll_ctl()");
        exit(1);
    }//end if
}//end epoll_ctl_add

int main(int argc, char *argv[]) {

    int core = 8;

    if(argc == 2) {
        core = atoi(argv[1]);
    }//end if

    signal(SIGINT, handler);

    for(int i = 0 ; i < core ;i++) {

        if(fork() == 0) {
            stick_cpu_core(i);

            char dev[64];
            snprintf(dev, sizeof(dev), "zc:ens2f0@%d", i);
            pfring *ens2f0 = open_ring(dev);
            snprintf(dev, sizeof(dev), "zc:ens2f1@%d", i);
            pfring *ens2f1 = open_ring(dev);

            int epfd = epoll_create1(1);
            int ens2f0_fd, ens2f1_fd;
            ens2f0_fd = pfring_get_selectable_fd(ens2f0);
            ens2f1_fd = pfring_get_selectable_fd(ens2f1);

            epoll_ctl_add(epfd, ens2f0_fd, EPOLLIN);
            epoll_ctl_add(epfd, ens2f1_fd, EPOLLIN);

            struct epoll_event events[2];

            while(!stop) {
                int nfds = epoll_wait(epfd, events, 2, 1000);
                if(nfds < 0) {
                    perror("epoll_wait()");
                    break;
                } else if(nfds == 0) {
                    continue;
                }//end if

                for(int i = 0 ; i < nfds ; i++) {
                    pfring *r, *s;
                    if(events[i].data.fd == ens2f0_fd) {
                        r = ens2f0;
                        s = ens2f1;
                    } else {
                        r = ens2f1;
                        s = ens2f0;
                    }//end else

                    u_char *buf;
                    struct pfring_pkthdr hdr;
                    errno = 0;
                    hdr.caplen = 0;
                    int ret = pfring_recv(r, &buf, 0, &hdr, 0);
                    if(ret < 0 && errno != 0) {
                        perror("pfring_recv()");
                        continue;
                    }//end if

                    errno = 0;
                    ret = pfring_send(s, (char *)buf, hdr.caplen, 1);
                    if(ret < 0 && errno != 0) {
                        perror("pfring_send()");
                    }//end if
                }//end for
            }//end while

            pfring_close(ens2f0);
            pfring_close(ens2f1);

            exit(0);
        }//end if
    }//end for

    while(!stop) {
        sleep(1);
    }//end while
    sleep(3);

    return 0;
}//end main

When no packets incoming, it blocks in epoll_wait().

[cardigliano]

@QbsuranAlang do you experience the same issue with select()?

[cardigliano]

Please also note that in some case pfring_poll() should be used as not all the adapters provide a selectable fd. However ixgbe does provide a fd, and pfring_poll is not an option as you need to poll from multiple fds.

[QbsuranAlang]

@cardigliano thanks your reply. I replace epoll() with poll()/select() and it works fine, can you explain what causes it? And as you mention, How do I check my adapter provide a selectable fd? By driver or...?

[cardigliano]

In order to check is the adapter supports it, you should what is returned by pfring_get_selectable_fd. As of epoll, I need to dig more.

[QbsuranAlang]

OK, thanks your time!