`recv()` not working even for `tests/sockets/tcp_socket_echo_client.c`

[kostasrim]

I have a small library written in C that uses POSIX TCP sockets which I compile to WASM via emscripten. I then call the exported WASM functions from JS in node to establish a connection to the TCP server proxied by websockify. However, this does not fully work. Even though it can connect and send data, the JS client never receives data from the server.

For this reason, I tried to play around with the test cases found in tests/sockets/tcp_sockets_echo_client.c/ and I came up with this very minimal example (slightly simplified the boilerplate from tcp_sockets_echo_client.c).

#include <arpa/inet.h>
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>
#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif

static const char HANDSHAKE[] = "\x00\x01";

typedef struct {
   char *buffer;
   int length;
} msg_t;

// message to send to the server
#ifndef MESSAGE
#define MESSAGE "pingtothepong"
#endif

typedef enum { MSG_READ, MSG_WRITE } msg_state_t;

typedef struct {
    int fd; 
    msg_t msg;
    msg_state_t state;
} server_t;

server_t server;
msg_t echo_msg;
int echo_read;
int echo_wrote;

void finish(int result) {
    if (server.fd) {
      close(server.fd);
      server.fd = 0;
    }
  #ifdef __EMSCRIPTEN__
  #ifdef REPORT_RESULT
    REPORT_RESULT(result);
  #endif
    emscripten_force_exit(result);
  #else
    exit(result);
  #endif
}

int transport_send(int sockfd, const char *buf, size_t len) {
    size_t total_sent = 0;
    while (total_sent < len) {
      ssize_t sent_now =
          send(sockfd, buf + total_sent, len - total_sent, MSG_NOSIGNAL);
      if (sent_now == -1) {
        printf("failed to send\n");
        return -1; 
      }   
      total_sent += (size_t)sent_now;
    }
    printf("total send: %u\n", total_sent);
    return 0;
  }

int transport_recv(int sockfd, const char *buf, size_t len) {
    size_t total_received = 0;
    while (total_received < len) {
      ssize_t received_now =
        recv(sockfd, buf + total_received, len - total_received, 0);
      printf("transport_recv received: %d\n", received_now);
      if (received_now == 0 && errno != EINPROGRESS) {
        return -1;
      }
      if (received_now == -1) {
        printf("coulnd't receive\n");
        return -1;
      }
      total_received += (size_t)received_now;
    }
    return 0;
}

void main_loop() {
    static char out[1024 * 2];
    static int pos = 0;
    fd_set fdr;
    fd_set fdw;
    int res = 0;
    FD_ZERO(&fdr);
    FD_ZERO(&fdw);
    FD_SET(server.fd, &fdr);
    FD_SET(server.fd, &fdw);
    int sockfd = server.fd;
    res = select(server.fd + 1, &fdr, &fdw, NULL, NULL);
    static size_t state = 0;
    if (FD_ISSET(server.fd, &fdw) && state == 0) {
      if (res == -1) {
        perror("select failed");
        finish(EXIT_FAILURE);
      }
      printf("Trying to handshake\n");
      if (transport_send(sockfd, HANDSHAKE, strlen(HANDSHAKE)) != 0) {
        printf("errror sending\n");
        finish(EXIT_FAILURE);
      }
      ++state;
      return;
    }
    if (FD_ISSET(server.fd, &fdr) && state == 1) {
      int available;
      res = ioctl(server.fd, FIONREAD, &available);
      printf("available for reading %d\n", res);
      uint32_t server_version;
      if (transport_recv(sockfd, (char *)&server_version, 4) != 0) {
        printf("Unexpected\n");
      }
      return;
    }
    printf("Shouldn't reach here, socket not readable\n");
}

int main() {
    struct sockaddr_in addr;
    int res;

    memset(&server, 0, sizeof(server_t));
    server.state = MSG_WRITE;

    // setup the message we're going to echo
    memset(&echo_msg, 0, sizeof(msg_t));
    echo_msg.length = strlen(MESSAGE) + 1;
    echo_msg.buffer = malloc(echo_msg.length);
    strncpy(echo_msg.buffer, MESSAGE, echo_msg.length);

    echo_read = 0;
    echo_wrote = 0;

    // create the socket and set to non-blocking
    server.fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (server.fd == -1) {
      perror("cannot create socket");
      finish(EXIT_FAILURE);
    }

    // connect the socket
    memset(&addr, 0, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(8001);
    printf("This is inet \n");
    if (inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr) != 1) {
      perror("inet_pton failed");
      finish(EXIT_FAILURE);
    }

    res = connect(server.fd, (struct sockaddr *)&addr, sizeof(addr));
    if (res == -1 && errno != EINPROGRESS) {
      perror("connect failed");
      finish(EXIT_FAILURE);
    }

    {
      int z;
      struct sockaddr_in adr_inet;
      socklen_t len_inet = sizeof adr_inet;
      z = getsockname(server.fd, (struct sockaddr *)&adr_inet, &len_inet);
      if (z != 0) {
        perror("getsockname");
        finish(EXIT_FAILURE);
      }
      char buffer[1000];
      sprintf(buffer, "%s:%u", inet_ntoa(adr_inet.sin_addr),
              (unsigned)ntohs(adr_inet.sin_port));
      // TODO: This is not the correct result: We should have a auto-bound address
      char *correct = "0.0.0.0:0";
      printf("got (expected) socket: %s (%s), size %lu (%lu)\n", buffer, correct,
             strlen(buffer), strlen(correct));
      assert(strlen(buffer) == strlen(correct));
      assert(strcmp(buffer, correct) == 0);
    }

    {
      int z;
      struct sockaddr_in adr_inet;
      socklen_t len_inet = sizeof adr_inet;
      z = getpeername(server.fd, (struct sockaddr *)&adr_inet, &len_inet);
      if (z != 0) {
        perror("getpeername");
        finish(EXIT_FAILURE);
      }
      char buffer[1000];
      sprintf(buffer, "%s:%u", inet_ntoa(adr_inet.sin_addr),
              (unsigned)ntohs(adr_inet.sin_port));
      char correct[1000];
      sprintf(correct, "127.0.0.1:%u", 8001);
      printf("got (expected) socket: %s (%s), size %lu (%lu)\n", buffer, correct,
             strlen(buffer), strlen(correct));
      assert(strlen(buffer) == strlen(correct));
      assert(strcmp(buffer, correct) == 0);
    }

   emscripten_set_main_loop(main_loop, 1, 0);

    return EXIT_SUCCESS;
  }

Which I compile via:

emcc client.c -o out.js -s SOCKET_DEBUG -s LLD_REPORT_UNDEFINED  -s EXPORTED_FUNCTIONS='_main'

and run:

node out.js

and a very simple TCP server written in python and proxied with websockify:

HOST = '0.0.0.0'
PORT = 8002

with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.bind((HOST, PORT))
    s.listen()
    conn, addr = s.accept()
    with conn:
        print('Connected by', addr)
       while True:
            data = conn.recv(1024)
            if not data:
                print("oh no")
                break
        conn.send(str(12).encode('utf-8'))
        print("data sent")

Now the issue is that after the client connects to the server and sends the data, the condition if (FD_ISSET(server.fd, &fdr) && state == 1) in main_loop is never true and as a consequence, we get Shouldn't reach here, socket not readable in stdout. Furthermore, on the server-side, the output has data sent printed.

I have also tried to run python3 tests/runner.py sockets to trigger the test cases for sockets that all pass. However, the printed output only refers to the servers reading/sending data to the clients, therefore making me question if the clients actually receive any data (there are asserts in the test cases, but the output of the clients is not displayed by the runner.).

Therefore, I wanted to test and see what the actual output of the client is when the tests are run. Instead of using the test runner, I compiled the tcp_sockets_echo_client with:

 emcc test_sockets_echo_client.c -o out.js -s SOCKET_DEBUG -s LLD_REPORT_UNDEFINED -s EXPORTED_FUNCTIONS='_main'

and the tcp_sockets_echo_server with:

gcc server.c -o server  

Then I used websockify to proxy to the server and run the client with node: node out.js and the output:

Client:

connect: ws://127.0.0.1:8001/, binary
websocket adding peer: 127.0.0.1:8001
__syscall_getsockname 3
got (expected) socket: 0.0.0.0:0 (0.0.0.0:0), size 9 (9)
got (expected) socket: 127.0.0.1:8001 (127.0.0.1:8001), size 14 (14)
websocket handle open
Socket open fd = 3
websocket send (4 bytes): 14,0,0,0
do_msg_write: sending message header for 14 bytes
websocket send (14 bytes): 112,105,110,103,116,111,116,104,101,112,111,110,103,0
do_msg_write: wrote 14 bytes 14

Server:

do_msg_read: allocating 14 bytes for message
do_msg_read: read 14 bytes
do_msg_write: sending message header for 14 bytes
do_msg_write: wrote 14 bytes 14
do_msg_read: read 0 bytes

Confirming my suspicion that the clients never receive any data (that is if(!FD_ISSET(server.fd, &fdr))) is true. Is this a bug with recv implementation or is there something else that I am missing?

[sbc100]

Most likely you have a found a real bug. It also looks like there are some gaps in the test coverage. If you would be interesting in help to fix these issues PRs would be most welcome.

[kostasrim]

Hi Sam, thank you very much for confirming this/replying. As this is important for something I am working on I will take a look and update on this here. However, note that I am not that familiar with the codebase so I might ping for any questions :)

Again your fast response is very much appreciated :)

[jeromeatneotek]

We are having exactly the same issue on both windows and mac, has there been any further info? Happy to contribute.

[glennra]

We have been struggling to get some existing UDP client code to work in Emscripten. So, following some advice on another issue, I tried to follow the example in the tests. As pointed out above this particular test is broken. Here's some further info about the issue.

I built the websocket_to_posix_proxy program (MacOSX+Xcode 13.3)with the DEBUG #defines set.

test_sockets_echo_client.c built was with Emscripten and TEST_DGRAM set with the following options (deduced from inspecting the test runner Python code) :

  target_link_libraries(udp_client "-sLLD_REPORT_UNDEFINED -sSOCKET_DEBUG=1 -sWEBSOCKET_DEBUG=1 -sERROR_ON_UNDEFINED_SYMBOLS=1 --shared-memory")
  target_link_libraries(udp_client "-sPROXY_TO_PTHREAD")
  target_link_libraries(udp_client "-sWASM=0")
  target_link_libraries(udp_client "-sWEBSOCKET_URL=ws://localhost:7777/ -sWEBSOCKET_SUBPROTOCOL=binary")

Here is the output from websocket_to_posix_proxy when the test client is run:

websocket_to_posix_proxy server is now listening for WebSocket connections to ws://localhost:7777/
Established new proxy connection handler thread for incoming connection, at fd=4
Received: 47 45 54 20 2F 20 48 54 54 50 2F 31 2E 31 0D 0A 53 65 63 2D 57 65 62 53 6F 63 6B 65 74 2D 56 65 72 73 69 6F 6E 3A 20 31 33 0D 0A 53 65 63 2D 57 65 62 53 6F 63 6B 65 74 2D 4B 65 79 3A 20 6E 37 6C 34 59 38 63 77 78 6B 75 42 70 72 64 35 75 75 68 4A 6A 41 3D 3D 0D 0A 43 6F 6E 6E 65 63 74 69 6F 6E 3A 20 55 70 67 72 61 64 65 0D 0A 55 70 67 72 61 64 65 3A 20 77 65 62 73 6F 63 6B 65 74 0D 0A 53 65 63 2D 57 65 62 53 6F 63 6B 65 74 2D 45 78 74 65 6E 73 69 6F 6E 73 3A 20 70 65 72 6D 65 73 73 61 67 65 2D 64 65 66 6C 61 74 65 3B 20 63 6C 69 65 6E 74 5F 6D 61 78 5F 77 69 6E 64 6F 77 5F 62 69 74 73 0D 0A 48 6F 73 74 3A 20 6C 6F 63 61 6C 68 6F 73 74 3A 37 37 37 37 0D 0A 0D 0A
hashing key: "n7l4Y8cwxkuBprd5uuhJjA==258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
Sent handshake:
HTTP/1.1 101 Switching Protocols
Upgrade: websocket
Connection: Upgrade
Sec-WebSocket-Accept: mnjaTYnz87wIdX6AxxwYCIgvcjE=

Handshake received, entering message loop:
Received: 82 84 05 20 30 23 0B 20 30 23
Have 0+10==10 bytes now in queue
Received: FIN: 1, opcode: binary frame (0x2), mask: 0x23302005, payload length: 4 bytes, unmasked payload:
  0E 00 00 00
Received too small sockets call message! size: 4 bytes, expected at least 8 bytes
Cleared used bytes, got 0 left in fragment queue.
Received: 82 8E 19 CA DC 00 69 A3 B2 67 6D A5 A8 68 7C BA B3 6E 7E CA
Have 0+20==20 bytes now in queue
Received: FIN: 1, opcode: binary frame (0x2), mask: 0x00DCCA19, payload length: 14 bytes, unmasked payload:
  70 69 6E 67 74 6F 74 68  65 70 6F 6E 67 00
Unknown POSIX_SOCKET_MSG 1752461172 received!
Cleared used bytes, got 0 left in fragment queue.

It took me a bit of trial and error to get this to work because the link options for the client that are used in the examples do not seem to be consistent with the documentation.

It looks like the client is not encapsulating the proxied data properly for the proxy server. I'd like to debug this further but I'm not sure, if that is the proper diagnosis of the issue or where to find the code that is doing that.