websockets packet size limited to 2930 bytes

everslick commented 8 years ago

I have observed that my JSON encoded data gets silently corrupted if the packet size is bigger then 2930 bytes. for demonstration i wrote the following sketch:

#include <ESP8266WiFi.h>
#include <ESP8266WebServer.h>
#include <WebSocketsServer.h>

// install arduinoWebSockets
// git@github.com:Links2004/arduinoWebSockets.git

#define WIFI_SSID  "XXX"
#define WIFI_PASS  "xxx"

#define NUM_VALUES 268

static WebSocketsServer *websocket = NULL;
static ESP8266WebServer *webserver = NULL;
static const char *html_root_fmt   = NULL;

static void ws_event(uint8_t num, WStype_t type, uint8_t *payload, size_t length) {
  IPAddress ip = websocket->remoteIP(num);

  switch (type) {
    case WStype_DISCONNECTED:
      Serial.printf("WS:   client [%i] disconnected\n", num);
    break;

    case WStype_CONNECTED:
      Serial.printf("WS:   client [%i] connected from %d.%d.%d.%d url: %s\n",
        num, ip[0], ip[1], ip[2], ip[3], payload
      );
    break;

    case WStype_TEXT:
      if (!strcmp((const char *)payload, "test")) {      
        String response;

        response.reserve(4*1024);
        response += "              "; // 14 bytes reserved for header

        response += "{\"type\":\"test\", \"values\":[";
        for (int i=0; i<NUM_VALUES; i++) {
          response += String(RANDOM_REG32);
          if (i < NUM_VALUES - 1) response += ",";
        }
        response += "] } ";

        // this sends the buffer as it is, thus we have to reserve space for the websocket header
        websocket->sendTXT(num, (char *)response.c_str(),
          response.length()-WEBSOCKETS_MAX_HEADER_SIZE, true);

        // this creates a new buffer within websocket lib and it will make a copy
        //websocket->sendTXT(num, (char *)response.c_str(), response.length());

        Serial.printf("WS:   sent %i bytes\n", response.length());     
      }
    break;

    default:
      Serial.printf("WS:   unhandled event type: %i\n", type);
    break;
  }
}

static void handle_root_cb(void) {
  char *pagebuf = (char *)malloc(1024);

  if (!pagebuf) {
    Serial.printf("HTTP: failed to allocate page buffer\n"); return;
  }
  snprintf_P(pagebuf, 1024, html_root_fmt, WiFi.localIP().toString().c_str());

  webserver->send(200, "text/html", pagebuf);

  free(pagebuf);
}

void setup(void) {
  Serial.begin(115200);

  Serial.printf("WIFI: connecting ");
  WiFi.begin(WIFI_SSID, WIFI_PASS);
  while (WiFi.status() != WL_CONNECTED) {
    Serial.print("."); delay(300);
  }
  Serial.print(" IP=");
  Serial.println(WiFi.localIP());

  html_root_fmt = PSTR(" \
    <!DOCTYPE html>\n \
    <html><head>\n \
    <title>WEBSOCKET TEST</title>\n \
    <script>\n \
      var connection = new WebSocket('ws://%s:81/', ['test']);\n \
      var id;\n \
      \n \
      connection.onopen = function() {\n \
        connection.send('test');\n \
        id = setInterval(function() { connection.send('test'); }, 1000);\n \
      };\n \
      \n \
      connection.onerror = function(error) {\n \
        console.log('WebSocket Error ', error);\n \
        clearInterval(id);\n \
      };\n \
      \n \
      connection.onmessage = function(e) {\n \
        var d = JSON.parse(e.data);\n \
      }\n \
    </script>\n \
    </head><body>\n \
    websocket test\n \
    </body></html>\n \
  ");

  Serial.printf("HTTP: starting http server\n");
  webserver = new ESP8266WebServer(80);
  webserver->on("/", HTTP_GET,  handle_root_cb);
  webserver->begin();

  Serial.printf("WS:   starting websocket server\n");
  websocket = new WebSocketsServer(81, "", "test");
  websocket->onEvent(ws_event);
  websocket->begin();

  Serial.printf("MAIN: starting loop ...\n");
}

void loop(void) {
  websocket->loop();
  webserver->handleClient();
  yield();
}

discussing this issue on gitter, MeNoDev explained:

That is so because that is the maximum size usually for the first outgoing window in order for it to send larger packets, you need to split them into MTU chunks and send when it's possible there is a bit more to that if you want to get the transmission as a single message on the other end.

/me has hoped, that arduinoWebSockets would handle this. no?

Links2004 commented 8 years ago

the full message is send via one write call. In a normal architecture the splitting in to the right MTU size for TCP is jop of the network API / libs, but for the ESP I am not sure if its really is done for all cases.

at the first look i only find it for write_P / PGM_P: https://github.com/esp8266/Arduino/blob/master/libraries/ESP8266WiFi/src/WiFiClient.cpp#L186-L202

for ASYNC I have implemented it: https://github.com/me-no-dev/ESPAsyncTCP/blob/master/src/ESPAsyncTCPbuffer.cpp#L103-L143

will take a deeper look at the weekend.

for the arduinoWebSockets it make no difference if the data is received in one TCP packet or if it splittet over multible ones. you will get one API call for one websocket message independent of the TCP packets needed.

Swiftnesses commented 8 years ago

Any news on this issue?

Links2004 commented 7 years ago

https://github.com/Links2004/arduinoWebSockets/commits/write_big_data may fix it, but no one has tested it yet.

Links2004 commented 7 years ago

write_big_data is now part of master

Kamal-Sonani commented 1 year ago

https://github.com/marvinroger/async-mqtt-client/issues/74#issuecomment-362832999

works for me

Links2004 / arduinoWebSockets

websockets packet size limited to 2930 bytes #85