knolleary / pubsubclient

A client library for the Arduino Ethernet Shield that provides support for MQTT.
http://pubsubclient.knolleary.net/
MIT License
3.82k stars 1.47k forks source link

socket error on client when payload bigger than 4096 esp32 #925

Open marcelwinter04 opened 2 years ago

marcelwinter04 commented 2 years ago

Hello,

I'm currently facing the issue that I'm getting the error "Socket error on client xxxxxx, disconnecting" from my broker. But this is only the case when i publish data that is bigger than 4096 Bytes.

I have already set the buffer size a little bit bigger, but that does not seem to be the problem.

#include "WiFi.h"
#include <PubSubClient.h>
#include <WiFiClientSecure.h>

WiFiClientSecure clientMqtt;
PubSubClient mqttClient(clientMqtt);
mqttClient.setBufferSize(8192);
mqttClient.setKeepAlive(60);
mqttClient.setSocketTimeout(60);´

Funny thing is the broker gets this message. Until 432 everything is correct, but then the message starts again.

{"deviceID":"243bd1a3c9c8","data": {"0":1,"1":1,"2":4,"3":4,"4":4,"5":4,"6":4,"7":4,"8":4,"9":7,"10":1,"11":0,"12":0,"13":32766,"14":32766,"15":32766,"16":32766,"17":32766,"18":32766,"19":101,"20":2,"21":0,"22":0,"23":0,"24":0,"25":10,"26":30,"27":1,"28":25,"29":65531,"30":0,"31":20,"32":42,"33":1,"34":5,"35":3,"36":32766,"37":32766,"38":32766,"39":32766,"40":70,"41":100,"42":60,"43":50,"44":99,"45":15,"46":3,"47":3,"48":120,"49":3,"50":3,"51":99,"52":5,"53":1,"54":32766,"55":32766,"56":32766,"57":32766,"58":32766,"59":32766,"60":1,"61":0,"62":4,"63":100,"64":0,"65":8,"66":0,"67":0,"68":20,"69":44,"70":0,"71":0,"72":1,"73":1,"74":1,"75":9,"76":1,"77":1,"78":0,"79":0,"80":55,"81":45,"82":99,"83":65,"84":65,"85":0,"86":3,"87":0,"88":0,"89":1,"90":32766,"91":32766,"92":32766,"93":32766,"94":32766,"95":32766,"96":32766,"97":32766,"98":32766,"99":32766,"100":18,"101":10,"102":32766,"103":20,"104":35,"105":50,"106":65,"107":0,"108":1,"109":15,"110":3,"111":32766,"112":32766,"113":0,"114":18,"115":10,"116":220,"117":20,"118":35,"119":50,"120":65,"121":0,"122":1,"123":15,"124":3,"125":50,"126":3,"127":0,"128":18,"129":10,"130":200,"131":20,"132":35,"133":50,"134":65,"135":0,"136":1,"137":15,"138":3,"139":50,"140":3,"141":0,"142":18,"143":10,"144":280,"145":20,"146":35,"147":50,"148":65,"149":0,"150":1,"151":15,"152":3,"153":50,"154":3,"155":0,"156":18,"157":10,"158":208,"159":20,"160":35,"161":50,"162":65,"163":0,"164":1,"165":15,"166":3,"167":50,"168":3,"169":0,"170":18,"171":10,"172":200,"173":20,"174":35,"175":50,"176":65,"177":0,"178":1,"179":15,"180":3,"181":50,"182":3,"183":0,"184":18,"185":10,"186":200,"187":20,"188":35,"189":50,"190":65,"191":0,"192":1,"193":15,"194":3,"195":50,"196":3,"197":0,"198":18,"199":10,"200":200,"201":20,"202":35,"203":50,"204":65,"205":0,"206":1,"207":15,"208":3,"209":50,"210":3,"211":0,"212":0,"213":65534,"214":65496,"215":3,"216":24,"217":12,"218":2018,"219":26,"220":12,"221":2022,"222":1200,"223":1200,"224":1200,"225":1200,"226":1200,"227":1200,"228":32766,"229":32766,"230":32766,"231":32766,"232":32766,"233":32766,"234":1200,"235":1200,"236":1200,"237":1200,"238":1200,"239":1200,"240":5,"241":5,"242":5,"243":5,"244":5,"245":5,"246":5,"247":5,"248":0,"249":0,"250":0,"251":0,"252":0,"253":0,"254":0,"255":0,"256":0,"257":0,"258":0,"259":0,"260":0,"261":0,"262":0,"263":0,"264":15,"265":0,"266":0,"267":0,"268":0,"269":0,"270":0,"271":0,"272":0,"273":0,"274":1200,"275":1200,"276":15,"277":0,"278":0,"279":0,"280":0,"281":0,"282":0,"283":0,"284":0,"285":0,"286":1200,"287":1200,"288":15,"289":0,"290":0,"291":0,"292":0,"293":0,"294":0,"295":0,"296":0,"297":0,"298":1200,"299":1200,"300":15,"301":0,"302":0,"303":0,"304":0,"305":0,"306":600,"307":2200,"308":1200,"309":1200,"310":1200,"311":1200,"312":600,"313":2200,"314":1200,"315":1200,"316":1200,"317":1200,"318":600,"319":2200,"320":1200,"321":1200,"322":1200,"323":1200,"324":600,"325":2200,"326":1200,"327":1200,"328":1200,"329":1200,"330":600,"331":2200,"332":1200,"333":1200,"334":1200,"335":1200,"336":600,"337":2200,"338":1200,"339":1200,"340":1200,"341":1200,"342":600,"343":2200,"344":1200,"345":1200,"346":1200,"347":1200,"348":600,"349":2200,"350":1200,"351":1200,"352":1200,"353":1200,"354":600,"355":2200,"356":1200,"357":1200,"358":1200,"359":1200,"360":600,"361":2200,"362":1200,"363":1200,"364":1200,"365":1200,"366":600,"367":2200,"368":1200,"369":1200,"370":1200,"371":1200,"372":600,"373":2200,"374":1200,"375":1200,"376":1200,"377":1200,"378":600,"379":2200,"380":1200,"381":1200,"382":1200,"383":1200,"384":600,"385":2200,"386":1200,"387":1200,"388":1200,"389":1200,"390":600,"391":2200,"392":1200,"393":1200,"394":1200,"395":1200,"396":600,"397":2200,"398":1200,"399":1200,"400":1200,"401":1200,"402":600,"403":2200,"404":1200,"405":1200,"406":1200,"407":1200,"408":600,"409":2200,"410":1200,"411":1200,"412":1200,"413":1200,"414":600,"415":2200,"416":1200,"417":1200,"418":1200,"419":1200,"420":600,"421":2200,"422":1200,"423":1200,"424":1200,"425":1200,"426":600,"427":2200,"428":1200,"429":1200,"430":1200,"431":1200,"432"0�. device/243bd1a3c9c8/data{"deviceID":"243bd1a3c9c8","data": {"0":1,"1":1,"2":4,"3":4,"4":4,"5":4,"6":4,"7":4,"8":4,"9":7,"10":1,"11":0,"12":0,"13":32766,"14":32766,"15":32766,"16":32766,"17":32766,"18":32766,"19":101,"20":2,"21":0,"22":0,"23":0,"24":0,"25":10,"26":30,"27":1,"28":25,"29":65531,"30":0,"31":20,"32":42,"33":1,"34":5,"35":3,"36":32766,"37":32766,"38":32766,"39":32766,"40":70,"41":100,"42":60,"43":50,"44":99,"45":15,"46":3,"47":3,"48":120,"49":3,"50":3,"51":99,"52":5,"53":1,"54":32766,"55":32766,"56":32766,"57":32766,"58":32766,"59":32766,"60":1,"61":0,"62":4,"63":100,"64":0,"65":8,"66":0,"67":0,"68":20,"69":44,"70":0,"71":0,"72":1,"73":1,"74":1,"75":9,"76":1,"77":1,"78":0,"79":0,"80":55,"81":45,"82":99,"83":65,"84":65,"85":0,"86":3,"87":0,"88":0,"89":1,"90":32766,"91":32766,"92":32766,"93":32766,"94":32766,"95":32766,"96":32766,"97":32766,"98":32766,"99":32766,"100":18,"101":10,"102":32766,"103":20,"104":35,"105":50,"106":65,"107":0,"108":1,"109":15,"110":3,"111":32766,"112":32766,"113":0,"114":18,"115":10,"116":220,"117":20,"118":35,"119":50,"120":65,"121":0,"122":1,"123":15,"124":3,"125":50,"126":3,"127":0,"128":18,"129":10,"130":200,"131":20,"132":35,"133":50,"134":65,"135":0,"136":1,"137":15,"138":3,"139":50,"140":3,"141":0,"142":18,"143":10,"144":280,"145":20,"146":35,"147":50,"148":65,"149":0,"150":1,"151":15,"152":3,"153":50,"154":3,"155":0,"156":18,"157":10,"158":208,"159":20,"160":35,"161":50,"162":65,"163":0,"164":1,"165":15,"166":3,"167":50,"168":3,"169":0,"170":18,"171":10,"172":200,"173":20,"174":35,"175":50,"176":65,"177":0,"178":1,"179":15,"180":3,"181":50,"182":3,"183":0,"184":18,"185":10,"186":200,"187":20,"188":35,"189":50,"190":65,"191":0,"192":1,"193":15,"194":3,"195":50,"196":3,"197":0,"198":18,"199":10,"200":200,"201":20,"202":35,"203":50,"204":65,"

This looks like my string has an overflow.

"431":1200,"432"0�. device/243bd1a3c9c8/data{"deviceID":"243bd1a3c9c8","data": {"0":1,"1":1,"2":4,"3":4,"4":4,"
 String publishTopic = "";
  String socketString = "{\"deviceID\":\"";
  socketString += macString;
  socketString += "\",\"data\": {";

  for (int i = 0; i < 600; i++)
  {
    socketString += "\"";
    socketString += i;
    socketString += "\":";
    socketString += getValue(i);
    if (i < 599)
    {
      socketString += ",";
    }
  }
  socketString += " }}";

  publishTopic = "device/";
  publishTopic += macString;
  publishTopic += "/data";
  mqttClient.publish(publishTopic.c_str(), socketString.c_str());
  Serial.println("send mqtt");

Edit: I'm using Arduino as Component

marcelwinter04 commented 2 years ago

Update:

I'm now using the .print() function to send data. First that seemed to fix the problem but then i recognized that this function is extremly slow (~5 Seconds to send the data).

Now I'm using a combination of both approaches. But i'm not really happy with that. Maybe anyone could help me?

void sendDeviceData()
{
  int sendLength = 0;
  String publishTopic = "";
  String socketString = "{\"deviceID\":\"";
  socketString += macString;
  socketString += "\",\"data\": {";
  socketString += " }}"; 

  sendLength = socketString.length();

  for (int i = 0; i < 1200; i++)
  {
    socketString = "\"";
    socketString += i;
    socketString += "\":";
    socketString += getValue(i);
    if (i < 1199)
    {
      socketString += ",";
    }
    sendLength += socketString.length();
  }

  Serial.println(sendLength);

  publishTopic = "device/";
  publishTopic += macString;
  publishTopic += "/data";

  mqttClient.beginPublish(publishTopic.c_str(), (sendLength-1), false);
  mqttClient.print("{\"deviceID\":\"");
  mqttClient.print(macString);
  mqttClient.print("\",\"data\": {");
  for(int j = 0; j < 6; j++) {

    socketString = "";
    for (int i = 0; i < 200; i++)
    {
      socketString += "\"";
      socketString += i+(j*200);
      socketString += "\":";
      socketString += getValue(i+(j*200));
      if (i+(j*200) < 1199)
      {
        socketString += ",";
      }
    }
     mqttClient.print(socketString);
  }
  mqttClient.print("}}");

  mqttClient.endPublish();

  //mqttClient.publish(publishTopic.c_str(), socketString.c_str());
  Serial.println("send mqtt");
}
fsommer1968 commented 2 years ago

Hi , I would place the callback into one string (8192 quite much for this little mc, i hope you have enough heap free during operation esp. when you are using tls with no buffersize limitation..... Next I would reserve enough space for the string by using reserve(): String socketString; socketString.reserve(4098), because frequent socketString += without reserve() first copies the string every time >> heap fragmentation.

marcelwinter04 commented 2 years ago

@fsommer1968 thanks for your help, i will try this.