Test 1.3.7 MQTT "listusr" causes system reboot

[windy54]

I am using the 1.3.7 debug version on a node-mcu controller.

Having looked at the (earlier) version of the code the RFID system has to be connected as a local client.

So I have a Raspberry PI acting as a wifi Access Point and running my mqtt broker. I can monitor MQTT messages output by the RFID system, so I have moved onto to sending mqtt messages to the RFID system. Remote door opening works. When I tried "listusrs " about 20 users were listed then the system crashed, re-booted and because it could not connect to the Raspberry PI it booted into AP mode. The Raspberry PI is still functioning as an AP. There are about 100 valid users in the system.

I have attached the complete dump from the debug output to the terminal window, apart from passwords and valid usernames, in case this helps. Might be too much information!

`[ INFO ] JSON msg :{"cmd": "listusr","doorip": "192.168.4.6"}" [ INFO ] List usees [ INFO ] getUserList [ INFO ] Mqtt Publish:{"command":"userfile","uid":"6671d220","user":"stevetest","acctype":1,"acctype2":null,"acctype3":null,"acctype4":null,"validuntil":2145916800} [ INFO ] Mqtt Publish:{"command":"userfile","uid":"eb6681b","user":"anothertest","acctype":1,"acctype2":null,"acctype3":null,"acctype4":null,"validuntil":2145916800} [ INFO ] Mqtt Publish: another 16 valid messages which I have not shown as they are real users [ INFO Soft WDT reset

stack>>>

ctx: sys sp: 3fffe880 end: 3fffffb0 offset: 01b0 3fffea30: 3ffe94d4 00000000 3ffe94c4 4021e726
3fffea40: 3fff5e68 3fffea84 3fffea80 00000000
3fffea50: 3fff5e68 3fffeaf4 3fff1368 4021a290
3fffea60: 0000000d 00000001 3fff5f6c 4021a659
3fffea70: 3ffe94be 00000000 00000086 40204320
3fffea80: 3fffea88 00000074 3fffeb18 4021a844
3fffea90: 3fff5e68 3fffeaf4 3fffeb18 4025ffa3
3fffeaa0: 00000000 00000000 00000000 36ffeb96
3fffeab0: 40225dc8 00000000 000003e8 4021bc44
3fffeac0: 3fff4694 3fff4664 00000000 00000000
3fffead0: 3fff15d8 402258c8 00000000 3fff5e5c
3fffeae0: 00000200 402258c8 3ffea2a9 3fff59d4
3fffeaf0: 00000800 3fff5f6c 0074007f ff3a2264
3fffeb00: 37646165 00366434 00003664 3fff5224
3fffeb10: 3fff470c 3fff15d8 3fff46f4 000d000f
3fffeb20: fffea200 3fff5de4 3fffeba8 4021a659
3fffeb30: 3fff4568 3fffeba0 3fff1368 4021a874
3fffeb40: 00000000 3fff4c7c 3fff1368 3fffeb90
3fffeb50: 3fffeb64 3fffeba0 3fff1368 402052d1
3fffeb60: 00646d63 7473696c 00727375 726f6f64
3fffeb70: 31007069 312e3239 342e3836 3200362e
3fffeb80: 3836312e 362e342e 00227d22 4021002c
3fffeb90: 402258d8 3fffeba0 000000ff 00000028
3fffeba0: 3fffeb90 3fffeba8 3fffebb8 3fffeb60
3fffebb0: 00000002 3fffeb64 00000000 3fffeb6c
3fffebc0: 00000002 3fffeb73 00000014 4026827b
3fffebd0: 40264822 3ffeea90 40101cbf 3ffeea90
3fffebe0: 00000014 3fff50a8 00000008 3fff5240
3fffebf0: 402637a8 3ffeea90 3ffef220 3fff50a8
3fffec00: 00000000 40252d4f 3ffeffa8 00000000
3fffec10: 00000000 00000002 00000000 3ffeea90
3fffec20: 3fff525a 40105603 402646ef 00000000
3fffec30: ffffffff 00000000 3ffeb4c1 00000000
3fffec40: 4026473e 3ffeea90 3fff2ac4 00000001
3fffec50: 4026484a 3ffeea90 3fff2ac4 3ffeea90
3fffec60: 00000002 00000000 00000020 40101b7a
3fffec70: 3ffeb372 402637f3 3ffeea90 3fff50a8
3fffec80: 00000000 40252d4f 3ffeffa8 4024551c
3fffec90: 00000000 00000002 00000000 3ffeea90
3fffeca0: 3fffeb90 3fffeb8a 3fffeb7f 3fff2e0a
3fffecb0: 3fff4cc4 00000003 3fff2878 4024cc3c
3fffecc0: 402526f4 40244ea9 3fff24d0 3fff2e8c
3fffecd0: 2e323931 2e383631 00362e34 00ac0f00
3fffece0: 00000000 00000000 0000001f 4010585d
3fffecf0: 4000050c 3fff2878 3fff4cc4 00000000
3fffed00: 3fff4d16 3fff4c7c 00000000 402242da
3fffed10: ffffffd0 3ffeffa8 ffffffd0 00000002
3fffed20: 3ffeffa8 00000000 3fffee0c 40212394
3fffed30: 0000002a 3fffee0b 000000ff 3fffee80
3fffed40: 402051b8 3ffef818 402242e4 402242c4
3fffed50: 00000000 3fff4cc4 3fff0f24 40224dd9
3fffed60: 3fff4c7c 3fff310c 00000000 3fff4cc4
3fffed70: 3fff1608 00000454 00000454 00000000
3fffed80: 3fffedf0 0000002a 3fff4464 40224d51
3fffed90: 0000002a 00000000 0000002a 00000000
3fffeda0: 3fff4d0a 40233726 3fff447c 40213806
3fffedb0: 0000002a 00000000 0000002a 00000000
3fffedc0: 3fff4d0a 00000036 3fff0f24 40212199
3fffedd0: 3fff4f0c 3ffeedd8 402113c4 40224cc0
3fffede0: 3fff4c2c 1e0b194b 40211378 40224d18
3fffedf0: 0000000c 4bc6a7f0 0007b0ec 3fff4cc4
3fffee00: 3fff10ad 3fff4464 4bc6a7f0 00000000
3fffee10: 7fffffff 3ffebbf0 40100c16 3fff4cc4
3fffee20: 00000000 3fffee70 3fff0f24 402121cc
3fffee30: 00000000 3fffee70 3fff0f24 40208596
3fffee40: 3fff4f4c 40102c44 3fffc200 00000022
3fffee50: 3ffe0000 1e0b1732 60000600 3fff4f4c
3fffee60: 3fff4cc4 00000000 3fff0f24 40208856
3fffee70: 3fff30a4 3fff30b4 00000020 3fff2710
3fffee80: 3fff2878 3fff2920 3fff4cc4 3fff2710
3fffee90: 3fff4f4c 3fff26f4 3fff26f5 40248998
3fffeea0: 0000006c 40100f54 00000014 0000006c
3fffeeb0: 3fff2878 0604a8c0 0000007c 4021f03c
3fffeec0: 40250000 0000006c 3fffef30 3fff2878
3fffeed0: 3fff2920 3fff4cc4 3fff4ce2 4024d78d
3fffeee0: 00000014 00000000 3fff2878 4021edf8
3fffeef0: 3fffdc80 3fff2e8c 3fff4914 3fff2ed4
3fffef00: 00000008 3fff2878 3fff4cc4 40245495
3fffef10: 3fffdc80 3fff2e8c 3fff4914 402452cb
3fffef20: 4026de4a 3fff2e8c 3fff4914 4026de5b
3fffef30: 3fff4cd4 3fff4cc4 00000002 00000000
3fffef40: 40263583 00000000 3fff4914 4026965f
3fffef50: 40000f49 3fffdab0 3fffdab0 40000f49
3fffef60: 40000e19 40001878 00000004 1be88cad
3fffef70: 3fffff10 aa55aa55 000000ee 40105084
3fffef80: 4010508a 00000004 1be88cad eb5ac902
3fffef90: 4010000d 9c6f6a3b 73f24d9a 3347e2b0
3fffefa0: 40101238 3fffef3c 401011d9 3ffffe58
3fffefb0: 3fffffc0 00000000 00000000 feefeffe
3fffefc0: feefeffe feefeffe feefeffe feefeffe
3fffefd0: feefeffe feefeffe feefeffe feefeffe
3fffefe0: feefeffe feefeffe feefeffe feefeffe
3fffeff0: feefeffe feefeffe feefeffe feefeffe
3ffff000: feefeffe feefeffe feefeffe feefeffe
3ffff010: feefeffe feefeffe feefeffe feefeffe
3ffff020: feefeffe feefeffe feefeffe feefeffe
3ffff030: feefeffe feefeffe feefeffe feefeffe
3ffff040: feefeffe feefeffe feefefæU® Xùz%ü [ INFO ] ESP RFID v1.3.1 Flash real id: 001640E0 Flash real size: 4194304

Flash ide size: 4194304 Flash ide speed: 40000000 Flash ide mode: DIO Flash Chip configuration ok.

[ INFO ] Config file found { "command": "configfile", "network": { "bssid": "00:0F:60:06:98:21", "ssid": "deleted", "wmode": 0, "hide": 0, "pswd": "deleted", "offtime": 0, "dhcp": 1, "ip": "", "subnet": "", "gateway": "", "dns": "", "apip": "192.168.4.1", "apsubnet": "255.255.255.0" }, "hardware": { "readerType": 0, "wgd0pin": 4, "wgd1pin": 5, "sspin": 15, "rfidgain": 112, "wifipin": 255, "rtype": 0, "rpin": 4, "rtime": 400, "ltype": 0, "buttonpin": 5, "doorstatpin": 2, "numrelays": 1, "relay2": { "rtype": 1, "ltype": 0, "rpin": 4, "rtime": 400 }, "relay3": { "rtype": 1, "ltype": 0, "rpin": 4, "rtime": 400 }, "relay4": { "rtype": 1, "ltype": 0, "rpin": 4, "rtime": 400 }, "readertype": 0, "openlockpin": 5 }, "general": { "hostnm": "deleted", "restart": 0, "pswd": "admin" }, "mqtt": { "enabled": 1, "host": "192.168.4.1", "port": 1883, "topic": "maindoor", "user": "", "pswd": "", "syncrate": "60", "mqttlog": 1 }, "ntp": { "server": "pool.ntp.org", "interval": 30, "timezone": 0 }, "logmaintenance": { "enabled": 1, "rolloverkb": "10", "maxlogfilesnumber": 5, "spiffwatch": 1 } } [ INFO ] Trying to setup RFID Hardware

[ INFO ] WiFi STA Connected [ INFO ] RFID SS_PIN: 15 and Gain Factor: 112 [ INFO ] MFRC522 Version: 0x92 = v2.0 [ INFO ] Trying to connect WiFi: DELETED [ INFO ] WiFi BSSID: 0:15:96:6:152:33 [ INFO ] BSSID locked

[ WARN ] Couldn't connect in time [ INFO ] ESP-RFID is running in Fallback AP Mode ` By the way , a great project, if I can get platform.io running I will try and fix some other issues I have found

Steve

[ingeninge]

I had the same issue and set the Serial line speed up to 115200. (main.cpp line ~170)

But I do not get more than 9 Users via MQTT. I get them all over serial debug though.

[dm2302git]

At the moment i've the same problem. Someone find an reason for this issue?

[kimamov]

Me and @dm2302git are trying to fix this for a while now. Seems like its a problem with the small TCP buffer size. The Problem comes from the async-mqtt-client library https://github.com/marvinroger/async-mqtt-client that is used. There are multiple open issues about it and it gets somewhat explained here https://github.com/marvinroger/async-mqtt-client/blob/master/docs/3.-Memory-management.md

Unfortunately I am not much of a low level programmer but usually I would try to solve that with async methods since delaying the while loop or adding retry logic seems to crash the device.

[kimamov]

Okay we figured something out. Replacing the AsyncMqttClient with this version: https://github.com/marvinroger/async-mqtt-client/tree/develop lets you send way more data by using the ESPAsyncTCPbuffer write for sending. You will have to use QOS1 instead of 0 tho.

This was the issue inside the AsyncMqttClient repo https://github.com/marvinroger/async-mqtt-client/issues/212 kleini fixed it here https://github.com/kleini/async-mqtt-client/commit/f1b42054815ad82fed4519bce7febb7f1601560f

[altuntasali]

Okay we figured something out. Replacing the AsyncMqttClient with this version: https://github.com/marvinroger/async-mqtt-client/tree/develop lets you send way more data by using the ESPAsyncTCPbuffer write for sending. You will have to use QOS1 instead of 0 tho.

This was the issue inside the AsyncMqttClient repo marvinroger/async-mqtt-client#212 kleini fixed it here kleini/async-mqtt-client@f1b4205

Unfortunately this didnt solve my problem. I can only get 8 users. I'd really appreciate if you help me

[kimamov]

Okay we figured something out. Replacing the AsyncMqttClient with this version: https://github.com/marvinroger/async-mqtt-client/tree/develop lets you send way more data by using the ESPAsyncTCPbuffer write for sending. You will have to use QOS1 instead of 0 tho. This was the issue inside the AsyncMqttClient repo marvinroger/async-mqtt-client#212 kleini fixed it here kleini/async-mqtt-client@f1b4205

Unfortunately this didnt solve my problem. I can only get 8 users. I'd really appreciate if you help me

Does your device keep sending until it crashes now? Can you send more users than before with that fix? In my case I could send users until the device ran out of memory and crashed with that fix around 12-14. But eventually we switched to another solution by using my server for iteration. Do you use a server for receiving data?

[altuntasali]

Yes it keeps sending and doesnt crash but i could send 8 users before, too. when I check the publish() it returns 0 after 8 messages I use adafruit.io to receive data.

[kimamov]

Yes it keeps sending and doesnt crash but i could send 8 users before, too. when I check the publish() it returns 0 after 8 messages I use adafruit.io to receive data.

are you sure you replaced your AsyncMqttClient (also the debug version) with https://github.com/kleini/async-mqtt-client if so can you try to change QOS1 inside the send user function to 0 and check if only 1 user is send?

[altuntasali]

When i use QoS0 it sends between 7-9 users. QoS 1 sends 9 users now(debug off). Maybe there is a limitation on adafruit.io? But as far as i can test library change did not do anything

[kimamov]

When i use QoS0 it sends between 7-9 users. QoS 1 sends 9 users now(debug off). Maybe there is a limitation on adafruit.io? But as far as i can test library change did not do anything

Hmm no idea then. We used esp8266 but in general its a problem with the available memory. Your TCP Buffer can't be bigger than your devices memory and inside a while loop it gets filled up before getting flushed. There are some build options that increase your devices memory that way you could maybe double the amount but it is not a stable solution.

We fixed the problem by implementing pagination like that

void ICACHE_FLASH_ATTR getUserList(int todo,unsigned int skip=0,unsigned int take=3) {
    String stopic (mqttTopic);
    if (todo == 0)  // Userlist to add to DB
        stopic = stopic + "/accesslist";
    else if (todo == 1) // Just for List
        stopic = stopic + "/send";

    Dir dir = SPIFFS.openDir("/P/");
    #ifdef DEBUG
        Serial.println("[ INFO ] getUserList");
    #endif

    unsigned int i=0;
    unsigned int lastIndex=skip+take-1; // last index for 1 item is 0

    while (i<=lastIndex) {
        if(!dir.next()){ // check if the next file exists
            // if not leave early
            DynamicJsonBuffer jsonBuffer;
            JsonObject &json = jsonBuffer.createObject();
            json["cmd"]="listUserState";
            json["done"]=true;
            json["lastIndex"]=i;

            String mqttBuffer;
            json.printTo(mqttBuffer);
            mqttClient.publish(stopic.c_str(), 0, false, mqttBuffer.c_str()); 
            return;
        }
        if(i>=skip){
            String uid = dir.fileName();
            uid.remove(0, 3);
            File f = SPIFFS.open(dir.fileName(), "r");
            size_t size = f.size();
            std::unique_ptr<char[]> buf(new char[size]);
            f.readBytes(buf.get(), size);
            DynamicJsonBuffer jsonBuffer;
            JsonObject &json = jsonBuffer.parseObject(buf.get());
            if (json.success()) {

                if (mqttClient.connected()) {
                    String mqttBuffer;
                    json.printTo(mqttBuffer);
                    mqttClient.publish(stopic.c_str(), 0, false, mqttBuffer.c_str()); // returns 0 on failure;

                    #ifdef DEBUG
                        Serial.print("[ INFO ] Mqtt Publish:");
                        Serial.println(mqttBuffer);
                    #endif
                }
            }
        }
        i++;

    }
    DynamicJsonBuffer jsonBuffer;
    JsonObject &json = jsonBuffer.createObject();
    json["cmd"]="listUserState";
    json["done"]=false;
    json["lastIndex"]=i; // can be used to figure out from where to continue
    String mqttBuffer;
    json.printTo(mqttBuffer);
    mqttClient.publish(stopic.c_str(), 0, false, mqttBuffer.c_str()); 
}

it gets called like this:

if (strcmp(command, "getuser") == 0){
        #ifdef DEBUG
            Serial.println("[ MARE ] Get User List");
        #endif
        uint8_t skip=0;
        uint8_t take=3;
        if(root.containsKey("skip")){
            skip=root["skip"];
        }
        if(root.containsKey("take")){
            take=root["take"];
        }
        getUserList(0, skip, take);
        return;
    }

    else if (strcmp(command, "listusr") == 0){
        #ifdef DEBUG
            Serial.println("[ INFO ] List users");
        #endif
        uint8_t skip=0;
        uint8_t take=3;
        if(root.containsKey("skip")){
            skip=root["skip"];
        }
        if(root.containsKey("take")){
            take=root["take"];
        }
        getUserList(1, skip, take);
        return;
    }

https://github.com/kantimam/esp-rfid/blob/dev/src/mqtt.esp

your server will call for the next page until it receives done

[altuntasali]

I tried this but i dont think i am doing it right. It always sends same 3 users.

[kimamov]

I tried this but i dont think i am doing it right. It always sends same 3 users.

well you have to ask for the next "page" of users yourself. If you know JS here is the code I use for it.

export async function generateAllReaderKeys(req: Request, res: Response) {
    // tell the reader to send us all keys that are currently stored on it
    if(gettingKeyList){
        return res.status(500).send({message: "already getting the keylist of another reader. try again in a few seconds"})
    }
    try {
        const {readerId}=JSON.parse(req.query.filter as string);
        if(readerId===undefined){
            return res.status(500).send({message: "your filter does not contain a readerId"})
        }
        const readerRepository: Repository<Reader> = getRepository(Reader);
        const reader=await readerRepository.findOne(readerId);

        if(!reader){
            return res.status(404).send({message: `could not find reader with the provided id: ${readerId}`})
        }

        if(!client.connected){
            return res.status(500).send({error: "connection to the MQTT client was lost"})
        }
        const keyBatchLength=3;
        /* TODO: get the reader from the doorid then use its ip to the command */
        client.publish("devnfc", JSON.stringify({
            cmd: "listusr",
            doorip: reader.ip,
            skip: 0,
            take: keyBatchLength  // get the first 3 keys on the controller
        }))

        gettingKeyList=true;

        let timeout=null;
        const keys=[];

        const sendResponse=()=>{ 
            console.log("last answer was to long ago listener removed")
            client.off('message', messageHandler);
            gettingKeyList=false;
            console.log(`keyCount=${keys.length}`)
            res.set('Content-Range', `key ${0}-${keys.length-1}/${keys.length}`)
            return res.send(keys); // send the response and return

        }

        const messageHandler=(topic: string, message: Buffer)=> {
            if(topic==="devnfc/send" || topic==="/devnfc/send"){

                try {
                    const messageString = message.toString()
                    const messageJSON = JSON.parse(messageString)
                    if(messageJSON.cmd==="adduser"){
                        console.log(messageJSON.uid);

                        const keyObj={
                            id: messageJSON.uid,
                            readerIp: reader.ip,
                            readerId: reader.id,
                            uid: messageJSON.uid,
                            name: messageJSON.user,
                            acctype: messageJSON.acctype? 1 : 0,
                            acctype2: messageJSON.acctype2? 1 : 0,
                            acctype3: messageJSON.acctype3? 1 : 0,
                            acctype4: messageJSON.acctype4? 1 : 0,
                            acctype5: messageJSON.acctype5? 1 : 0,
                            acctype6: messageJSON.acctype6? 1 : 0,
                            validUntil: dateFromUnix(messageJSON.validuntil)

                        };
                        keys.push(keyObj); // add the key to the array
                        createReaderKey(keyObj); // create a db entry for the key
                        console.log("timer extended after receiving new key")

                        if(timeout) clearTimeout(timeout); // stop the timer
                        timeout=setTimeout(sendResponse, 3000); // wait another 3 seconds for the next key or listUserState message

                    }else if(messageJSON.cmd==="listUserState"){
                        /* json["cmd"]="listUserState";
                            json["done"]=false;
                            json["lastIndex"]=i;  
                        */
                       if(timeout) clearTimeout(timeout);
                       if(messageJSON.done){
                            // if everything goes well we return the response from here
                            if(timeout) clearTimeout(timeout);
                            return sendResponse();
                       }else{
                            if(timeout) clearTimeout(timeout); 
                            timeout=setTimeout(sendResponse, 3000);

                            client.publish("devnfc", JSON.stringify({ // if done is not true get another batch of keys
                                cmd: "listusr",
                                doorip: reader.ip,
                                skip: keys.length, // start the batch at keys.length
                                take: keyBatchLength  // get the first 3 keys on the controller
                            }))
                       }
                    }

            }
                catch (error) {
                    //console.log(error)
                    console.log("error parsing following content as json: " )
                }
            }
        }

        if(!client.connected){
            return res.status(500).send({error: "connection to the MQTT client was lost"})
        }
        timeout=setTimeout(()=>{ // initialy give the controller 6 seconds to return the first key otherwise there are no and we can return
            console.log("timed out and listener removed")
            client.off('message', messageHandler);
            gettingKeyList=false;
            return res.status(404).send({
                ip: reader.ip,
                keys: keys,
                message: "looks like there are no keys on the reader"
            });

        }, 6000);

        client.on('message', messageHandler)
    } catch (error) {
        return res.status(500).send({error})
    }

}

[altuntasali]

Thanks a lot.