after 1 hour aprox, ESP32 doesn´t read changes in Real Tieme Database (RTDB)

fjmduran commented 1 year ago

Hello, Describe the bug Until this weekend everything work OK, but yesterday I realized that ESP32 did not update when data change in real time database (RTDB). Today I have checked and the behaviour is the same, but I have been able to see ESP32 leave update change in RTDB after second token refresh.

To Reproduce Steps to reproduce the behavior:

Load sketch
Wait until second token refresh time
To check ESP32 does not read updating RTDB

Expected behavior I hope ESP32 always can read changes in RTDB

Screenshots I have attached screenshot of Arduino serial monitor

IDE and its version:

Arduino v1.8.15
Library Firebase-ESP32 v4.0.3

Code

#include <WiFi.h>
#include <FirebaseESP32.h>
#include <HTTPClient.h>
#include <TimeLib.h>
#include <esp_int_wdt.h>
#include <esp_task_wdt.h>

// Provide the token generation process info.
#include <addons/TokenHelper.h>

// Provide the RTDB payload printing info and other helper functions.
#include <addons/RTDBHelper.h> //TODO

#define FIREBASE_HOST "https://*****.europe-west1.firebasedatabase.app"
#define API_KEY "*****"
#define USER_EMAIL "*****"
#define USER_PASSWORD "*****"

#define WIFI_SSID "*****"
#define WIFI_PASSWORD "*****"

// Your Domain name with URL path or IP address with path
String serverName = "https://*****.cloudfunctions.net/"; // function for send ESP32 info
String ubicationId = "/test";
String microId = "/porton";

#define NTP_SERVER "pool.ntp.org"

#define BlinkLED 12
#define rele01 15
#define strokeEnd 14
#define strokeEndLed 13

int period = 1000;
unsigned long time_now = 0;

const int millisecondsDoorOpened = 1000;
int intSendESPInfo = 3600 * 12;
int contSendESPInfo = intSendESPInfo; // para que nada más iniciar la placa envíe su info
int timeOutCounter = 0;
bool openStrokeEnd = false;

// Define the Firebase Data object
FirebaseData fbdo;
FirebaseData stream;

// Define the FirebaseAuth data for authentication data
FirebaseAuth auth;

// Define the FirebaseConfig data for config data
FirebaseConfig config;

FirebaseJson updateData;
FirebaseJson json;

void setup()
{
    pinMode(BlinkLED, OUTPUT);
    pinMode(rele01, OUTPUT);
    pinMode(strokeEndLed, OUTPUT);
    pinMode(strokeEnd, INPUT_PULLUP);

    // Apago los relés
    digitalWrite(rele01, true);

    Serial.begin(115200);
    // connect to WiFi
    Serial.printf("Conectando a %s ", WIFI_SSID);
    WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
    while (WiFi.status() != WL_CONNECTED)
    {
        delay(500);
        Serial.print(".");
    }
    Serial.println(" CONECTADO A INTERNET");

    // Assign the project host and api key (required)
    config.api_key = API_KEY;

    // Assign the user sign in credentials
    auth.user.email = USER_EMAIL;
    auth.user.password = USER_PASSWORD;

    /* Assign the RTDB URL (required) */
    config.database_url = FIREBASE_HOST;

    /* Assign the callback function for the long running token generation task */
    config.token_status_callback = tokenStatusCallback; // see addons/TokenHelper.h

    // Initialize the library with the Firebase authen and config.
    Firebase.begin(&config, &auth);

    // Optional, set AP reconnection in setup()
    Firebase.reconnectWiFi(true);

    Firebase.setDoubleDigits(5);

    if (!Firebase.beginStream(stream, ubicationId))
        Serial.printf("sream begin error, %s\n\n", stream.errorReason().c_str());

    /*     // Optional, set number of error retry
        Firebase.setMaxRetry(fbdo, 10);

        // Optional, set number of error resumable queues
        Firebase.setMaxErrorQueue(fbdo, 30);

        // Optional, set the size of HTTP response buffer
        // Prevent out of memory for large payload but data may be truncated and can't determine its type.
        fbdo.setResponseSize(8192);    */

    /*  Firebase.setStreamCallback(fbdo, streamCallback, streamTimeoutCallback);
     if (!Firebase.beginStream(fbdo, path))
     {
         // Could not begin stream connection, then print out the error detail
         Serial.println(fbdo.errorReason());
     } */
}

void loop()
{
    if (millis() > time_now + period)
    {
        runEachPeriod();
    }

    if (Firebase.ready() && millis() > time_now + period)
    {
        if (!Firebase.readStream(stream))
        {
            Serial.printf("stream read error, %s\n\n", stream.errorReason().c_str());
            upResetCounter();
        }

        if (stream.streamTimeout())
        {
            // Stream timeout occurred
            Serial.println("Stream timeout, resume streaming...");
            upResetCounter();
        }

        if (stream.streamAvailable())
        {
            Serial.println("Stream data...");
            Serial.println(stream.dataType());
            if (stream.dataType() == "json")
            {
                toRunStreamData();
            }
            else
            {
                Serial.println(stream.stringData());
            }
        }
    }
}

void runEachPeriod()
{
    time_now = millis();
    if (WiFi.status() != WL_CONNECTED)
        return;

    bailaLed();
    checkSendESPInfo();
    checkStokeEnd();
}

void checkSendESPInfo()
{
    if (contSendESPInfo >= intSendESPInfo)
    {
        contSendESPInfo = 0;
        const int freeMemory = ESP.getFreeHeap();
        String body = "{\"ubicationId\":\"" + ubicationId + "\",\"microId\":\"" + microId + "\",\"esp32Information\":{\"memory\":" + String(freeMemory) + ",\"timeOutCounter\":" + timeOutCounter + ",\"esp32Time\":\"" + getESP32Time() + "\"}}";
        Serial.println(body);
        // sendHTTP("updateESP32Information", body);
    }
    contSendESPInfo++;
}

void checkStokeEnd()
{
    bool openStrokeEndNow = digitalRead(strokeEnd);

    if (openStrokeEnd != openStrokeEndNow && openStrokeEndNow)
    {
        Serial.println("Send warning");
        String body = "{\"ubicationId\":\"" + ubicationId + "\"}";
        // sendHTTP("warningDevice", body); TODO
    }
    if (openStrokeEnd != openStrokeEndNow)
    {
        updateBoolVariableRTDB("/porton", "strokeEnd", openStrokeEndNow);
    }
    digitalWrite(strokeEndLed, openStrokeEndNow);
    openStrokeEnd = openStrokeEndNow;
}

void bailaLed()
{
    digitalWrite(BlinkLED, !digitalRead(BlinkLED));
}

void openDoor(String deviceToModify)
{
    if (deviceToModify == "/porton")
    {
        digitalWrite(rele01, false);
    }

    delay(millisecondsDoorOpened);

    if (deviceToModify == "/porton")
    {
        digitalWrite(rele01, true);
    }

    updateBoolVariableRTDB(deviceToModify, "openDoor", false);
}

void updateBoolVariableRTDB(String deviceToModify, String fieldName, bool value)
{
    FirebaseJson json;
    json.set(fieldName, value);
    if (Firebase.updateNode(fbdo, ubicationId + "/" + deviceToModify, json))
    {
        Serial.println(deviceToModify + "/" + fieldName + " to " + value);
    }
    else
    {
        Serial.println(deviceToModify + "/" + fieldName + ": Error changing value to " + value);
        Serial.println(fbdo.errorReason());
    }
}

void toRunStreamData()
{
    FirebaseJson *json = stream.to<FirebaseJson *>();
    checkDoor(*json, stream.dataPath());
}

boolean checkDoor(FirebaseJson json, String deviceToModify)
{
    FirebaseJsonData jsonData;
    json.get(jsonData, "openDoor");
    if (jsonData.success)
    {
        if (jsonData.boolValue)
        {
            openDoor(deviceToModify);
        }
        return true;
    }
    return false;
}

void sendHTTP(String endPointFunction, String body)
{
    if (WiFi.status() == WL_CONNECTED)
    {
        static WiFiClient wifi;
        HTTPClient httpClient;

        httpClient.begin(serverName + endPointFunction); // Specify the URL

        // GET
        // int httpCode = httpClient.GET(); // Make the request

        // POST
        httpClient.addHeader("Content-Type", "application/json"); // Specify content-type header
        int httpCode = httpClient.POST(body);                     // Make the request

        if (httpCode > 0)
        { // Check for the returning code
            if (httpCode == HTTP_CODE_OK)
            {
                Serial.println(httpCode);
                String payload = httpClient.getString();
                Serial.println(payload);
            }
            else
            {
                Serial.printf("[HTTP] GET... failed, error: %s\n", httpClient.errorToString(httpCode).c_str());
            }
        }
        else
        {
            Serial.println("Error on HTTP request");
            Serial.printf("Firewall, host?");
        }
        httpClient.end(); // Free the resources
    }
    else
    {
        Serial.println("WiFi Disconnected");
    }
}

String getESP32Time()
{
    time_t t = now();
    String _day = static_cast<String>(day(t));
    String _hour = static_cast<String>(hour(t));
    String _minute = static_cast<String>(minute(t));
    String _second = static_cast<String>(second(t));
    String date = _day + "-" + _hour + ":" + _minute + ":" + _second;
    return date;
}

void hard_restart()
{
    Serial.println("REINICIO FORZADO");
    esp_task_wdt_init(1, true);
    esp_task_wdt_add(NULL);
    while (true)
        ;
}

void upResetCounter()
{
    Serial.println(timeOutCounter++);
    if (timeOutCounter >= 1000)
    {
        timeOutCounter = 0;
        hard_restart();
    }
}

mobizt commented 1 year ago

I can run your code and the stream works normally but you should avoid using String object and string literal joining with library string argument because this programming syntax is for Arduino String only.

if (Firebase.updateNode(fbdo, ubicationId + "/" + deviceToModify, json))

should be

if (Firebase.updateNode(fbdo, String(ubicationId + "/" + deviceToModify), json))

mobizt commented 1 year ago

The String and string literal joining is not support in all my libraries that use MB_String class.

This will give unexpected result in your code above and is what you called bug.

fjmduran commented 1 year ago

Thanks for your time. I have updated my code with your suggestions:

I have deleted String object and string literal joining
I check free memory and never is low by 100k
I have added line number to debug message But it has same behaviour.

You can see serial monitor output here

Mi code now is:


#include <WiFi.h>
#include <FirebaseESP32.h>
#include <HTTPClient.h>
#include <TimeLib.h>
#include <esp_int_wdt.h>
#include <esp_task_wdt.h>

// Provide the token generation process info.
#include <addons/TokenHelper.h>

// Provide the RTDB payload printing info and other helper functions.
#include <addons/RTDBHelper.h>

#define FIREBASE_HOST "https://****rtdb.europe-west1.firebasedatabase.app"
#define API_KEY "****"
#define USER_EMAIL "****"
#define USER_PASSWORD "****"

#define WIFI_SSID "****"
#define WIFI_PASSWORD "****"

// Your Domain name with URL path or IP address with path
String serverName = "https://europe-west2****.cloudfunctions.net/"; 
String ubicationId = "/test";
String microId = "/porton";

#define NTP_SERVER "pool.ntp.org"

#define BlinkLED 12
#define rele01 15
#define strokeEnd 14
#define strokeEndLed 13

int period = 1000;
unsigned long time_now = 0;

const int millisecondsDoorOpened = 1000;
int intSendESPInfo = 3600 * 12;
int contSendESPInfo = intSendESPInfo; // para que nada más iniciar la placa envíe su info
int timeOutCounter = 0;
bool openStrokeEnd = false;

// Define the Firebase Data object
FirebaseData fbdo;
FirebaseData stream;

// Define the FirebaseAuth data for authentication data
FirebaseAuth auth;

// Define the FirebaseConfig data for config data
FirebaseConfig config;

FirebaseJson updateData;
FirebaseJson json;

void setup()
{
    pinMode(BlinkLED, OUTPUT);
    pinMode(rele01, OUTPUT);
    pinMode(strokeEndLed, OUTPUT);
    pinMode(strokeEnd, INPUT_PULLUP);

    // Apago los relés
    digitalWrite(rele01, true);

    Serial.begin(115200);
    // connect to WiFi
    Serial.printf("Conectando a %s ", WIFI_SSID);
    WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
    while (WiFi.status() != WL_CONNECTED)
    {
        delay(500);
        Serial.print(".");
    }
    Serial.println(" CONECTADO A INTERNET");

    // Assign the project host and api key (required)
    config.api_key = API_KEY;

    // Assign the user sign in credentials
    auth.user.email = USER_EMAIL;
    auth.user.password = USER_PASSWORD;

    /* Assign the RTDB URL (required) */
    config.database_url = FIREBASE_HOST;

    /* Assign the callback function for the long running token generation task */
    config.token_status_callback = tokenStatusCallback; // see addons/TokenHelper.h

    // Initialize the library with the Firebase authen and config.
    Firebase.begin(&config, &auth);

    // Optional, set AP reconnection in setup()
    Firebase.reconnectWiFi(true);

    Firebase.setDoubleDigits(5); // TODO

    if (!Firebase.beginStream(stream, String(ubicationId + microId)))
    {
        String message = "sream begin error, " + stream.errorReason();
        printInConsole(message, __LINE__);
    }
}

void loop()
{
    if (millis() > time_now + period)
    {
        runEachPeriod();
    }

    if (Firebase.ready() && millis() > time_now + period) // TODO
    {
        if (!Firebase.readStream(stream))
        {
            String message ="stream read error, %s\n\n" + stream.errorReason();
            printInConsole(message, __LINE__);
            upResetCounter();
        }

        if (stream.streamTimeout())
        {
            // Stream timeout occurred
            String message ="Stream timeout, resume streaming...";
            printInConsole(message, __LINE__);
            upResetCounter();
        }

        if (stream.streamAvailable())
        {
            printInConsole("Stream data...", __LINE__);
            if (stream.dataType() == "json")
            {
                toRunStreamData();
            }
            else
            {
                printInConsole(stream.stringData(), __LINE__);
            }
        }
    }
}

void runEachPeriod()
{
    time_now = millis();
    if (WiFi.status() != WL_CONNECTED)
        return;

    bailaLed();
    checkSendESPInfo();
    checkStokeEnd();
}

void checkSendESPInfo()
{
    if (contSendESPInfo >= intSendESPInfo)
    {
        contSendESPInfo = 0;
        const int freeMemory = ESP.getFreeHeap();
        String body = "{\"ubicationId\":\"" + ubicationId + "\",\"microId\":\"" + microId + "\",\"esp32Information\":{\"memory\":" + String(freeMemory) + ",\"timeOutCounter\":" + timeOutCounter + ",\"esp32Time\":\"" + getESP32Time() + "\"}}";
        printInConsole(body, __LINE__);
        // sendHTTP("updateESP32Information", body);
    }
    contSendESPInfo++;
}

void printInConsole(String message, int lineNumber)
{
    const String freeMemory = String(ESP.getFreeHeap());
    Serial.println("[Line " + String(lineNumber) + "]"+"[Free memory " + freeMemory + "][Time " +getESP32Time()+ "] -> " + message);
}

void checkStokeEnd()
{
    bool openStrokeEndNow = digitalRead(strokeEnd);

    if (openStrokeEnd != openStrokeEndNow && openStrokeEndNow)
    {
        printInConsole("Send warning", __LINE__);
        String body = "{\"ubicationId\":\"" + ubicationId + "\"}";
        // sendHTTP("warningDevice", body); TODO
    }
    if (openStrokeEnd != openStrokeEndNow)
    {
        updateBoolVariableRTDB("strokeEnd", openStrokeEndNow);
    }
    digitalWrite(strokeEndLed, openStrokeEndNow);
    openStrokeEnd = openStrokeEndNow;
}

void bailaLed()
{
    digitalWrite(BlinkLED, !digitalRead(BlinkLED));
}

void openDoor()
{

    digitalWrite(rele01, false);

    delay(millisecondsDoorOpened);

    digitalWrite(rele01, true);

    updateBoolVariableRTDB("openDoor", false);
}

void updateBoolVariableRTDB(String fieldName, bool value)
{
    FirebaseJson json;
    json.set(fieldName, value);
    if (Firebase.updateNode(fbdo, String(ubicationId + "/" + microId), json))
    {
        String message = microId + "/" + fieldName + " to " + value;
        printInConsole(message, __LINE__);
    }
    else
    {
        String message = microId + "/" + fieldName + ": Error changing value to " + value;
        printInConsole(message, __LINE__);
        printInConsole(fbdo.errorReason(), __LINE__);
    }
}

void toRunStreamData()
{
    FirebaseJson *json = stream.to<FirebaseJson *>();
    checkDoor(*json);
}

boolean checkDoor(FirebaseJson json)
{
    FirebaseJsonData jsonData;
    json.get(jsonData, "openDoor");
    if (jsonData.success)
    {
        if (jsonData.boolValue)
        {
            openDoor();
        }
        return true;
    }
    return false;
}

void sendHTTP(String endPointFunction, String body)
{
    if (WiFi.status() == WL_CONNECTED)
    {
        static WiFiClient wifi;
        HTTPClient httpClient;

        httpClient.begin(serverName + endPointFunction); // Specify the URL

        // GET
        // int httpCode = httpClient.GET(); // Make the request

        // POST
        httpClient.addHeader("Content-Type", "application/json"); // Specify content-type header
        int httpCode = httpClient.POST(body);                     // Make the request

        if (httpCode > 0)
        { // Check for the returning code
            if (httpCode == HTTP_CODE_OK)
            {

                printInConsole(String(httpCode), __LINE__);
                String payload = httpClient.getString();
                printInConsole(payload, __LINE__);
            }
            else
            {
                String message = "[HTTP] GET... failed, error: " + httpClient.errorToString(httpCode);
                printInConsole(message, __LINE__);
            }
        }
        else
        {
            printInConsole("Error on HTTP request", __LINE__);
            printInConsole("Firewall, host?", __LINE__);
        }
        httpClient.end(); // Free the resources
    }
    else
    {
        printInConsole("WiFi Disconnected", __LINE__);
    }
}

String getESP32Time()
{
    time_t t = now();
    String _day = static_cast<String>(day(t));
    String _hour = static_cast<String>(hour(t));
    String _minute = static_cast<String>(minute(t));
    String _second = static_cast<String>(second(t));
    String date = _day + "-" + _hour + ":" + _minute + ":" + _second;
    return date;
}

void hard_restart()
{
    printInConsole("REINICIO FORZADO", __LINE__);
    esp_task_wdt_init(1, true);
    esp_task_wdt_add(NULL);
    while (true)
        ;
}

void upResetCounter()
{
    if (timeOutCounter >= 1000)
    {
        timeOutCounter = 0;
        hard_restart();
    }
}```

Please, you see anything else?

Regards,

mobizt commented 1 year ago

Are you new to this Firebase library.

You may don't understand the stream operation of library correctly.

You must play with example until you understand the stream behavior in all stream event cases before adapt it.

At first stream connection or reconnection after timed out, library sends the http request to server with additional headers to tell server that this connection is in stream mode and server will send the response payload back which is all data at the streaming path.

Library will fire the streamAvialible with whole data as it get from sever which is not data from the changes in database.

Then if the changes occurred in your database under the streaming path, server will push that changed data to devices.

You must use these functions to determine the stream event, its path and data.

https://github.com/mobizt/Firebase-ESP32/blob/6722bef924d06477043006dcc71132111019e88f/examples/DataChangesListener/NoCallback/NoCallback.ino#L146-L162

mobizt commented 1 year ago

As long as this function stream.streamTimeout() returns false, your device was connected to server normally and stream won't be missed.

This web client example shows the performance of library and Firebase that guarantee no data missed during stream.

You can force token to expire in 20 seconds with this config

config.signer.preRefreshSeconds = 5600 - 20;

mobizt commented 1 year ago

THIS IS FOR YOUR INFORMATION

The library was intensively tested 24/7 for years and I strongly confirm that it works normally in all properly use cases.

As library developer and maintainer, I'm so busy and can't help someone to code or find the bug in the program but provide the full documentation and examples that user can learn and test.

Then before you open the issue, try and follow the examples and make sure you understand the concepts of Firebase and the usage of the library.

Don't ignore the debug printing during test as most functions return boolean status of operation. The fbdo.errorReason() will give you the error detail.

Before process the data from stream or response payload, print it all to make sure you get what you expected.

If you are sure that it is library issue, you should provide the minimum, clean (no 3rd library), clear but complete code that I can test to reproduce the issue.

If you are not sure, start with the Discussions instead.

fjmduran commented 1 year ago

Thanks for your time again.

I am running basic example and I think I have been able to find my problem. After second token refresh, the event type is auth_revoked.

I have projects with more one years using your library and everthing was OK until this weekend.

Do you know what I must do for solve it? My RTDB rules are:

I have search about auth_revoked but it is unsuccess.

Regards,

mobizt commented 1 year ago

You may need to upgrade to blaze plan when quota limits reached.

fjmduran commented 1 year ago

I have already blaze plan

mobizt commented 1 year ago

Firebase may limit your account due to some security reasons.

You can contact the Firebase support team or create the new project.

Don't ask me about Firebase policy about security, privacy and limitations.

fjmduran commented 1 year ago

OK, thanks

mobizt / Firebase-ESP32

after 1 hour aprox, ESP32 doesn´t read changes in Real Tieme Database (RTDB) #237

The library was intensively tested 24/7 for years and I strongly confirm that it works normally in all properly use cases.

Don't ignore the debug printing during test as most functions return boolean status of operation. The `fbdo.errorReason()` will give you the error detail.

Before process the data from stream or response payload, print it all to make sure you get what you expected.

mobizt / Firebase-ESP32

after 1 hour aprox, ESP32 doesn´t read changes in Real Tieme Database (RTDB) #237

The library was intensively tested 24/7 for years and I strongly confirm that it works normally in all properly use cases.

Don't ignore the debug printing during test as most functions return boolean status of operation. The fbdo.errorReason() will give you the error detail.

Before process the data from stream or response payload, print it all to make sure you get what you expected.

Don't ignore the debug printing during test as most functions return boolean status of operation. The `fbdo.errorReason()` will give you the error detail.