Downloading .bin files - Githubissues

SiavashSkynet commented 2 years ago

Hi,

So I would like to download a .bin file (exported .bin by Arduino IDE). Unfortunately download operation does not work properly. When it is here :

  while (readLength < contentLength && client.connected() /*&& millis() - timeout < 10000L*/) {
    int i = 0;
    while (client.available()) {
      if (!file.print((client.read())))
      {
          Serial.println("error writing character to SPIFFS");
      }
      readLength++;
      if (readLength % (contentLength / 13) == 0) {
        printPercent(readLength, contentLength);
      }
      timeout = millis();
    }
  }

The program out put on Serial monitor is always showing error writing character to SPIFFS and keep proceeding the operation. Look at the image below:

Anyway after that the file is not downloaded successfully. And in other word I could not update my firmware form (/update.bin) .

I did try FileDownload example so many times and put lot's of hours on it. But did not gain any good results and stuck here. I put my whole code below FMI:


// Select your modem:
#define TINY_GSM_MODEM_SIM800
// #define TINY_GSM_MODEM_SIM808
// #define TINY_GSM_MODEM_SIM900
// #define TINY_GSM_MODEM_A6
// #define TINY_GSM_MODEM_A7
// #define TINY_GSM_MODEM_M590
// #define TINY_GSM_MODEM_ESP8266
// #define TINY_GSM_MODEM_XBEE

// Increase RX buffer if needed
#define TINY_GSM_RX_BUFFER 1024

//#define NO_FS_GLOBALS
#include <FS.h>
#include <TinyGsmClient.h>
#include <CRC32.h>

// Uncomment this if you want to see all AT commands
//#define DUMP_AT_COMMANDS

// Set serial for debug console (to the Serial Monitor, default speed 115200)
#define SerialMon Serial

// Use Hardware Serial on Mega, Leonardo, Micro
//#define SerialAT Serial1

#define MODEM_RST 1
#define MODEM_TX 2
#define MODEM_RX 3

// or Software Serial on Uno, Nano
#include <SoftwareSerial.h>
SoftwareSerial SerialAT(MODEM_RX, MODEM_TX); // RX, TX

// Your GPRS credentials
// Leave empty, if missing user or pass
const char apn[]  = "at&t";  // replace with your apn
const char user[] = "";
const char pass[] = "";

// Server details
const char server[] = "exampleserver.com"; // replace with your server name
const int  port = 80;

const char resource[]  = "/blink.bin"; // my program is a simple blink on LED_BUILTIN pin, so u can use it as well
uint32_t knownCRC32    = 0x6f50d767;
uint32_t knownFileSize = 1024;   // In case server does not send it

#ifdef DUMP_AT_COMMANDS
  #include <StreamDebugger.h>
  StreamDebugger debugger(SerialAT, SerialMon);
  TinyGsm modem(debugger);
#else
  TinyGsm modem(SerialAT);
#endif

TinyGsmClient  client(modem);

void setup() {
  // Set console baud rate
  SerialMon.begin(19200);
  delay(10);
  Serial.println("Start of sketch ...");

  if (!SPIFFS.begin())
  {
      Serial.println("SPIFFS Mount Failed");
      return;
  }
  SPIFFS.format();
//  listDir(SPIFFS, "/", 0);

  // Set GSM module baud rate
  SerialAT.begin(19200);
  delay(3000);

  pinMode(MODEM_RST, OUTPUT);
  digitalWrite(MODEM_RST, LOW);
  delay(5);
  digitalWrite(MODEM_RST, HIGH);
  delay(100);
  digitalWrite(MODEM_RST, LOW);

  // Restart takes quite some time
  // To skip it, call init() instead of restart()
  SerialMon.println(F("Initializing modem..."));
  modem.restart();

  String modemInfo = modem.getModemInfo();
  SerialMon.print(F("Modem: "));
  SerialMon.println(modemInfo);

  // Unlock your SIM card with a PIN
  modem.simUnlock("1234");
}

void printPercent(uint32_t readLength, uint32_t contentLength) {
  // If we know the total length
  if (contentLength != -1) {
    SerialMon.print("\r ");
    SerialMon.print((100.0 * readLength) / contentLength);
    SerialMon.print('%');
  } else {
    SerialMon.println(readLength);
  }
}

bool sim_unlock(char* pin)
{   
        bool unlocked = false;
        int status = modem.getSimStatus();
        Serial.println("Check sim pin status to unlock: %d \n");
        Serial.println(status);
        if (status == SIM_LOCKED)
        {
            Serial.println("Unlocking sim\n");
            bool unlocked = modem.simUnlock(pin);
            Serial.println( unlocked);
            return unlocked;
      }
        return unlocked;
}

void loop() {
  SerialMon.print(F("Waiting for network..."));
  if (!modem.waitForNetwork()) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" OK");

  SerialMon.print(F("Connecting to "));
  SerialMon.print(apn);
  if (!modem.gprsConnect(apn, user, pass)) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" OK");

  SerialMon.print(F("Connecting to "));
  SerialMon.print(server);
  if (!client.connect(server, port)) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" OK");

  // Make a HTTP GET request:
  client.print(String("GET ") + resource + " HTTP/1.1\r\n");
  client.print(String("Host: ") + server + "\r\n\r\n\r\n\r\n");
  client.print("Connection: close\r\n\r\n\r\n\r\n");

  long timeout = millis();
  while (client.available() == 0) {
    if (millis() - timeout > 5000L) {
      SerialMon.println(F(">>> Client Timeout !"));
      client.stop();
      delay(10000L);
      return;
    }
  }

  SerialMon.println(F("Reading response header"));
  uint32_t contentLength = knownFileSize;

  File file = SPIFFS.open("/update.bin", "a+");

  while (client.connected()) {
    String line = client.readStringUntil('\n');
    line.trim();
    SerialMon.println(line);    // Uncomment this to show response header
    line.toLowerCase();
    if (line.startsWith("content-length:")) {
      contentLength = line.substring(line.lastIndexOf(':') + 1).toInt();
    } else if (line.length() == 0) {
      break;
    }
  }

  SerialMon.println(F("Reading response data"));
  timeout = millis();
  uint32_t readLength = 0;
  CRC32 crc;

  unsigned long timeElapsed = millis();
  printPercent(readLength, contentLength);
  while (readLength < contentLength && client.connected() /*&& millis() - timeout < 10000L*/) {
    int i = 0;
    while (client.available()) {
      if (!file.print((client.read())))
      {
          Serial.println("error writing character to SPIFFS");
      }
      readLength++;
      if (readLength % (contentLength / 13) == 0) {
        printPercent(readLength, contentLength);
      }
      timeout = millis();
    }
  }
  printPercent(readLength, contentLength);
  timeElapsed = millis() - timeElapsed;
  SerialMon.println();

  // Shutdown

  client.stop();
  SerialMon.println(F("Server disconnected"));

  modem.gprsDisconnect();
  SerialMon.println(F("GPRS disconnected"));

  float duration = float(timeElapsed) / 1000;

  SerialMon.println();
  SerialMon.print("Content-Length: ");   SerialMon.println(contentLength);
  SerialMon.print("Actually read:  ");   SerialMon.println(readLength);
  SerialMon.print("Calc. CRC32:    0x"); SerialMon.println(crc.finalize(), HEX);
  SerialMon.print("Known CRC32:    0x"); SerialMon.println(knownCRC32, HEX);
  SerialMon.print("Duration:       ");   SerialMon.print(duration); SerialMon.println("s");

  Serial.println("starting Update after 3 seconds  ");
  for (int i = 0; i < 3; i++)
  {
      Serial.print(String(i) + "...");
      delay(1000);
  }

  updateFromFS();

  // Do nothing forevermore
  while (true) {
    delay(1000);
  }
}

void updateFromFS()
{
    File updateBin = SPIFFS.open("/update.bin", "r");
    if (updateBin)
    {

        size_t updateSize = updateBin.size();

        if (updateSize > 0)
        {
            Serial.println("start of updating");
            performUpdate(updateBin, updateSize);
        }
        else
        {
            Serial.println("Error, file is empty");
        }

        updateBin.close();

        // whe finished remove the binary from sd card to indicate end of the process
        //fs.remove("/update.bin");
    }
    else
    {
        Serial.println("can't open update.bin");
    }
}

void performUpdate(Stream &updateSource, size_t updateSize)
{
    if (Update.begin(updateSize))
    {
        size_t written = Update.writeStream(updateSource);
        if (written == updateSize)
        {
            Serial.println("writings : " + String(written) + " successfully");
        }
        else
        {
            Serial.println("only writing : " + String(written) + "/" + String(updateSize) + ". Retry?");
        }
        if (Update.end())
        {
            Serial.println("OTA accomplished!");
            if (Update.isFinished())
            {
                Serial.println("OTA ended. restarting!");
                ESP.restart();
            }
            else
            {
                Serial.println("OTA didn't finish? something went wrong!");
            }
        }
        else
        {
            Serial.println("Error occured #: " + String(Update.getError()));
        }
    }
    else
    {
        Serial.println("without enough space to do OTA");
    }
}

SiavashSkynet commented 2 years ago

FMI: I am using ESP8266 . I got no issue with ESP32. maybe because of update.h in core of esp32 in espressif . any tips ?

SiavashSkynet commented 2 years ago

Well it was not library issue and So I should close it.

zark-zeugan commented 2 years ago

Hi,

So I would like to download a .bin file (exported .bin by Arduino IDE). Unfortunately download operation does not work properly. When it is here :

  while (readLength < contentLength && client.connected() /*&& millis() - timeout < 10000L*/) {
    int i = 0;
    while (client.available()) {
      if (!file.print((client.read())))
      {
          Serial.println("error writing character to SPIFFS");
      }
      readLength++;
      if (readLength % (contentLength / 13) == 0) {
        printPercent(readLength, contentLength);
      }
      timeout = millis();
    }
  }

The program out put on Serial monitor is always showing error writing character to SPIFFS and keep proceeding the operation. Look at the image below:

Anyway after that the file is not downloaded successfully. And in other word I could not update my firmware form (/update.bin) .

I did try FileDownload example so many times and put lot's of hours on it. But did not gain any good results and stuck here. I put my whole code below FMI:

// Select your modem:
#define TINY_GSM_MODEM_SIM800
// #define TINY_GSM_MODEM_SIM808
// #define TINY_GSM_MODEM_SIM900
// #define TINY_GSM_MODEM_A6
// #define TINY_GSM_MODEM_A7
// #define TINY_GSM_MODEM_M590
// #define TINY_GSM_MODEM_ESP8266
// #define TINY_GSM_MODEM_XBEE

// Increase RX buffer if needed
#define TINY_GSM_RX_BUFFER 1024

//#define NO_FS_GLOBALS
#include <FS.h>
#include <TinyGsmClient.h>
#include <CRC32.h>

// Uncomment this if you want to see all AT commands
//#define DUMP_AT_COMMANDS

// Set serial for debug console (to the Serial Monitor, default speed 115200)
#define SerialMon Serial

// Use Hardware Serial on Mega, Leonardo, Micro
//#define SerialAT Serial1

#define MODEM_RST 1
#define MODEM_TX 2
#define MODEM_RX 3

// or Software Serial on Uno, Nano
#include <SoftwareSerial.h>
SoftwareSerial SerialAT(MODEM_RX, MODEM_TX); // RX, TX

// Your GPRS credentials
// Leave empty, if missing user or pass
const char apn[]  = "at&t";  // replace with your apn
const char user[] = "";
const char pass[] = "";

// Server details
const char server[] = "exampleserver.com"; // replace with your server name
const int  port = 80;

const char resource[]  = "/blink.bin"; // my program is a simple blink on LED_BUILTIN pin, so u can use it as well
uint32_t knownCRC32    = 0x6f50d767;
uint32_t knownFileSize = 1024;   // In case server does not send it

#ifdef DUMP_AT_COMMANDS
  #include <StreamDebugger.h>
  StreamDebugger debugger(SerialAT, SerialMon);
  TinyGsm modem(debugger);
#else
  TinyGsm modem(SerialAT);
#endif

TinyGsmClient  client(modem);

void setup() {
  // Set console baud rate
  SerialMon.begin(19200);
  delay(10);
  Serial.println("Start of sketch ...");

  if (!SPIFFS.begin())
  {
      Serial.println("SPIFFS Mount Failed");
      return;
  }
  SPIFFS.format();
//  listDir(SPIFFS, "/", 0);

  // Set GSM module baud rate
  SerialAT.begin(19200);
  delay(3000);

  pinMode(MODEM_RST, OUTPUT);
  digitalWrite(MODEM_RST, LOW);
  delay(5);
  digitalWrite(MODEM_RST, HIGH);
  delay(100);
  digitalWrite(MODEM_RST, LOW);

  // Restart takes quite some time
  // To skip it, call init() instead of restart()
  SerialMon.println(F("Initializing modem..."));
  modem.restart();

  String modemInfo = modem.getModemInfo();
  SerialMon.print(F("Modem: "));
  SerialMon.println(modemInfo);

  // Unlock your SIM card with a PIN
  modem.simUnlock("1234");
}

void printPercent(uint32_t readLength, uint32_t contentLength) {
  // If we know the total length
  if (contentLength != -1) {
    SerialMon.print("\r ");
    SerialMon.print((100.0 * readLength) / contentLength);
    SerialMon.print('%');
  } else {
    SerialMon.println(readLength);
  }
}

bool sim_unlock(char* pin)
{   
        bool unlocked = false;
        int status = modem.getSimStatus();
        Serial.println("Check sim pin status to unlock: %d \n");
        Serial.println(status);
        if (status == SIM_LOCKED)
        {
            Serial.println("Unlocking sim\n");
            bool unlocked = modem.simUnlock(pin);
            Serial.println( unlocked);
            return unlocked;
      }
        return unlocked;
}

void loop() {
  SerialMon.print(F("Waiting for network..."));
  if (!modem.waitForNetwork()) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" OK");

  SerialMon.print(F("Connecting to "));
  SerialMon.print(apn);
  if (!modem.gprsConnect(apn, user, pass)) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" OK");

  SerialMon.print(F("Connecting to "));
  SerialMon.print(server);
  if (!client.connect(server, port)) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" OK");

  // Make a HTTP GET request:
  client.print(String("GET ") + resource + " HTTP/1.1\r\n");
  client.print(String("Host: ") + server + "\r\n\r\n\r\n\r\n");
  client.print("Connection: close\r\n\r\n\r\n\r\n");

  long timeout = millis();
  while (client.available() == 0) {
    if (millis() - timeout > 5000L) {
      SerialMon.println(F(">>> Client Timeout !"));
      client.stop();
      delay(10000L);
      return;
    }
  }

  SerialMon.println(F("Reading response header"));
  uint32_t contentLength = knownFileSize;

  File file = SPIFFS.open("/update.bin", "a+");

  while (client.connected()) {
    String line = client.readStringUntil('\n');
    line.trim();
    SerialMon.println(line);    // Uncomment this to show response header
    line.toLowerCase();
    if (line.startsWith("content-length:")) {
      contentLength = line.substring(line.lastIndexOf(':') + 1).toInt();
    } else if (line.length() == 0) {
      break;
    }
  }

  SerialMon.println(F("Reading response data"));
  timeout = millis();
  uint32_t readLength = 0;
  CRC32 crc;

  unsigned long timeElapsed = millis();
  printPercent(readLength, contentLength);
  while (readLength < contentLength && client.connected() /*&& millis() - timeout < 10000L*/) {
    int i = 0;
    while (client.available()) {
      if (!file.print((client.read())))
      {
          Serial.println("error writing character to SPIFFS");
      }
      readLength++;
      if (readLength % (contentLength / 13) == 0) {
        printPercent(readLength, contentLength);
      }
      timeout = millis();
    }
  }
  printPercent(readLength, contentLength);
  timeElapsed = millis() - timeElapsed;
  SerialMon.println();

  // Shutdown

  client.stop();
  SerialMon.println(F("Server disconnected"));

  modem.gprsDisconnect();
  SerialMon.println(F("GPRS disconnected"));

  float duration = float(timeElapsed) / 1000;

  SerialMon.println();
  SerialMon.print("Content-Length: ");   SerialMon.println(contentLength);
  SerialMon.print("Actually read:  ");   SerialMon.println(readLength);
  SerialMon.print("Calc. CRC32:    0x"); SerialMon.println(crc.finalize(), HEX);
  SerialMon.print("Known CRC32:    0x"); SerialMon.println(knownCRC32, HEX);
  SerialMon.print("Duration:       ");   SerialMon.print(duration); SerialMon.println("s");

  Serial.println("starting Update after 3 seconds  ");
  for (int i = 0; i < 3; i++)
  {
      Serial.print(String(i) + "...");
      delay(1000);
  }

  updateFromFS();

  // Do nothing forevermore
  while (true) {
    delay(1000);
  }
}

void updateFromFS()
{
    File updateBin = SPIFFS.open("/update.bin", "r");
    if (updateBin)
    {

        size_t updateSize = updateBin.size();

        if (updateSize > 0)
        {
            Serial.println("start of updating");
            performUpdate(updateBin, updateSize);
        }
        else
        {
            Serial.println("Error, file is empty");
        }

        updateBin.close();

        // whe finished remove the binary from sd card to indicate end of the process
        //fs.remove("/update.bin");
    }
    else
    {
        Serial.println("can't open update.bin");
    }
}

void performUpdate(Stream &updateSource, size_t updateSize)
{
    if (Update.begin(updateSize))
    {
        size_t written = Update.writeStream(updateSource);
        if (written == updateSize)
        {
            Serial.println("writings : " + String(written) + " successfully");
        }
        else
        {
            Serial.println("only writing : " + String(written) + "/" + String(updateSize) + ". Retry?");
        }
        if (Update.end())
        {
            Serial.println("OTA accomplished!");
            if (Update.isFinished())
            {
                Serial.println("OTA ended. restarting!");
                ESP.restart();
            }
            else
            {
                Serial.println("OTA didn't finish? something went wrong!");
            }
        }
        else
        {
            Serial.println("Error occured #: " + String(Update.getError()));
        }
    }
    else
    {
        Serial.println("without enough space to do OTA");
    }
}

Hi! Could you share if this method is working? I am trying to do the same thing on my esp32 with a sim800l module. However, it does not seem to be working.

SiavashSkynet commented 2 years ago

Hi, That works properly for me. If you have any issues please check these items before going forward:

[ ] .bin file has been created for ESP32.
[ ] choose proper board on arduino tools bar. ( for this example I offer ESP32 dev module)
[ ] check that the host can respond to the HTTP request.

zark-zeugan commented 2 years ago

I created the .bin file using Sketch -> Export compiled Binary option.
Yes, the board selected is ESP32 Dev Module.
I am using firebase storage and using TinyGsmClientSecure for my client request to modem for HTTPS communication and it gives me a proper response if I check it externally.

The problem I am having is that the content length from the server is 267648 however when I print the content length from the file it shows 267008 or something that is always less than the content length. and after that I get the following error:

Writes : 265596 successfully
E (106913) esp_image: invalid segment length 0xffff0001
Error occured #: 9

What I want to ask is this:

If the server hosting the .bin could have to do anything with this?
Why am I unable to write the complete response bin data to the file?
Does applying external voltage have anything to do with this?

Also, here is my code for reference : My Code

SiavashSkynet commented 2 years ago

To be honest I have not used firebase library yet, So I should study more and test it to give you a good answer about that. But before that I suggest you to change .bin file and use a simple and more smaller file like blink.bin . After that if the problem still exists change the way you connect to server and use wifiClient instead of using GPRS to get sure about the server. And finally check the file system. you should prepare enough space for the OTA.

zark-zeugan commented 2 years ago

To be honest I have not used firebase library yet, So I should study more and test it to give you a good answer about that. But before that I suggest you to change .bin file and use a simple and more smaller file like blink.bin . After that if the problem still exists change the way you connect to server and use wifiClient instead of using GPRS to get sure about the server. And finally check the file system. you should prepare enough space for the OTA.

@SiavashSkynet Can you suggest any free servers for me to test the HTTP query?

SiavashSkynet commented 2 years ago

Sorry I do not know any websites who does support HTTP instead of HTTPS for download. I did search a lot and did not find any. I used my own server for my usage.

vshymanskyy / TinyGSM

Downloading .bin files #630