connectSSL hangs

[SandroSartoni]

Hi,

I am using the ESP_SSLClient library on a LilyGo TCALL v1.4 with a SIM800H module mounted inside. The project I am working on measures a few parameters (air humidity, temperature and soil moisture), then sends them to a server through HTTPS. I can only rely on GSM, since the place where this board will be installed does not have access to WiFi.

Below is the code I have written:

#include "DHT.h"
#include <esp_task_wdt.h>

//////////////////////////////
// Sensor Variables Section //
//////////////////////////////

#define WDT_TIMEOUT   6*60  // Watchdog for three minutes
#define DHTTYPE       DHT11 // We're using DHT 11 for this project
#define DHTPIN        15    // Serial communication for DHT sensor goes through this pin
#define SOILSENS      34    // The analog signal from the soil mosture sensor is fed in this pin
#define MEASUREMENTS  5     // How many measurements should be computed in a measurement cycle

// Define the dht variable holding type and pin information
DHT dht(DHTPIN, DHTTYPE);

// Temperature and air humidity variables are defined here
float temperature[MEASUREMENTS];
float air_humidity[MEASUREMENTS];

// Soil moisture variables defined here
const int air_moisture = 3390;            // This is the minimum value we can get through the soil moisture sensor
const int water_moisture = 1490;          // This is the maximum value we can get through the soil moisture sensor
int soil_moisture_val;                    // This variable will then be scaled wrt the air and water moisture levels
int soil_moisture_percent[MEASUREMENTS];  // Final moisture value holding array

////////////////////////////////
// Internet Variables Section //
////////////////////////////////
// GPRS credentials (leave empty, if not needed)
const char apn[]      = "TM"; // APN (example: internet.vodafone.pt) use https://wiki.apnchanger.org
const char gprsUser[] = "";   // GPRS User
const char gprsPass[] = "";   // GPRS Password

// SIM card PIN (leave empty, if not defined)
const char simPIN[]   = "1503"; 

// Server details
// The server variable can be just a domain name or it can have a subdomain. It depends on the service you are using
const char server[] = <my_server>;   // domain name: example.com, maker.ifttt.com, etc
const int  port = 443;                                                   // server port number

const char token [] = <my_token>;

// TTGO T-Call pins
#define MODEM_RST            5
#define MODEM_PWKEY          4
#define MODEM_POWER_ON       23
#define MODEM_TX             27
#define MODEM_RX             26
#define I2C_SDA              21
#define I2C_SCL              22

// Set serial for debug console (to Serial Monitor, default speed 115200)
#define SerialMon Serial
// Set serial for AT commands (to SIM800 module)
#define SerialAT Serial1

// Configure TinyGSM library
#define TINY_GSM_MODEM_SIM800      // Modem is SIM800
#define TINY_GSM_RX_BUFFER   1024  // Set RX buffer to 1Kb

#include <Wire.h>
#include <TinyGsmClient.h>
#include "ESP_SSLClient.h"

TinyGsm modem(SerialAT);

// I2C for SIM800 (to keep it running when powered from battery)
TwoWire I2CPower = TwoWire(0);

// TinyGSM and ESP_SSLClient for Internet connection
TinyGsmClient client(modem);
ESP_SSLClient ssl_client;

#define uS_TO_S_FACTOR  1000000UL   /* Conversion factor for micro seconds to seconds */
#define TIME_TO_SLEEP   3520        /* Time ESP32 will go to sleep (in seconds) 3520 seconds = 58 minutes, 40 seconds */
#define MAX_RETRY       10          /* How many times a POST request should be attempted before giving up */

#define IP5306_ADDR          0x75
#define IP5306_REG_SYS_CTL0  0x00

////////////////////////////
// USER DEFINED FUNCTIONS //
////////////////////////////
// Function: setPowerBoostKeepOn
bool setPowerBoostKeepOn(int en){
  I2CPower.beginTransmission(IP5306_ADDR);
  I2CPower.write(IP5306_REG_SYS_CTL0);
  if (en) {
    I2CPower.write(0x37); // Set bit1: 1 enable 0 disable boost keep on
  } else {
    I2CPower.write(0x35); // 0x37 is default reg value
  }
  return I2CPower.endTransmission() == 0;
}

// Function: htmlGet
// Performs the get request
boolean htmlGet(const char* URI) {
  if (ssl_client.connect(server, port)) {

    Serial.print("[HTTPS] Upgrade to HTTPS...");
    if (!ssl_client.connectSSL()) {
      Serial.println(" failed");
      return false;
    }

    Serial.println(" ok");

    // HTML response is stored here
    String msg;

    Serial.println("[HTTPS] Send GET request...");
    ssl_client.print(String("GET ") + URI + " HTTP/1.1\r\n");
    ssl_client.print(String("Host: ") + server + "\r\n");
    ssl_client.print("Content-Type: application/json\r\n");
    ssl_client.print(String("Authorization: Bearer ") + token + "\r\n");
    ssl_client.print("Connection: close\r\n\r\n");

    unsigned long ms = millis();
    while (!ssl_client.available() && millis() - ms < 3000) {
        delay(0);
    }
    Serial.println();
    while (ssl_client.available()) {
        msg += (char)ssl_client.read();
    }

      if(msg.substring(9, 12) == "200" || msg.substring(9, 12) == "201") {
      ssl_client.stop();
      return true;
    }

  }
  else
    Serial.println("[HTTPS] Connection to server failed");

  ssl_client.stop();
  return false;

}

// Function: htmlPost
// Performs the post request
boolean htmlPost(const char* URI, const char* payload, int payload_length) {
  if (ssl_client.connect(server, port)) {

    Serial.print("[HTTPS] Upgrade to HTTPS...");
    if (!ssl_client.connectSSL()) {
      Serial.println(" failed");
      return false;
    }

    Serial.println(" ok");

    // HTML response is stored here
    String msg;

    // Start of transmission
    Serial.println("[HTTPS] Send POST request...");
    ssl_client.print(String("POST ") + URI + " HTTP/1.1\r\n");
    ssl_client.print(String("Host: ") + server + "\r\n");
    ssl_client.print("Content-Type: application/json\r\n");
    ssl_client.print(String("Authorization: Bearer ") + token + "\r\n");
    ssl_client.print("Content-Length: ");
    ssl_client.print(payload_length);
    ssl_client.print("\r\n\r\n");
    ssl_client.print(payload);

    // Check for correctness
    unsigned long ms = millis();
    while (!ssl_client.available() && millis() - ms < 10000L) {
      delay(10);
    }

    while (ssl_client.available()) {
      msg += (char)ssl_client.read();
    }

    if(msg.substring(9, 12) == "200" || msg.substring(9, 12) == "201") {
      ssl_client.stop();
      return true;
    }
    Serial.println(msg);  
  }
  else
    Serial.println("[HTTPS] Connection to server failed");

  ssl_client.stop();
  return false;

}

///////////////////
// Setup Section //
///////////////////
void setup() {
  // Set serial monitor debugging window baud rate to 115200
  Serial.begin(115200);
  Serial.println("[SETUP] Setting up peripherals...");

  // Add watchdog
  esp_task_wdt_init(WDT_TIMEOUT, true); // Enable panic so ESP32 restarts
  esp_task_wdt_add(NULL);               // Add current thread to WDT watch

  // Initialize humidity and temperature sensor
  dht.begin();

  // Setup code for modem
  // Start I2C communication
  I2CPower.begin(I2C_SDA, I2C_SCL, 400000);

  // Keep power when running from battery
  bool isOk = setPowerBoostKeepOn(1);
  SerialMon.println(String("[SETUP] IP5306 KeepOn ") + (isOk ? "OK" : "FAIL"));

  // Set modem reset, enable, power pins
  pinMode(MODEM_PWKEY, OUTPUT);
  pinMode(MODEM_RST, OUTPUT);
  pinMode(MODEM_POWER_ON, OUTPUT);
  digitalWrite(MODEM_PWKEY, LOW);
  digitalWrite(MODEM_RST, LOW);
  digitalWrite(MODEM_POWER_ON, HIGH);
  delay(1000);
  digitalWrite(MODEM_RST, HIGH);
  delay(1000);

  // Set GSM module baud rate and UART pins
  SerialAT.begin(115200, SERIAL_8N1, MODEM_RX, MODEM_TX);
  delay(3000);

  //secure_presentation_layer.setCACert(hass_root_ca);

  // Restart SIM800 module, it takes quite some time
  // To skip it, call init() instead of restart()
  SerialMon.println("[TinyGSM] Initializing modem...");
  modem.restart();
  // use modem.init() if you don't need the complete restart

  String modemInfo = modem.getModemInfo();
  SerialMon.print("[TinyGSM] Modem Info: ");
  SerialMon.println(modemInfo);

  // Unlock your SIM card with a PIN if needed
  if (strlen(simPIN) && modem.getSimStatus() != 3 ) {
    modem.simUnlock(simPIN);
  }

  // Check network presence
  SerialMon.print("[TinyGSM] Waiting for network...");
  if (!modem.waitForNetwork()) {  
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" success");

  if (modem.isNetworkConnected()) { SerialMon.println("[TinyGSM] Network connected"); }

  // Connect to APN
  SerialMon.print(F("[TinyGSM] Connecting to "));
  SerialMon.print(apn);
  if (!modem.gprsConnect(apn, gprsUser, gprsPass)) {
    SerialMon.println(" fail");
    delay(10000);
    return;
  }
  SerialMon.println(" success");

  if (modem.isGprsConnected()) { SerialMon.println("[TinyGSM] GPRS connected"); }

  // Ignore server ssl certificate verification
  ssl_client.setInsecure();

  // Set the receive and transmit buffers size in bytes for memory allocation (512 to 16384).
  ssl_client.setBufferSizes(4096 /* rx */, 512 /* tx */);

  /** Call setDebugLevel(level) to set the debug
   * esp_ssl_debug_none = 0
   * esp_ssl_debug_error = 1
   * esp_ssl_debug_warn = 2
   * esp_ssl_debug_info = 3
   * esp_ssl_debug_dump = 4
   */
  ssl_client.setDebugLevel(4);

  // Assign the basic client
  // Due to the basic_client pointer is assigned, to avoid dangling pointer, basic_client should be existent
  // as long as it was used by ssl_client for transportation.
  ssl_client.setClient(&client, false);

  // Configure the wake up source as timer wake up  
  esp_sleep_enable_timer_wakeup(TIME_TO_SLEEP * uS_TO_S_FACTOR);

  Serial.println("[SETUP] Weather station is ready to start!");

}

//////////////////
// LOOP SECTION //
//////////////////
void loop() {

  // The goal here is to compute several measurements and average them to get a measurement as accurate as possible.
  // For this reason, 30 measurements are taken before the data is sent to the server.
  for(int i = 0; i < MEASUREMENTS; i++) {

    SerialMon.print("[LOOP] Measurement number ");
    SerialMon.println(i);

    // Reading temperature or humidity takes about 250 milliseconds!
    air_humidity[i] = dht.readHumidity();
    // Read temperature as Celsius
    temperature[i] = dht.readTemperature();

    // Check if any reads failed and exit early (to try again).
    while (isnan(air_humidity[i]) || isnan(temperature[i])) {
      // Reading temperature or humidity takes about 250 milliseconds!
      air_humidity[i] = dht.readHumidity();
      // Read temperature as Celsius
      temperature[i] = dht.readTemperature();
    }

    // Compute soil moisture value
    soil_moisture_val = analogRead(SOILSENS);
    soil_moisture_percent[i] = map(soil_moisture_val, air_moisture, water_moisture, 0, 100);

    // Cut values exceeding bounds
    if(soil_moisture_percent[i] > 100)
      soil_moisture_percent[i] = 100;
    else if(soil_moisture_percent[i] < 0)
      soil_moisture_percent[i] = 0;

    // Sleep for 4 seconds (to account for measurement time)
    delay(4000);
  }

  // Compute the average of the previous measurements
  float air_humidity_avg = 0.0;
  float temperature_avg = 0.0;
  float soil_moisture_avg = 0.0;

  // First add...
  for(int i = 0; i < MEASUREMENTS; i++) {
    air_humidity_avg += air_humidity[i];
    temperature_avg += temperature[i];
    soil_moisture_avg += soil_moisture_percent[i];
  }

  // Then divide by number of measurements
  air_humidity_avg /= MEASUREMENTS;
  temperature_avg /= MEASUREMENTS;
  soil_moisture_avg /= MEASUREMENTS;

  // Print relevant info
  SerialMon.println("[LOOP] Measurement phase is complete. Following measurements will be transmitted:");
  SerialMon.print(F("[LOOP] Air humidity: "));
  SerialMon.print(air_humidity_avg);
  SerialMon.println(F(" %"));
  SerialMon.print(F("[LOOP] Temperature: "));
  SerialMon.print(temperature_avg);
  SerialMon.println(F(" °C"));
  SerialMon.print(F("[LOOP] Soil humidity: "));
  SerialMon.print(soil_moisture_avg);
  SerialMon.println(F(" %"));

  // Compute heat index in Celsius (isFahreheit = false)
  //float hic = dht.computeHeatIndex(temperature, air_humidity, false);
  //Serial.print(F("Heat index: "));
  //Serial.print(hic);
  //Serial.println(F("°C"));

  // Prepare the json file to send it
  char convert_meas[5];
  String json_data; //= "{\"state\": ";
  uint8_t retry = 0;

  // Air Humidity
  sprintf(convert_meas, "%.1f", air_humidity_avg);
  json_data = String("{\"state\": ") + convert_meas + ", \"attributes\": {\"unit_of_measurement\": \"%\"}}";

  while(!htmlPost("/api/states/sensor.weather_sensor_humidity_home", json_data.c_str(), json_data.length()) && (retry < MAX_RETRY)) {
    Serial.println("[LOOP] HTTPS post failed, retrying in 5 seconds");
    retry++;
    delay(5000);    
  }

  Serial.println("[LOOP] Humidity data successfully transmitted");
  delay(5000);

  // Temperature
  sprintf(convert_meas, "%.1f", temperature_avg);
  retry = 0;
  json_data = String("{\"state\": ") + convert_meas + ", \"attributes\": {\"unit_of_measurement\": \"°C\"}}"; 

  while(!htmlPost("/api/states/sensor.weather_sensor_temp_home", json_data.c_str(), json_data.length()) && (retry < MAX_RETRY)) {
    Serial.println("[LOOP] HTTPS post failed, retrying in 5 seconds");
    retry;
    delay(5000);
  }

  Serial.println("[LOOP] Temperature data successfully transmitted");
  delay(5000);

  // Soil moisture
  sprintf(convert_meas, "%.1f", soil_moisture_avg);
  retry = 0;
  json_data = String("{\"state\": ") + convert_meas + ", \"attributes\": {\"unit_of_measurement\": \"%\"}}";

  while(!htmlPost("/api/states/sensor.weather_sensor_soil_moisture_home", json_data.c_str(), json_data.length()) && (retry < MAX_RETRY)) {
    Serial.println("[LOOP] HTTPS post failed, retrying in 5 seconds");
    retry++;
    delay(5000);
  }

  Serial.println("[LOOP] Soil moisture data successfully transmitted");
  delay(5000);

  Serial.println("[LOOP] Device now is going to sleep...");

  // This will put the ESP32 to sleep
  esp_deep_sleep_start();

}

The issue I am encountering is that from time to time the connectSSL() function fails, and that can happen multiple times in a row thus stalling the whole transmission process. Setting the debug option to 4, this is what I read when such failures occur:

INFO.mConnectBasicClient: Basic client connected! INFO.mConnectSSL: Start connection. INFO.mConnectSSL: Wait for SSL handshake. INFO.mUpdateEngine: State RECVREC INFO.mUpdateEngine: State RECVREC INFO.mRunUntil: SSL state changed. INFO.mRunUntil: State RECVREC INFO.mRunUntil: Expected bytes count: 5 ERROR.mRunUntil: SSL internals timed out! ERROR.mConnectSSL: Failed to initlalize the SSL layer. ERROR.mConnectSSL: Unknown error code.

Is it something related to my code/settings? How can I fix this problem? Sometimes the board will be able to connect at the first try and everything goes smoothly, sometimes it takes few retries and other times is completely stuck. I am using the latest version of this library.

[mobizt]

You may have to increase TINY_GSM_RX_BUFFER in GSM library.

The rx buffer setting via setBufferSizes should be larger than your data + some cipher suite.

This depends on the fragmentation supports on server.

If server does not support fragmentation, this buffer should set to 16k which is possible in ESP32.

[SandroSartoni]

Hi @mobizt, I have updated both TINY_GSM_RX_BUFFER and setBuffersizes function to have 16384 bytes for the RX buffer but the error still occurs. It does not happen for every transmission, sometimes it goes through just fine while other times it shows the error I reported in the previous post. On another note, a new error that I had never seen before came up in one of the POST requests:

ERROR.mUpdateEngine: Error reading bytes from basic client. Write Error: 0 ERROR.mConnectSSL: Failed to initlalize the SSL layer. ERROR.mConnectSSL: Unknown error code

[mobizt]

You're starting SSL connection (SSL handshake) without stopping the current connection which there are some data that is waiting for read.

You have to review your code logic to make it work properly.

[SandroSartoni]

Thank you for your reply. I am not sure what you mean by stopping the current connection: the code I reported is taken from one of the sketch examples reported in the library, the "http upgrade" one.

[mobizt]

I mean session or TCP session hope you understand.

The session was not re-used in the examples, and it works normally.

The error showed that you start a new connection while the session was opened and the BearSSL engine is in writing or reading for incoming data state. You have to terminate the sever connection before starting new connection.

This is your code logic and timing issue.

[mobizt]

The http upgrade is not needed for normal application unless IMAP and SMTP applications.

For GSM module, the simple example is here. https://github.com/mobizt/ESP_SSLClient/blob/main/examples/gsm/gsm.ino