Problems writing to Client using EthernetWebServer_SSL_STM32 and SdFat libraries

[ExDes99]

Environment: Arduino IDE version: 1.8.13 STM32_MCU_based_boards: 2.0.0 Board: NUCLEO-F767ZI Libraries: STM32duino LwIP: 2.1.2, STM32Ethernet: 1.2.0, EthernetWebServer_SSL_STM32: 1.3.0, SdFat: 2.0.6 System Information: Linux Desktop-Mint-19 5.4.0-42-generic #46~18.04.1-Ubuntu SMP Fri Jul 10 07:21:24 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux

Context: I am attempting to write a file found on an SD card to a connected Ethernet client. The file opens fine, and I have no problems writing to the client. For testing purposes I also write the lines read from the file to the serial port at the same time as being written to the client. What is happening though, is that a larger portion of file contents are displayed in the serial monitor, but the browser only receives about 2900 characters. The actual file, not including the header, sent separately, is about 260000 bytes long. I ran a test sending the file's content via 8000 "client.print(...)" statements and this worked fine.

I have attached code. Anything not necessary to the basic function of the code has been stripped out to give us the minimum required to debug plus additional code for debug purposes. I am using the libraries _EthernetWebServer_SSLSTM32 & SdFat. My code is derived from the WebServer example. No errors are registered during operation.

The link to the SdFat library is: https://github.com/greiman/SdFat

Serial printout at startup:

Start WebServer on NUCLEO_F767ZI, using LAN8742A Ethernet & STM32Ethernet Library
EthernetWebServer_SSL_STM32 v1.3.0
You're connected to the network, IP = 192.168.2.164

Code as follows:

#include "defines.h"
#include "SdFat.h"

/****************************************************************************************************************************
  WebServer.ino - Simple Arduino web server sample for ESP8266 AT-command shield
  For STM32F/L/H/G/WB/MP1 with built-in Ethernet LAN8742A (Nucleo-144, DISCOVERY, etc) or W5x00/ENC28J60 shield/module

  EthernetWebServer_SSL_STM32 is a library for STM32 using the Ethernet shields to run WebServer and Client with/without SSL
  Use SSLClient Library code from https://github.com/OPEnSLab-OSU/SSLClient

  Built by Khoi Hoang https://github.com/khoih-prog/EthernetWebServer_SSL_STM32
  Licensed under MIT license
 *****************************************************************************************************************************/
#define REPLY_HEADER    "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n"

    void Process_Client_Request()
    {
        FsFile      file;
        char        line[500];
        char        Received_Character = '\0';
        uint32_t    Character_Index    =   0;
        uint32_t    n;
        uint32_t    File_Length_Bytes;
        uint32_t    File_Length_Lines;

        // listen for incoming clients
        EthernetClient client = server.available();

        static  ArduinoOutStream cout(Serial);

        if (client)
        {
            // an http request ends with a blank line
            bool currentLineIsBlank = true;

            while (client.connected())
            {
                if (client.available())
                {
                    char c = client.read();
                    Serial.write(c);
                    // if you've gotten to the end of the line (received a newline
                    // character) and the line is blank, the http request has ended,
                    // so you can send a reply
                    if (c == '\n' && currentLineIsBlank)
                    {
                        Serial.println(F("Sending response:\n"));

                        // send a standard http response header
                        // use \r\n instead of many println statements to speedup data send
                        client.print(REPLY_HEADER);
                        Serial.println(F(REPLY_HEADER));

                        // Open the SD card
                        if (!sd.begin(SD_CONFIG)) 
                        {
                            sd.initErrorHalt(&Serial);
                        }

                        // Specify the file to read
                        char fileName[] = "test.htm";

                        // Open the file to READ from the SD card
                        if (!file.open(fileName, O_READ)) 
                        {
                            sd.errorHalt(&Serial, F("open failed"));
                        }

                        File_Length_Bytes = file.fileSize();
                        File_Length_Lines = 0;

                        Serial.print(__DATE__); Serial.print(", "); Serial.println(__TIME__);

                        while ((n = file.fgets(line, sizeof(line))) > 0)
                        {
                            // Write the line to the client
                            client.print(line);

                            // Print the line
                            Serial.print(line);

                            File_Length_Lines++;
                        }
                        file.close();
                        break;
                    }

                    if (c == '\n')
                    {
                        // you're starting a new line
                        currentLineIsBlank = true;
                    }

                    else if (c != '\r')
                    {
                        // you've gotten a character on the current line
                        currentLineIsBlank = false;
                    }
                }
            }
            // give the web browser time to receive the data
            delay(10);

            // close the connection:
            client.stop();
            Serial.println(F("\n\nClient disconnected"));

            Serial.println("\n");
            Serial.print("Total bytes read: "); Serial.println(File_Length_Bytes);
            Serial.print("Total lines read: "); Serial.println(File_Length_Lines);
            Serial.println("");
        }   
    }

Abbreviated contents of the file test.htm

<html><body>
This is line number: 1<br>
This is line number: 2<br>
This is line number: 3<br>
This is line number: 4<br>
This is line number: 5<br>

...

This is line number: 7997<br>
This is line number: 7998<br>
This is line number: 7999<br>
This is line number: 8000<br>
</html></body>

Abbreviated output to the Serial monitor:

<html><body>
This is line number: 1<br>
This is line number: 2<br>
This is line number: 3<br>
This is line number: 4<br>
This is line number: 5<br>

...

This is line number: 198<br>
This is line number: 199<br>
This is line number: 200<br>
This is line number: 201<br>
This is line number: 202<br>

Writes to the serial monitor stop here.

Abbreviated output to the web client:

<html><body>
This is line number: 1<br>
This is line number: 2<br>
This is line number: 3<br>
This is line number: 4<br>
This is line number: 5<br>

...

This is line number: 99<br>
This is line number: 100<br>
This is line number: 101<br>
This is line number: 102<br>
This is line number: 103<br>
This i

Writes to the client stop here.

These are things that I have tried:

1. Different makes & sizes of SD cards using different formats - no change
2. Modified to file.fgets(line, sizeof(line), "\n")) & client.write(line, countOfBytesReadWithfget); - no change
3. I looked into https://github.com/OPEnSLab-OSU/SSLClient#implementation-gotchas. They mention changes to allow for larger output buffer sizes to the Ethernet.h file. I have determined that the code is using the ethernet.h file found here: _Arduino/libraries/STM32duinoLwIP/src/netif/ethernet.h. This file does not have the options to increase the buffer size. Tried going through the various header files referenced but have not been able to find a setting for send buffering.
4. To eliminate possible timing issues I have added a delay of varying lengths to the inner while() loop. These had an adverse effect and were again removed.
5. I have tried client.write(line, n);, client.print(line); and cout << line; methods - identical results using all three.
6. I tried to use the standard Arduino SD library instead of the SdFat library. I also did a client.write(myFile.read()); instead of reading to the 'line' string variable, otherwise code unchanged. Same results - output to the client stops after about 104 of the 8000 line transferred.

[khoih-prog]

Thanks for using the library and I'm impressed at the sophisticated ways you spent to try solving the issue.

However, please realize that this is the MCU world, with many limitations, such as board's resources, many new and not totally tested and matured libraries (STM32Ethernet, LwIP, String, etc.), and we can not expect the similar results as you're doing with PCs (for example : sending too large HTML files, etc.)

This file does not have the options to increase the buffer size. Tried going through the various header files referenced but have not been able to find a setting for send buffering.

The actual file, not including the header, sent separately, is about 260000 bytes long.

Same results - output to the client stops after about 104 of the 8000 line transferred.

With these researches and findings, I suspect many issues, spreading across many libraries / STM32 core

1. Heap Fragmentation when using Arduino String
2. Some severe bugs in STM32Ethernet / LwIP libraries
3. Even STM32 core

I don't think I have enough time as well as reach to spend on this and solve the issue. And I believe you already spent lot of time here.

The notorious issue of sending very large via Ethernet / WiFi has been posted many times and you can search by Googling.

I know that you already post an issue on STM32thernet library as well as STM32 forum to ask for help and advice from many good experts (much better than me) there. Problems writing to Client using EthernetWebServer_SSL_STM32 and SdFat libraries

I'd appreciate it if you can update or post a link so that we all (other users and myself) and benefit from what you experience.

I'm sorry I have to close the issue now as this has nothing to do with this library (the issue persists even you don't use this library) and I won't spend time on this. Good Luck.

Best Regards,

[khoih-prog]

I think the issue is your code (dealing with SD, etc), not the SM32Ethernet library. Check

1. Your code if there is any issue
2. If there is conflict in using SPI bus (SD card + LAN8742A, etc)

I just have some time to test and it seems OK for writing multiple of 10,000 lines from memory (not SD) and OK, using the following simple code

#define REPLY_HEADER    "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n"
#define HTML_DATA_START       "<!DOCTYPE HTML>\r\n"
#define HTML_DATA_BODY        "<h4>Hello World from F767ZI running WebServer!</h4>\r\n"
#define HTML_DATA_END         "</html>\r\n"

void loop()
{
  // listen for incoming clients
  EthernetClient client = server.available();

  if (client)
  {
    Serial.println(F("New client"));
    // an http request ends with a blank line
    bool currentLineIsBlank = true;

    while (client.connected())
    {
      if (client.available())
      {
        char c = client.read();
        Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        if (c == '\n' && currentLineIsBlank)
        {
          Serial.println(F("Sending response"));

          // send a standard http response header
          // use \r\n instead of many println statements to speedup data send
          client.print(REPLY_HEADER);

          client.print(HTML_DATA_START);

#define TOTAL_LINES     10000

          for (uint16_t i = 0; i < TOTAL_LINES; i++)
          {      
            client.print(HTML_DATA_BODY);
            Serial.print("Line # : "); Serial.print(i); Serial.print(" => "); 
            Serial.println(HTML_DATA_BODY);
          }

          client.print(HTML_DATA_END);

          break;
        }

        if (c == '\n')
        {
          // you're starting a new line
          currentLineIsBlank = true;
        }
        else if (c != '\r')
        {
          // you've gotten a character on the current line
          currentLineIsBlank = false;
        }
      }
    }
    // give the web browser time to receive the data
    delay(10);

    // close the connection:
    client.stop();
    Serial.println(F("Client disconnected"));
  }
}

and terminal output

Start WebServer on NUCLEO_F767ZI, using LAN8742A Ethernet & STM32Ethernet Library
EthernetWebServer_SSL_STM32 v1.3.0
[ETHERNET_WEBSERVER] Board : NUCLEO_F767ZI , setCsPin: 10
[ETHERNET_WEBSERVER] Default SPI pinout:
[ETHERNET_WEBSERVER] MOSI: 11
[ETHERNET_WEBSERVER] MISO: 12
[ETHERNET_WEBSERVER] SCK: 13
[ETHERNET_WEBSERVER] SS: 10
[ETHERNET_WEBSERVER] =========================
You're connected to the network, IP = 192.168.2.154
New client
GET / HTTP/1.1
Host: 192.168.2.154
Connection: keep-alive
Cache-Control: max-age=0
DNT: 1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Referer: http://192.168.2.154/
Accept-Encoding: gzip, deflate
Accept-Language: en-GB,en-US;q=0.9,en;q=0.8,vi;q=0.7

Sending response
Line # : 0 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 1 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 2 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 3 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 4 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 5 => <h4>Hello World from F767ZI running WebServer!</h4>

...

Line # : 9989 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9990 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9991 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9992 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9993 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9994 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9995 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9996 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9997 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9998 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9999 => <h4>Hello World from F767ZI running WebServer!</h4>

Client disconnected
New client
GET /favicon.ico HTTP/1.1
Host: 192.168.2.154
Connection: keep-alive
Pragma: no-cache
Cache-Control: no-cache
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36
DNT: 1
Accept: image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8
Referer: http://192.168.2.154/
Accept-Encoding: gzip, deflate
Accept-Language: en-GB,en-US;q=0.9,en;q=0.8,vi;q=0.7

Sending response
Line # : 0 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 1 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 2 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 3 => <h4>Hello World from F767ZI running WebServer!</h4>

...

>Hello World from F767ZI running WebServer!</h4>

Line # : 9996 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9997 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9998 => <h4>Hello World from F767ZI running WebServer!</h4>

Line # : 9999 => <h4>Hello World from F767ZI running WebServer!</h4>

Client disconnected