Open philippesikora opened 1 year ago
SimonRafferty
commented
1 year ago I agree, that's a better approach - and close to how I started. I kept having stability problems (contrary to what you'd expect) with the ESP. However, I'm not as familiar with it as you - so yours will more likely work! I'll give it a go when I'm home from work.
philippesikora
commented
1 year ago Here is the sketch with loop() once with reserve() in setup(). ESP32 should be more stable.
//**
// Functional Description // Modules and functions integrated in ESP32-WROOM-32D and ESP32-WROOM-32U. // CPU and Internal Memory // ESP32-D0WD contains a dual-core Xtensa® 32-bit LX6 MCU. The internal memory includes: // • 448 KB of ROM for booting and core functions. // • 520 KB of on-chip SRAM for data and instructions. // • 8 KB of SRAM in RTC, which is called RTC FAST Memory and can be used for data storage; it is accessed // by the main CPU during RTC Boot from the Deep-sleep mode. // • 8 KB of SRAM in RTC, which is called RTC SLOW Memory and can be accessed by the co-processor during the Deep-sleep mode. // 1 Kbit of eFuse: 256 bits are used for the system (MAC address and chip configuration) and the remaining // 768 bits are reserved for customer applications, including flash-encryption and chip-ID.
// Board: ESP32 Dev // Interface into the Solcast Solar forcasting API // https://toolkit.solcast.com.au // You will need to create a FREE account. This will allow a limited number // of forcasts per day. // Once you've registered and entered the details of your solar array, location // etc, it will give you a URL which you can use to get the data. // You'll also get an API key which is effectively your login. // Enter these details below along with your WiFi credentials - and this will // show the solar forcast in half hour intervals for tomorrow as well as a // cumulative total. // The intention of this is to enable automation of your solar Inverter // Battery charging based on solar gain // // Simon Rafferty 2023 //**
//#include
const char ssid = ""; // Replace with your network name const char password = ""; // Replace with your network password const char* apiKey = ""; // Replace with your Solcast API key
String url = "https://api.solcast.com.au/world_pv_power/forecasts?latitude=43.529171&longitude=6.148583&capacity=2.70&tilt=30&azimuth=-135&loss_factor=0.85&install_date=2020-10-26&format=json"; // Replace with the URL given on Solcast. URL is specific to your location etc
String period_end;
String timeStamp;
String sHour;
String sMinute;
String pv_estimate;
bool runOnce = false;
float dEstimate; double Solcast_tomorrow = 0.0;
HTTPClient http;
void setup() {
period_end.reserve(32); // 32 Bytes in memory to save for String manipulation. "2023-04-03T14:00:00.0000000Z" timeStamp.reserve(32); // 32 Bytes in memory to save for String manipulation. "T14:00:00.0000000" sHour.reserve(16); // 16 Bytes sMinute.reserve(16); // 16 Bytes pv_estimate.reserve(16); // 16 Bytes
Serial.begin(115200); WiFi.begin(ssid, password); while (WiFi.status() != WL_CONNECTED) { delay(1000); Serial.println("Connecting to WiFi..."); } Serial.println("Connected to WiFi"); }
void loop() {
if ((WiFi.status() == WL_CONNECTED) && (runOnce == false)) {
http.useHTTP10(true);
http.begin(url);
// Add User-Agent header
http.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Firefox/87.0");
http.addHeader("Accept-Encoding", "gzip, deflate, br");
http.addHeader("Authorization", "Bearer " + String(apiKey));
int httpCode = http.GET();
Serial.print("HTTP response code: ");
Serial.println(httpCode);
if (httpCode > 0) {
DynamicJsonDocument doc(16384); // 1024 *16 -- file size to be cnfirmed 1024 * 16 ???
DeserializationError error = deserializeJson(doc, http.getStream());
if (error) {
Serial.print("deserializeJson() failed: ");
Serial.println(error.c_str());
return;
}
bool bTomorrow = false;
double forecast_tomorrow = 0.0;
int nCount = 0;
for (JsonObject forecast : doc["forecasts"].as<JsonArray>()) {
period_end = forecast["period_end"].as<String>(); // "2023-04-03T14:00:00.0000000Z"
int splitT = period_end.indexOf("T");
int posHour = period_end.indexOf(":");
timeStamp = period_end.substring(splitT, period_end.length()-1); //Time without preceeding T : "T14:00:00.0000000"
sHour = timeStamp.substring(1, 3);
sMinute = timeStamp.substring(4, 6);
pv_estimate = forecast["pv_estimate"].as<String>();
int nHour = sHour.toInt();
int nMinute = sMinute.toInt();
float dEstimate = pv_estimate.toDouble();
int nTMins = nHour*60+nMinute;
if(nTMins < 30) bTomorrow = true; //Crossed midnight, get thext 24h of data
if(bTomorrow && (nCount < 48)) {
//There are 48 readings in 24h - so once the first midnight is detected, get the next 48 readings
nCount++;
Solcast_tomorrow = Solcast_tomorrow + dEstimate / 2;
}
}
} else {
Serial.println("HTTP request failed");
}
http.end();
runOnce = true;
Serial.println();
Serial.print("Solcast_tomorrow "); Serial.print(Solcast_tomorrow); Serial.print("kWh"); Serial.println(); }
// Do nothing }
philippesikora
commented
1 year ago Heap fragmentation occurs when you allocate/deallocate blocks of variable size (typically Strings), which is not the case here for the following code. Code stable with ESP32 -:)
void solcast(void) {
unsigned long startTime; unsigned long endTime; unsigned long routineTime; double dTomorrowTotal_kWh = 0.0;
startTime = millis();
Serial.println("Solcast");
if (WiFi.status() == WL_CONNECTED) {
HTTPClient http;
const size_t authHeaderSize = 50;
char authHeader[authHeaderSize];
http.useHTTP10(true); // Use HTTP/1.0 protocol
http.begin(url); // Make HTTP request
// Add User-Agent header
http.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Firefox/87.0");
http.addHeader("Accept-Encoding", "gzip, deflate, br");
snprintf(authHeader, authHeaderSize, "Bearer %s", apiKey);
http.addHeader("Authorization", authHeader);
int httpCode = http.GET(); /* start connection and send HTTP header */
Serial.print("HTTP response code: "); // if response code = 429 Too Many Requests
Serial.println(httpCode); // 401 lacks valid authentication credentials for the target resource
if (httpCode > 0) {
/*
Data structures 9328 Bytes needed to stores the JSON objects and arrays in memory
Strings 9031 Bytes needed to stores the strings in memory
Total (minimum) 18359 Minimum capacity for the JsonDocument.
Total (recommended) 24576 Including some slack in case the strings change, and rounded to a power of two
*/
DynamicJsonDocument doc(24576); // size: 24576 provided by https://arduinojson.org/v6/assistant/#/step1
DeserializationError error = deserializeJson(doc, http.getStream());
if (error) { // Test if parsing succeeds
Serial.print("deserializeJson() failed: ");
Serial.println(error.c_str());
return;
}
bool bTomorrow = false;
int nCount = 0;
Serial.println("debug_step_1 deserializeJson");
for (JsonObject forecast : doc["forecasts"].as<JsonArray>()) {
// {"forecasts":[{"pv_estimate":0.9028,"pv_estimate10":0.902,"pv_estimate90":0.9028,"period_end":"2023-06-18T15:00:00.0000000Z",
// "period":"PT30M"},{"pv_estimate":0.6931,"pv_estimate10":0.6584450000000001,"pv_estimate90":0.6934,
// "period_end":"2023-06-18T15:30:00.0000000Z","period":"PT30M"}, .....]}
/*
sonObject forecast: This declares a new JsonObject variable called forecast that will be used to iterate
over each element in the "forecasts" array.
"doc" has been parsed, and it contains an array of "forecasts" objects.
The code uses a range-based for loop to iterate through each "forecast" object in the "forecasts" array
Inside the loop, a character array called "period_end" is defined with a maximum size of 30 characters
The "period_end" value from the "forecast" object is extracted and copied into the "period_end" character array using strlcpy
inside the loop for {} , you can access and manipulate the elements of the "forecasts" array using the forecast variable.
Each iteration of the loop will provide a new JsonObject representing an element in the array,
allowing you to access its fields and perform operations on it.
*/
const char* period_end = forecast["period_end"].as<const char*>(); // "period_end":"2023-06-18T20:30:00.0000000Z"
double dEstimate = forecast["pv_estimate"].as<double>(); // "pv_estimate":0.6931
/*
The period_end and pv_estimate values are extracted from the forecast object using forecast["period_end"].as<const char*>()
and forecast["pv_estimate"].as<const char*>() respectively. These values are then assigned to const char* pointers.
*/
int nHour = atoi(period_end + 11); // Atoi() stops reding at the first non-numeric character
int nMinute = atoi(period_end + 14); // Atoi() stops reding at the first non-numeric character
int nTMins = nHour*60+nMinute;
if(nTMins < 30) bTomorrow = true; // Crossed midnight, get thext 24h of data
if(bTomorrow && (nCount < 48)) {
// There are 48 readings in 24h -
// so once the first midnight is detected, get the next 48 readings
nCount++;
dTomorrowTotal_kWh = dTomorrowTotal_kWh + dEstimate / 2;
}
}
} else {
Serial.println("HTTP request failed");
}
http.end();
endTime = millis();
routineTime = (endTime - startTime); // according to test routineTime = 2.3s
Serial.println();
Serial.print("solcast_tomorrow "); Serial.print(dTomorrowTotal_kWh); Serial.print(" kWh"); Serial.println();
Serial.print("Routine_time "); Serial.print(routineTime); Serial.print(" ms"); Serial.println(); }
}
I try to improve the reliability of the code using String class. Passing the result of http.getString() directly to deserializeJson() is inefficient because it would copy the complete response in RAM before parsing. We can do much better by letting ArduinoJson pull the bytes from the HTTP response To do that, we must get HTTPClient’s underlying Stream by calling http.getStream() instead of http.getString(). https://arduinojson.org/v6/how-to/use-arduinojson-with-httpclient/
We can significantly improve this code’s performance by calling String::reserve() before serializeJson() to avoid heap fragmentation when we use String class. Repeated allocations of the same size don’t cause fragmentation; so, we could keep our objects in the heap but always use the same size by calling String::reserve() .
period_end.reserve(1024); timeStamp.reserve(16); sMinute.reserve(16); pv_estimate.reserve(16);
https://cpp4arduino.com/2018/11/21/eight-tips-to-use-the-string-class-efficiently.html https://cpp4arduino.com/2018/11/06/what-is-heap-fragmentation.html
I modified the code as follows. What do you think?
//**
// Functional Description // Modules and functions integrated in ESP32-WROOM-32D and ESP32-WROOM-32U. // CPU and Internal Memory // ESP32-D0WD contains a dual-core Xtensa® 32-bit LX6 MCU. The internal memory includes: // • 448 KB of ROM for booting and core functions. // • 520 KB of on-chip SRAM for data and instructions. // • 8 KB of SRAM in RTC, which is called RTC FAST Memory and can be used for data storage; it is accessed // by the main CPU during RTC Boot from the Deep-sleep mode. // • 8 KB of SRAM in RTC, which is called RTC SLOW Memory and can be accessed by the co-processor during the Deep-sleep mode. // 1 Kbit of eFuse: 256 bits are used for the system (MAC address and chip configuration) and the remaining // 768 bits are reserved for customer applications, including flash-encryption and chip-ID.
// Board: ESP32 Dev // Interface into the Solcast Solar forcasting API // https://toolkit.solcast.com.au // You will need to create a FREE account. This will allow a limited number // of forecasts per day. // Once you've registered and entered the details of your solar array, location // etc, it will give you a URL which you can use to get the data. // You'll also get an API key which is effectively your login. // Enter these details below along with your WiFi credentials - and this will // show the solar forcast in half hour intervals for tomorrow as well as a // cumulative total. // The intention of this is to enable automation of your solar Inverter // Battery charging based on solar gain // // Simon Rafferty 2023 //**
include
include
define ARDUINOJSON_ENABLE_ARDUINO_STRING 1 / need to force the support of String /
include
include
const char ssid = ""; // Replace with your network name const char password = ""; // Replace with your network password const char* apiKey = [Your API Key]; // Replace with your Solcast API key String url = "https://api.solcast.com.au/world_pv_power/forecasts?latitude=43.529171&longitude=6.148583&capacity=2.70&tilt=30&azimuth=-135&loss_factor=0.85&install_date=2020-10-26&format=json";
const char* ntpServer = "pool.ntp.org"; const long gmtOffset_sec = 0; const int daylightOffset_sec = 3600;
String period_end; String timeStamp; String sHour; String sMinute;
String pv_estimate;
void setup() {
period_end.reserve(1024); timeStamp.reserve(16); sMinute.reserve(16); pv_estimate.reserve(16);
Serial.begin(115200); WiFi.begin(ssid, password); while (WiFi.status() != WL_CONNECTED) { delay(1000); Serial.println("Connecting to WiFi..."); } Serial.println("Connected to WiFi");
// Make the HTTP request HTTPClient http; http.useHTTP10(true); http.begin(url);
// Add User-Agent header http.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Firefox/87.0"); http.addHeader("Accept-Encoding", "gzip, deflate, br");
http.addHeader("Authorization", "Bearer " + String(apiKey));
int httpCode = http.GET(); Serial.print("HTTP response code: "); Serial.println(httpCode);
if (httpCode > 0) {
} else { Serial.println("HTTP request failed"); }
http.end();
}
void loop() { // Do nothing }