search

esp8266 / Arduino

ESP8266 core for Arduino
GNU Lesser General Public License v2.1
16.05k stars 13.33k forks source link

can not open WiFiClientSecure to one particular site #2856

Closed ghost closed 6 years ago

ghost commented 7 years ago

Basic Infos

Hardware

Hardware: Adafruit HUZZAH ?ESP-12? Core Version: ?2.3.0

Description

Hello. I can not connect to sdo.gsfc.nasa.gov with https. Many other sites such as google.com work fine. How can I figure out more what is going on with this one site? Thanks much.

Settings in IDE

Module: ?Adafruit HUZZAH ESP8266? Flash Size: ?4MB/1MB? CPU Frequency: ?80Mhz? Flash Mode: ?qio? Flash Frequency: ?40Mhz? Upload Using: ?OTA / SERIAL? Reset Method: ?ck / nodemcu?

Sketch

``

include

include

include

include

include

include

static char ssid[] = "clearskyinstitute"; // your network SSID (name) static char pass[] = "from651plan"; // your network password

void setup() { Serial.begin (115200);

if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("no shield");
    while (1)
       ESP.wdtFeed();
}

// we only want station mode, not access too
WiFi.mode(WIFI_STA);

Serial.println ("connecting to WiFi");
WiFi.begin (ssid, pass);                            // non-blocking, poll with status()
uint32_t t0 = millis();
uint32_t timeout = 15000UL;
while (WiFi.status() != WL_CONNECTED) {
    ESP.wdtFeed();
    if (millis() > t0 + timeout) {
         Serial.println ("no wifi");
         while(1);
              ESP.wdtFeed();
    }
    delay(1000);
}

Serial.println ("trying connection");
ESP.wdtFeed();
WiFiClientSecure sdo_client;
// if (!sdo_client.connect("google.com", 443)) {
if (!sdo_client.connect("sdo.gsfc.nasa.gov", 443)) {
    Serial.println("connection failed");
    while (1)
        ESP.wdtFeed();
}

Serial.println ("ok");

}

void loop() { ESP.wdtFeed(); }

``

messages here



connecting to WiFi
trying connection
connection failed
ghost commented 7 years ago

Hi, me again. In case it sheds any light on this, I tried retrieving the file I want from this site using curl -v. I've pasted the -v output here in case it helps. Thanks again.

`csiimac: curl -v 'https://sdo.gsfc.nasa.gov/assets/img/latest/f_211_193_171_170.jpg' > x.jpg

rohit-rcrohit7 commented 7 years ago

for ssl support (https website) , you need to include websites ssl certificate code , i.e. fingerprint. for example

https://sdo.gsfc.nasa.gov fingerprint is 8D:6B:9B:BD:AC:E4:55:A8:D1:CA:54:67:6F:22:FE:9D:1A:F1:2E:70

I would suggest you to use library and see their examples . https://github.com/esp8266/Arduino/tree/master/libraries/ESP8266HTTPClient

ghost commented 7 years ago

Thanks thorburn1. I was unaware of the HTTPClient class. But still no joy:

`#include

include

include

include

include

static char ssid[] = "clearskyinstitute"; // your network SSID (name) static char pass[] = "from651plan"; // your network password

void setup() { Serial.begin (115200);

Serial.print ("heap 1: "); Serial.println (ESP.getFreeHeap());

if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("no shield");
    while (1)
       ESP.wdtFeed();
}

// we only want station mode, not access too
WiFi.mode(WIFI_STA);

Serial.println ("connecting to WiFi");
WiFi.begin (ssid, pass);                            // non-blocking, poll with status()
uint32_t t0 = millis();
uint32_t timeout = 15000UL;
while (WiFi.status() != WL_CONNECTED) {
    ESP.wdtFeed();
    if (millis() > t0 + timeout) {
         Serial.println ("no wifi");
         while(1);
              ESP.wdtFeed();
    }
    delay(1000);
}

Serial.print ("heap 2: "); Serial.println (ESP.getFreeHeap());

Serial.println ("trying connection");
HTTPClient sdo_http;
ESP.wdtFeed();
if (!sdo_http.begin ("https://sdo.gsfc.nasa.gov/assets/img/latest/f_211_193_171_170.jpg",
            "8D:6B:9B:BD:AC:E4:55:A8:D1:CA:54:67:6F:22:FE:9D:1A:F1:2E:70")) {
    Serial.println ("begin fails");
    while (1)
        ESP.wdtFeed();
}
Serial.print ("heap 3: "); Serial.println (ESP.getFreeHeap());

int get_status = sdo_http.GET();
ESP.wdtFeed();
if (get_status < 0) {
    Serial.print ("GET: "); Serial.println (sdo_http.errorToString(get_status));
    while (1) 
      ESP.wdtFeed();
}

WiFiClient sdo_client = sdo_http.getStream();
while (!sdo_client.available()) {
    if (!sdo_client.connected()) {
        sdo_client.stop();
        Serial.println ("disconnect");
        while (1)
          ESP.wdtFeed();
    }
    Serial.print((char)sdo_client.read());
}

Serial.print ("heap 4: "); Serial.println (ESP.getFreeHeap());

}

void loop() { ESP.wdtFeed(); } `

Output:

heap 1: 41416 connecting to WiFi heap 2: 40872 trying connection heap 3: 40568 GET: connection refused

rohit-rcrohit7 commented 7 years ago

include

include

include

include

include

static char ssid[] = "clearskyinstitute"; // your network SSID (name) static char pass[] = "from651plan"; // your network password

void setup() { Serial.begin (115200);

Serial.print ("heap 1: "); Serial.println (ESP.getFreeHeap());

if (WiFi.status() == WL_NO_SHIELD) { Serial.println("no shield"); while (1) ESP.wdtFeed(); }

// we only want station mode, not access too WiFi.mode(WIFI_STA);

Serial.println ("connecting to WiFi"); WiFi.begin (ssid, pass); // non-blocking, poll with status() uint32_t t0 = millis(); uint32_t timeout = 15000UL; while (WiFi.status() != WL_CONNECTED) { ESP.wdtFeed(); if (millis() > t0 + timeout) { Serial.println ("no wifi"); while(1); ESP.wdtFeed(); } delay(1000); }

Serial.print ("heap 2: "); Serial.println (ESP.getFreeHeap());

Serial.println ("trying connection"); HTTPClient sdo_http; ESP.wdtFeed(); if (!sdo_http.begin ("https://sdo.gsfc.nasa.gov/assets/img/latest/f_211_193_171_170.jpg", "8D 6B 9B BD AC E4 55 A8 D1 CA 54 67 6F 22 FE 9D 1A F1 2E 70")) { Serial.println ("begin fails"); while (1) ESP.wdtFeed(); } Serial.print ("heap 3: "); Serial.println (ESP.getFreeHeap());

int get_status = sdo_http.GET(); ESP.wdtFeed(); if (get_status < 0) { Serial.print ("GET: "); Serial.println (sdo_http.errorToString(get_status)); while (1) ESP.wdtFeed(); }

WiFiClient sdo_client = sdo_http.getStream(); while (!sdo_client.available()) { if (!sdo_client.connected()) { sdo_client.stop(); Serial.println ("disconnect"); while (1) ESP.wdtFeed(); } Serial.print((char)sdo_client.read()); }

Serial.print ("heap 4: "); Serial.println (ESP.getFreeHeap()); }

void loop() { ESP.wdtFeed(); }

rohit-rcrohit7 commented 7 years ago

Initial problem of yours was error connection failed ; Now connection refused error came from the server itself.So you managed to connect to server , but probably could not authenticate.Therefore HTTP GET response code from server was connection refused. Try the code I edited above and let me know .Also enable debug mode , deug mode :all

ghost commented 7 years ago

I looks like the only change in your version is changing the colons in the fingerprints to blanks. Sorry but it makes no difference.

Regarding debug, I would try it if I knew how, can you elaborate? I found this page https://github.com/esp8266/Arduino/blob/master/doc/Troubleshooting/debugging.md but my IDE does not have these options (even in English :-). I am running Arduino IDE 1.8.1 on macOS.

I tried several other https sites and they all work fine once I learned how to get and use their fingerprints. Evidently there's something strange about sdo.gsfc.nasa.gov. This is an important site for my application, I hope you can help me figure out what the problem is.

A few more questions along the way please?

  1. why do I need to fool with fingerprints at all? The command line program curl can get https pages without knowing fingerprints at all. I accept it means the site could be a spoof but I'm willing to take that chance in this case.

  2. The GET() method seems to collect the entire page into RAM. This will be a problem in my application. Is there a way to get it char-by-char? I was hoping my use of WiFiClient.read() would do this but I can tell from my heap reports the whole page is in memory immediately after GET() so the WiFiClient.read() is just pulling from there, no help. I can also tell the heap is not restored even after I call stop() and end(). How can I release the heap used by these classes?

Here is my latest version of the toy sketch which works fine fetching wired.com's home page (output at the bottom):

`#include

include

include

include

include

static char ssid[] = "clearskyinstitute"; // your network SSID (name) static char pass[] = "from651plan"; // your network password

void setup() { Serial.begin (115200);

Serial.print ("heap 1: "); Serial.println (ESP.getFreeHeap());

if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("no shield");
    while (1)
       ESP.wdtFeed();
}

// we only want station mode, not access too
WiFi.mode(WIFI_STA);

Serial.println ("connecting to WiFi");
WiFi.begin (ssid, pass);                            // non-blocking, poll with status()
uint32_t t0 = millis();
uint32_t timeout = 15000UL;
while (WiFi.status() != WL_CONNECTED) {
    ESP.wdtFeed();
    if (millis() > t0 + timeout) {
         Serial.println ("no wifi");
         while(1);
              ESP.wdtFeed();
    }
    delay(1000);
}

Serial.print ("heap 2: "); Serial.println (ESP.getFreeHeap());

Serial.println ("trying connection");
HTTPClient sdo_http;
ESP.wdtFeed();
// if (!sdo_http.begin ("https://sdo.gsfc.nasa.gov/assets/img/latest/f_211_193_171_170.jpg",
            // "8D 6B 9B BD AC E4 55 A8 D1 CA 54 67 6F 22 FE 9D 1A F1 2E 70")) {
if (!sdo_http.begin ("https://www.wired.com/index.html",
            "E8 BC F9 5F 59 AB 96 13 00 3D 0E 59 46 6C 6B 5E 7A AA 12 82")) {
    Serial.println ("begin fails");
    while (1)
        ESP.wdtFeed();
}
Serial.print ("heap 3: "); Serial.println (ESP.getFreeHeap());

int get_status = sdo_http.GET();
ESP.wdtFeed();
if (get_status < 0) {
    Serial.print ("GET: "); Serial.println (sdo_http.errorToString(get_status));
    while (1)
      ESP.wdtFeed();
}

Serial.print ("heap 4: "); Serial.println (ESP.getFreeHeap());

WiFiClient sdo_client = sdo_http.getStream();
while (sdo_client.connected()) {
    while (!sdo_client.available())
      ESP.wdtFeed();
    Serial.print((char)sdo_client.read());
}

Serial.print ("heap 5: "); Serial.println (ESP.getFreeHeap());

sdo_client.stop();

Serial.print ("heap 6: "); Serial.println (ESP.getFreeHeap());

sdo_http.end();

Serial.print ("heap 7: "); Serial.println (ESP.getFreeHeap());

}

void loop() { ESP.wdtFeed(); } `

Output:

heap 1: 41248 connecting to WiFi heap 2: 41016 trying connection heap 3: 40760 heap 4: 22432 (snip) heap 5: 22688 heap 6: 22688 heap 7: 22728 `

rohit-rcrohit7 commented 7 years ago

I don't have an esp right now to check and don't have a mac , but in windows, under the tools menu, there are settings to change debug options. Anyways doesn't matter. I am thinking it has to do something with the image.

can you try the below code and tell me the output you getting?

include

include

include

include

define USE_SERIAL Serial

ESP8266WiFiMulti WiFiMulti;

void setup() {

USE_SERIAL.begin(115200);

// USE_SERIAL.setDebugOutput(true);

USE_SERIAL.println();
USE_SERIAL.println();
USE_SERIAL.println();

for(uint8_t t = 4; t > 0; t--) {
    USE_SERIAL.printf("[SETUP] WAIT %d...\n", t);
    USE_SERIAL.flush();
    delay(1000);
}

WiFiMulti.addAP("clearskyinstitute", "from651plan");

}

void loop() { // wait for WiFi connection if((WiFiMulti.run() == WL_CONNECTED)) {

    HTTPClient http;

    USE_SERIAL.print("[HTTP] begin...\n");
    // configure traged server and url
    http.begin("https://sdo.gsfc.nasa.gov/assets/img/latest/f_211_193_171_170.jpg", "8D 6B 9B BD AC E4 55 A8 D1 CA 54 67 6F 22 FE 9D 1A F1 2E 70"); //HTTPS

    USE_SERIAL.print("[HTTP] GET...\n");
    // start connection and send HTTP header
    int httpCode = http.GET();

    // httpCode will be negative on error
    if(httpCode > 0) {
        // HTTP header has been send and Server response header has been handled
        USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode);

        // file found at server
        if(httpCode == HTTP_CODE_OK) {
            String payload = http.getString();
            USE_SERIAL.println(payload);
        }
    } else {
        USE_SERIAL.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
    }

    http.end();
}

delay(10000);

}

rohit-rcrohit7 commented 7 years ago

also, run this code and tell me, once tomorrow I get my esp I can tell you your exact problem in your code I think.

include

include

const char ssid = "clearskyinstitute"; const char password = "from651plan";

const char* host = "sdo.gsfc.nasa.gov"; const int httpsPort = 443;

// Use web browser to view and copy // SHA1 fingerprint of the certificate const char* fingerprint = "8D 6B 9B BD AC E4 55 A8 D1 CA 54 67 6F 22 FE 9D 1A F1 2E 70";

void setup() { Serial.begin(115200); Serial.println(); Serial.print("connecting to "); Serial.println(ssid); WiFi.begin(ssid, password); while (WiFi.status() != WL_CONNECTED) { delay(500); Serial.print("."); } Serial.println(""); Serial.println("WiFi connected"); Serial.println("IP address: "); Serial.println(WiFi.localIP());

// Use WiFiClientSecure class to create TLS connection WiFiClientSecure client; Serial.print("connecting to "); Serial.println(host); if (!client.connect(host, httpsPort)) { Serial.println("connection failed"); return; }

if (client.verify(fingerprint, host)) { Serial.println("certificate matches"); } else { Serial.println("certificate doesn't match"); }

String url = "/assets/img/latest/f_211_193_171_170.jpg"; Serial.print("requesting URL: "); Serial.println(url);

client.print(String("GET ") + url + " HTTP/1.1\r\n" + "Host: " + host + "\r\n" + "User-Agent: BuildFailureDetectorESP8266\r\n" + "Connection: close\r\n\r\n");

Serial.println("request sent"); while (client.connected()) { String line = client.readStringUntil('\n'); if (line == "\r") { Serial.println("headers received"); break; } } String line = client.readStringUntil('\n'); if (line.startsWith("{\"state\":\"success\"")) { Serial.println("esp8266/Arduino CI successfull!"); } else { Serial.println("esp8266/Arduino CI has failed"); } Serial.println("reply was:"); Serial.println("=========="); Serial.println(line); Serial.println("=========="); Serial.println("closing connection"); }

void loop() { }

ghost commented 7 years ago

Output from Multi version:

WAIT 4... [SETUP] WAIT 3... [SETUP] WAIT 2... [SETUP] WAIT 1... [HTTP] begin... [HTTP] GET... [HTTP] GET... failed, error: connection refused [HTTP] begin... [HTTP] GET... [HTTP] GET... failed, error: connection refused [HTTP] begin... [HTTP] GET... [HTTP] GET... failed, error: connection refused [HTTP] begin... [HTTP] GET... [HTTP] GET... failed, error: connection refused

ghost commented 7 years ago

Output from WiFiClientSecure version doesn't get far:

connecting to clearskyinstitute ...... WiFi connected IP address: 192.168.0.76 connecting to sdo.gsfc.nasa.gov connection failed

rohit-rcrohit7 commented 7 years ago

Ok, I will look into it once I come back tomorrow.Working out these from mobile sucks.

ghost commented 7 years ago

Copy that. Thanks thorburn1.

ghost commented 7 years ago

I found I can get more debug output by calling Serial.setDebugOutput(true).

Here is the output. All the numbers make sense for my network. But I am surprised it mentions SNTP.

` connecting to clearskyinstitute scandone f 0, scandone state: 0 -> 2 (b0) state: 2 -> 3 (0) state: 3 -> 5 (10) add 0 aid 9 cnt

connected with clearskyinstitute, channel 2 dhcp client start... .....ip:192.168.0.76,mask:255.255.255.0,gw:192.168.0.1 . WiFi connected IP address: 192.168.0.76 connecting to sdo.gsfc.nasa.gov please start sntp first ! connection failed pm open,type:2 0 `

ghost commented 7 years ago

Note the sketch works fine with wired.com using its fingerprint. I tried to paste the output here but the wiki messes up the formatting bad.

rohit-rcrohit7 commented 7 years ago

I think the error is due to the certificate size , it can't handle it.Runs out of memory probably. There was a fix for it . https://github.com/esp8266/Arduino/issues/1816 Go through this post and see igrr had posted a fix for it.

ghost commented 7 years ago

He says he posted a fix to 2.3.0 but that's what I'm already using so I guess it's still too large.

FYI, here is what I am working on : http://clearskyinstitute.com/ham/HamClock . The solar image in the upper right no longer works because the sdo site changed from http to https on Jan 1 2017. Good for them I suppose but bad for IoT.

I think I'm out of luck. Many thanks for your time and efforts thorburn1.

rohit-rcrohit7 commented 7 years ago

Its okay , I will see it now.

On 16 January 2017 at 10:11, pidloop notifications@github.com wrote:

He says he posted a fix to 2.3.0 but that's what I'm already using so I guess it's still too large.

FYI, here is what I am working on : http://clearskyinstitute.com/ ham/HamClock . The solar image in the upper right no longer works because the sdo site changed from http to https on Jan 1 2017. Good for them I suppose but bad for IoT.

I think I'm out of luck. Many thanks for your time and efforts thorburn1.

— You are receiving this because you commented. Reply to this email directly, view it on GitHub https://github.com/esp8266/Arduino/issues/2856#issuecomment-272772737, or mute the thread https://github.com/notifications/unsubscribe-auth/AS3BTr1hK85mcxODAMTQNM9COVGwB6DBks5rSvT7gaJpZM4LjgcO .

digidhamu commented 7 years ago

@thorburn1

Same error as it is not connecting. Also attached code and debug message as well for your reference. Please advise.

Error without Debug

connecting to KarunGuest
..
WiFi connected
IP address: 
172.16.42.3
connecting to admin.efficientdriver.net
connection failed

Code

#include <ESP8266WiFi.h>
#include <WiFiClientSecure.h>

const char* ssid = "ssid";
const char* password = "password";

//const char* host = "api.github.com";
const char* host = "admin.efficientdriver.net";
const int httpsPort = 443;

// Use web browser to view and copy
// SHA1 fingerprint of the certificate
//const char* fingerprint = "CF 05 98 89 CA FF 8E D8 5E 5C E0 C2 E4 F7 E6 C3 C7 50 DD 5C";
const char* fingerprint = "63 7D 90 0A 1A C8 54 D5 52 41 4A 3F 61 A6 9A 7E C6 15 E2 5D";

void setup() {
  Serial.setDebugOutput(true);  
  Serial.begin(115200);
  Serial.println();
  Serial.print("connecting to ");
  Serial.println(ssid);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());

  // Use WiFiClientSecure class to create TLS connection
  WiFiClientSecure client;
  Serial.print("connecting to ");
  Serial.println(host);
  if (!client.connect(host, httpsPort)) {
    Serial.println("connection failed");
    return;
  }

  if (client.verify(fingerprint, host)) {
    Serial.println("certificate matches");
  } else {
    Serial.println("certificate doesn't match");
  }

  String url = "/repos/esp8266/Arduino/commits/master/status";
  Serial.print("requesting URL: ");
  Serial.println(url);

  client.print(String("GET ") + url + " HTTP/1.1\r\n" +
               "Host: " + host + "\r\n" +
               "User-Agent: BuildFailureDetectorESP8266\r\n" +
               "Connection: close\r\n\r\n");

  Serial.println("request sent");
  while (client.connected()) {
    String line = client.readStringUntil('\n');
    if (line == "\r") {
      Serial.println("headers received");
      break;
    }
  }
  String line = client.readStringUntil('\n');
  if (line.startsWith("{\"state\":\"success\"")) {
    Serial.println("esp8266/Arduino CI successfull!");
  } else {
    Serial.println("esp8266/Arduino CI has failed");
  }
  Serial.println("reply was:");
  Serial.println("==========");
  Serial.println(line);
  Serial.println("==========");
  Serial.println("closing connection");
}

void loop() {
}

Debug Output

connecting to ssid
sta config unchangedscandone
f 0, scandone
state: 0 -> 2 (b0)
state: 2 -> 3 (0)
state: 3 -> 5 (10)
add 0
aid 1
cnt 

connected with KarunGuest, channel 8
dhcp client start...
.ip:172.16.42.3,mask:255.255.255.0,gw:172.16.42.1
.
WiFi connected
IP address: 
172.16.42.3
connecting to admin.efficientdriver.net
:ref 1
please start sntp first !
State:  sending Client Hello (1)
:wr
:sent 88
:rn 7
:ww
:rd 5, 7, 0
:rdi 7, 5
:rd 2, 7, 5
:rdi 2, 2
:c0 2, 7
Alert: handshake failure
Error: SSL error 40
:wr
:rcl
:abort
:ww
Alert: unexpected message
Alert: close notify
connection failed
:ur 1
:del
rohit-rcrohit7 commented 7 years ago

It appears that connection fails during TLS handshake.

as per @igrr Two cipher suites supported by axTLS library on the ESP side (TLS_RSA_WITH_AES_128_CBC_SHA and TLS_RSA_WITH_AES_256_CBC_SHA) are not among the list of cipher suites supported by your server. This causes handshake failure, because if the server and client have no cipher suites in common, they can't talk to each other.

You may want to check your server configuration to enable one of these suites. Alternatively, if you can not or do not want to change TLS config for the whole server, you may set up a TLS proxy which will handle talking to ESPs only. The rest of your site will run with same TLS settings, but only the endpoints needed by ESP will be passed through a proxy. In this case the proxy should be configured to support these two mentioned cipher suites.

digidhamu commented 7 years ago

@thorburn1 Perfect. I have now configured as you advised and all fine now. Thanks again for your support.

devyte commented 6 years ago

BearSSL is merged in #4273 , with alternate BearSSL::WiFi* classes. Although axtls-based classes are still available and even the default, they are planned for deprecation and then retirement, hence won't be fixed. Any issues with BearSSL-based classes should be reported in new issues. Closing.