plugin updater: https client switch

[hbeni]

OpenSSL 3.0.0 was released recently. It should be fairly downward compatible, however we should ensure we are compatible to 3.0 once it gets widespread adoption, otherwise plugin execution might fail in the future. There is also a migration guide. => Recompiling against OpenSSL3 is basicly enough.

A new feature of OpenSSL3 is also a shipped simple http/https client. We might switch to that to replace the currently used 3d party http client library dependency (yhirose/cpp-httplib).

---

A possible alternative could be to use wolfSSL and tinyCurl.

[hbeni]

Debian testing is underway to transition to OpenSSL3.

A possible way for the transition period might be to statically link OpenSSL 1.1.1 until 3 is widespread rolled out (ie adjust openssl configure command in Makefile) The Windows build is already linked statically, affected would be the macos and linux targets.

[hbeni]

Asked the OpenSSL mailing list for an example: https://mta.openssl.org/pipermail/openssl-users/2022-June/015256.html

[hbeni]

This was the example code David von Oheimb provided.

Invocation is like this:

• no proxy: $ ./http_client https://example.com/ && echo ok
• proxy: $ https_proxy=localhost:8081 ./http_client https://example.com/ && echo ok

---

#include <openssl/http.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

BIO *bio_err = NULL;

typedef struct app_http_tls_info_st {
    const char *server;
    const char *port;
    int use_proxy;
    long timeout;
    SSL_CTX *ssl_ctx;
} APP_HTTP_TLS_INFO;

static const char *tls_error_hint(void)
{
    unsigned long err = ERR_peek_error();

    if (ERR_GET_LIB(err) != ERR_LIB_SSL)
        err = ERR_peek_last_error();
    if (ERR_GET_LIB(err) != ERR_LIB_SSL)
        return NULL; /* likely no TLS error */

    switch (ERR_GET_REASON(err)) {
    case SSL_R_WRONG_VERSION_NUMBER:
        return "The server does not support (a suitable version of) TLS";
    case SSL_R_UNKNOWN_PROTOCOL:
        return "The server does not support HTTPS";
    case SSL_R_CERTIFICATE_VERIFY_FAILED:
        return "Cannot authenticate server via its TLS certificate, likely due to mismatch with our trusted TLS certs or missing revocation status";
    case SSL_AD_REASON_OFFSET + TLS1_AD_UNKNOWN_CA:
        return "Server did not accept our TLS certificate, likely due to mismatch with server's trust anchor or missing revocation status";
    case SSL_AD_REASON_OFFSET + SSL3_AD_HANDSHAKE_FAILURE:
        return "TLS handshake failure. Possibly the server requires our TLS certificate but did not receive it";
    default:
        return NULL; /* no hint available for TLS error */
    }
}

static BIO *app_http_tls_close(BIO *bio)
{
    if (bio != NULL) {
        BIO *cbio;
        const char *hint = tls_error_hint();

        if (hint != NULL)
            BIO_printf(bio_err, "%s\n", hint);
        (void)ERR_set_mark();
        BIO_ssl_shutdown(bio);
        cbio = BIO_pop(bio); /* connect+HTTP BIO */
        BIO_free(bio); /* SSL BIO */
        (void)ERR_pop_to_mark(); /* hide SSL_R_READ_BIO_NOT_SET etc. */
        bio = cbio;
    }
    return bio;
}

/* HTTP callback function that supports TLS connection also via HTTPS proxy */
static BIO *app_http_tls_cb(BIO *bio, void *arg, int connect, int detail)
{
    APP_HTTP_TLS_INFO *info = (APP_HTTP_TLS_INFO *)arg;
    SSL_CTX *ssl_ctx = info->ssl_ctx;

    if (ssl_ctx == NULL) /* not using TLS */
        return bio;
    if (connect) {
        SSL *ssl;
        BIO *sbio = NULL;

        /* TODO adapt after callback design flaw is fixed, see #17088 */
        if ((info->use_proxy
             && !OSSL_HTTP_proxy_connect(bio, info->server, info->port,
                                         NULL, NULL, /* no proxy credentials */
                                         info->timeout, bio_err, "HTTP(S) client"))
                || (sbio = BIO_new(BIO_f_ssl())) == NULL) {
            return NULL;
        }
        if ((ssl = SSL_new(ssl_ctx)) == NULL) {
            BIO_free(sbio);
            return NULL;
        }

        /* TODO adapt after callback design flaw is fixed, see #17088 */
        SSL_set_tlsext_host_name(ssl, info->server); /* not critical to do */

        SSL_set_connect_state(ssl);
        BIO_set_ssl(sbio, ssl, BIO_CLOSE);

        bio = BIO_push(sbio, bio);
    } else if (!detail) { /* disconnect from TLS on error */
        bio = app_http_tls_close(bio);
    }
    return bio;
}

static BIO *app_http_get(const char *url, const char *proxy,
                         const char *no_proxy, SSL_CTX *ssl_ctx,
                         const STACK_OF(CONF_VALUE) *headers,
                         long timeout, const char *expected_content_type)
{
    APP_HTTP_TLS_INFO info;
    char *server;
    char *port;
    int use_ssl;
    BIO *mem = NULL;

    if (url == NULL)
        return NULL;

    if (!OSSL_HTTP_parse_url(url, &use_ssl, NULL /* userinfo */, &server, &port,
                             NULL /* port_num, */, NULL, NULL, NULL))
        return NULL;
    if (use_ssl && ssl_ctx == NULL)
        goto end;

    info.server = server;
    info.port = port;
    info.use_proxy = /* workaround for callback design flaw, see #17088 */
        OSSL_HTTP_adapt_proxy(proxy, no_proxy, server, use_ssl) != NULL;
    info.timeout = timeout;
    info.ssl_ctx = ssl_ctx;
    mem = OSSL_HTTP_get(url, proxy, no_proxy, NULL /* bio */, NULL /* rbio */,
                        app_http_tls_cb, &info, 0 /* buf_size */, headers,
                        expected_content_type, 0 /* expect_asn1 */,
                        OSSL_HTTP_DEFAULT_MAX_RESP_LEN, timeout);

 end:
    OPENSSL_free(server);
    OPENSSL_free(port);
    return mem;

}

int main(int argc, char *argv[])
{
    const char *url = argv[1];
    const char *proxy = NULL; // use default from environment variables
    const char *no_proxy = NULL; // use default from environment variables
    SSL_CTX *ssl_ctx = NULL;
    const STACK_OF(CONF_VALUE) *headers = NULL;
    long timeout = 2; // fail quicky for tests
    const char *expected_content_type = NULL; // "text/html; charset=UTF-8";
    BIO *bio = NULL;
    char buf[1000];
    int len;

    bio_err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
    if (strncmp(url, "https", 5) == 0)
        ssl_ctx = SSL_CTX_new(TLS_client_method());
    bio = app_http_get(url, proxy, no_proxy, ssl_ctx, headers, timeout,
                       expected_content_type);
    if (bio != NULL) {
    retry:
        while ((len = BIO_read(bio, buf, sizeof(buf))) > 0)
            printf("%.*s", len, buf);
        if (BIO_should_retry(bio))
            goto retry;
        if (ssl_ctx != NULL)
            bio = app_http_tls_close(bio);
    }
    BIO_free(bio);
    ERR_print_errors(bio_err);
    BIO_free(bio_err);
    return (bio != NULL ? EXIT_SUCCESS : EXIT_FAILURE);
}

[hbeni]

For OpenSSL3 Upgrade: Recompiling is enough.

We may split the issue into two, for the new https fetcher

[hbeni]

Split out new issue #170 for the OpenSSL building part. This issue here remains for switching to the new HTTPs client shipped with OpenSSL v3 for the plugin updater.