Http client redirect fails if Location header contains relative URL (IDFGH-13686)

CarlosDerSeher commented 2 months ago

Answers checklist.

[X] I have read the documentation ESP-IDF Programming Guide and the issue is not addressed there.
[X] I have updated my IDF branch (master or release) to the latest version and checked that the issue is present there.
[X] I have searched the issue tracker for a similar issue and not found a similar issue.

IDF version.

v5.1.1

Espressif SoC revision.

esp32 v1.0

Operating System used.

Windows

How did you build your project?

Eclipse IDE

If you are using Windows, please specify command line type.

CMD

Development Kit.

Mini D1 ESP32

Power Supply used.

USB

What is the expected behavior?

Http client should be redirected to to relative url (record_list.php) passed by the server with 302 status code.

What is the actual behavior?

http_parser_parse_url() returns with s_dead on the character _ and therefore http_client can't parse the url.

Steps to reproduce.

I have to say that I changed the .host entry below and also post_data isn't original, as I can't share the server and login data here.

esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
    static char *output_buffer;  // Buffer to store response of http request from event handler
    static int output_len;       // Stores number of bytes read
    switch(evt->event_id) {
        case HTTP_EVENT_ERROR:
            ESP_LOGI(TAG, "HTTP_EVENT_ERROR");
            break;
        case HTTP_EVENT_ON_CONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_CONNECTED");
            break;
        case HTTP_EVENT_HEADER_SENT:
            ESP_LOGI(TAG, "HTTP_EVENT_HEADER_SENT");
            break;
        case HTTP_EVENT_ON_HEADER:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);

            if (strstr(evt->header_key, "Set-Cookie")) {
                strcpy(session_cookie, strtok(evt->header_value, ";"));
                ESP_LOGI(TAG, "found cookie %s", session_cookie);
            }
            else if (strstr(evt->header_key, "Location")) {
                //strcpy(location, evt->header_value);
                if (location) {
                    free(location);
                }
                location = (char *)calloc(1, strlen(evt->header_value) + 2);
                if (evt->header_value[0] != '/') {
                    location[0] = '/';
                }
                strcat(location, evt->header_value);

                ESP_LOGI(TAG, "found location %s", location);
            }

            break;
        case HTTP_EVENT_ON_DATA:
            ESP_LOGD(TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);

            /*
             *  Check for chunked encoding is added as the URL for chunked encoding used in this example returns binary data.
             *  However, event handler can also be used in case chunked encoding is used.
             */
            if (!esp_http_client_is_chunked_response(evt->client)) {
                // If user_data buffer is configured, copy the response into the buffer
                int copy_len = 0;
                if (evt->user_data) {
                    copy_len = MIN(evt->data_len, (MAX_HTTP_OUTPUT_BUFFER - output_len));
                    if (copy_len) {
                        memcpy(evt->user_data + output_len, evt->data, copy_len);
                    }
                } else {
                    const int buffer_len = esp_http_client_get_content_length(evt->client);
                    if (output_buffer == NULL) {
                        output_buffer = (char *) malloc(buffer_len);
                        output_len = 0;
                        if (output_buffer == NULL) {
                            ESP_LOGE(TAG, "Failed to allocate memory for output buffer");
                            return ESP_FAIL;
                        }
                    }
                    copy_len = MIN(evt->data_len, (buffer_len - output_len));
                    if (copy_len) {
                        memcpy(output_buffer + output_len, evt->data, copy_len);
                    }
                }
                output_len += copy_len;
            }

            break;
        case HTTP_EVENT_ON_FINISH:
            ESP_LOGI(TAG, "HTTP_EVENT_ON_FINISH");
            if (output_buffer != NULL) {
                // Response is accumulated in output_buffer. Uncomment the below line to print the accumulated response
                // ESP_LOG_BUFFER_HEX(TAG, output_buffer, output_len);
                free(output_buffer);
                output_buffer = NULL;
            }
            output_len = 0;
            break;
        case HTTP_EVENT_DISCONNECTED:
            ESP_LOGI(TAG, "HTTP_EVENT_DISCONNECTED");
            int mbedtls_err = 0;
            esp_err_t err = esp_tls_get_and_clear_last_error((esp_tls_error_handle_t)evt->data, &mbedtls_err, NULL);
            if (err != 0) {
                ESP_LOGI(TAG, "Last esp error code: 0x%x", err);
                ESP_LOGI(TAG, "Last mbedtls failure: 0x%x", mbedtls_err);
            }
            if (output_buffer != NULL) {
                free(output_buffer);
                output_buffer = NULL;
            }
            output_len = 0;
            break;
        case HTTP_EVENT_REDIRECT:
            ESP_LOGI(TAG, "HTTP_EVENT_REDIRECT");

            esp_http_client_set_method(evt->client, HTTP_METHOD_GET);
            //esp_http_client_set_url(evt->client, location);
            esp_http_client_set_header(evt->client, "Content-Type", "text/html");
            esp_http_client_set_header(evt->client, "Cookie", session_cookie);

            err = esp_http_client_set_redirection(evt->client);

            free(location);
            location = NULL;

            break;
    }
    return ESP_OK;
}

#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
static void https_with_url(void)
{
    esp_http_client_config_t config = {
                .host = "put_server_here.at",
                .path = "/",
                .user_agent = "esp-idf/5.1.1 esp32",
                                .event_handler = _http_event_handler,
                .disable_auto_redirect = true,
                .transport_type = HTTP_TRANSPORT_OVER_SSL,
                                .crt_bundle_attach = esp_crt_bundle_attach,
                .keep_alive_enable = true,
    };
    esp_http_client_handle_t client = esp_http_client_init(&config);
    const char *post_data = "password=***&Submit=Start";

    esp_http_client_set_url(client, "/backend.php");
    esp_http_client_set_method(client, HTTP_METHOD_POST);
    esp_http_client_set_header(client, "Content-Type", "application/x-www-form-urlencoded");
    esp_http_client_set_post_field(client, post_data, strlen(post_data));

    esp_err_t err = esp_http_client_perform(client);

    if (err == ESP_OK) {
        ESP_LOGI(TAG, "HTTPS Status = %d, content_length = %"PRIu64,
                esp_http_client_get_status_code(client),
                esp_http_client_get_content_length(client));
    } else {
        ESP_LOGE(TAG, "Error perform http request %s", esp_err_to_name(err));
    }
    esp_http_client_cleanup(client);
}
#endif // CONFIG_MBEDTLS_CERTIFICATE_BUNDLE

static void http_test_task(void *pvParameters)
{
#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
    https_with_url();
#endif

    ESP_LOGI(TAG, "Finish http example");
#if !CONFIG_IDF_TARGET_LINUX
    vTaskDelete(NULL);
#endif
}

void app_main(void)
{
    esp_err_t ret = nvs_flash_init();
    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
      ESP_ERROR_CHECK(nvs_flash_erase());
      ret = nvs_flash_init();
    }
    ESP_ERROR_CHECK(ret);

    ESP_ERROR_CHECK(esp_netif_init());
    ESP_ERROR_CHECK(esp_event_loop_create_default());

    /* This helper function configures Wi-Fi or Ethernet, as selected in menuconfig.
     * Read "Establishing Wi-Fi or Ethernet Connection" section in
     * examples/protocols/README.md for more information about this function.
     */
    ESP_ERROR_CHECK(example_connect());
    ESP_LOGI(TAG, "Connected to AP, begin http example");

#if CONFIG_IDF_TARGET_LINUX
    http_test_task(NULL);
#else
    xTaskCreate(&http_test_task, "http_test_task", 8192, NULL, 5, NULL);
#endif
}

Debug Logs.

I (30) boot: ESP-IDF v5.1.1-dirty 2nd stage bootloader
I (30) boot: compile time Sep 12 2024 11:21:54
I (30) boot: Multicore bootloader
I (35) boot: chip revision: v1.0
I (39) boot.esp32: SPI Speed      : 40MHz
I (43) boot.esp32: SPI Mode       : DIO
I (48) boot.esp32: SPI Flash Size : 2MB
I (53) boot: Enabling RNG early entropy source...
I (58) boot: Partition Table:
I (62) boot: ## Label            Usage          Type ST Offset   Length
I (69) boot:  0 nvs              WiFi data        01 02 00009000 00006000
I (76) boot:  1 phy_init         RF data          01 01 0000f000 00001000
I (84) boot:  2 factory          factory app      00 00 00010000 00100000
I (91) boot: End of partition table
I (95) esp_image: segment 0: paddr=00010020 vaddr=3f400020 size=35e3ch (220732) map
I (184) esp_image: segment 1: paddr=00045e64 vaddr=3ffb0000 size=03738h ( 14136) load
I (189) esp_image: segment 2: paddr=000495a4 vaddr=40080000 size=06a74h ( 27252) load
I (201) esp_image: segment 3: paddr=00050020 vaddr=400d0020 size=98fb8h (626616) map
I (427) esp_image: segment 4: paddr=000e8fe0 vaddr=40086a74 size=0ec6ch ( 60524) load
I (464) boot: Loaded app from partition at offset 0x10000
I (464) boot: Disabling RNG early entropy source...
I (475) cpu_start: Multicore app
I (476) cpu_start: Pro cpu up.
I (476) cpu_start: Starting app cpu, entry point is 0x4008137c
0x4008137c: call_start_cpu1 at C:/Users/karl/espressif/esp-idf/esp-idf-v5.1.1/components/esp_system/port/cpu_start.c:154

I (0) cpu_start: App cpu up.
I (493) cpu_start: Pro cpu start user code
I (494) cpu_start: cpu freq: 160000000 Hz
I (494) cpu_start: Application information:
I (498) cpu_start: Project name:     esp_http_client_example
I (505) cpu_start: App version:      1
I (509) cpu_start: Compile time:     Sep 12 2024 11:21:40
I (515) cpu_start: ELF file SHA256:  fd7860e9b9ac4e7d...
I (521) cpu_start: ESP-IDF:          v5.1.1-dirty
I (526) cpu_start: Min chip rev:     v0.0
I (531) cpu_start: Max chip rev:     v3.99
I (536) cpu_start: Chip rev:         v1.0
I (541) heap_init: Initializing. RAM available for dynamic allocation:
I (548) heap_init: At 3FFAE6E0 len 00001920 (6 KiB): DRAM
I (554) heap_init: At 3FFB7FE0 len 00028020 (160 KiB): DRAM
I (560) heap_init: At 3FFE0440 len 00003AE0 (14 KiB): D/IRAM
I (567) heap_init: At 3FFE4350 len 0001BCB0 (111 KiB): D/IRAM
I (573) heap_init: At 400956E0 len 0000A920 (42 KiB): IRAM
I (581) spi_flash: detected chip: generic
I (584) spi_flash: flash io: dio
W (588) spi_flash: Detected size(4096k) larger than the size in the binary image header(2048k). Using the size in the binary image header.
I (602) app_start: Starting scheduler on CPU0
I (606) app_start: Starting scheduler on CPU1
I (606) main_task: Started on CPU0
I (616) main_task: Calling app_main()
I (646) example_connect: Start example_connect.
I (666) wifi:wifi driver task: 3ffbfc08, prio:23, stack:6656, core=0
I (676) wifi:wifi firmware version: ce9244d
I (676) wifi:wifi certification version: v7.0
I (676) wifi:config NVS flash: enabled
I (676) wifi:config nano formating: disabled
I (686) wifi:Init data frame dynamic rx buffer num: 32
I (686) wifi:Init management frame dynamic rx buffer num: 32
I (696) wifi:Init management short buffer num: 32
I (696) wifi:Init dynamic tx buffer num: 32
I (706) wifi:Init static rx buffer size: 1600
I (706) wifi:Init static rx buffer num: 10
I (706) wifi:Init dynamic rx buffer num: 32
I (716) wifi_init: rx ba win: 6
I (716) wifi_init: tcpip mbox: 32
I (726) wifi_init: udp mbox: 6
I (726) wifi_init: tcp mbox: 6
I (726) wifi_init: tcp tx win: 5744
I (736) wifi_init: tcp rx win: 5744
I (736) wifi_init: tcp mss: 1440
I (746) wifi_init: WiFi IRAM OP enabled
I (746) wifi_init: WiFi RX IRAM OP enabled
I (756) phy_init: phy_version 4670,719f9f6,Feb 18 2021,17:07:07
I (856) wifi:mode : sta (c4:4f:33:6a:54:45)
I (856) wifi:enable tsf
I (856) example_connect: Connecting to Aktiva_Elektronik_GmbH...
I (866) example_connect: Waiting for IP(s)
I (3276) wifi:new:<11,2>, old:<1,0>, ap:<255,255>, sta:<11,2>, prof:1
I (4436) wifi:state: init -> auth (b0)
I (4436) wifi:state: auth -> assoc (0)
I (4446) wifi:state: assoc -> run (10)
I (4456) wifi:connected with Aktiva_Elektronik_GmbH, aid = 2, channel 11, 40D, bssid = 30:b5:c2:23:0a:e4
I (4456) wifi:security: WPA2-PSK, phy: bgn, rssi: -53
I (4456) wifi:pm start, type: 1

I (4526) wifi:AP's beacon interval = 102400 us, DTIM period = 1
I (5466) esp_netif_handlers: example_netif_sta ip: 10.0.0.13, mask: 255.255.255.0, gw: 10.0.0.2
I (5466) example_connect: Got IPv4 event: Interface "example_netif_sta" address: 10.0.0.13
I (5646) example_connect: Got IPv6 event: Interface "example_netif_sta" address: fe80:0000:0000:0000:c64f:33ff:fe6a:5445, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (5646) example_common: Connected to example_netif_sta
I (5656) example_common: - IPv4 address: 10.0.0.13,
I (5656) example_common: - IPv6 address: fe80:0000:0000:0000:c64f:33ff:fe6a:5445, type: ESP_IP6_ADDR_IS_LINK_LOCAL
I (5666) HTTP_CLIENT: Connected to AP, begin http example
I (5686) main_task: Returned from app_main()
I (5686) wifi:<ba-add>idx:0 (ifx:0, 30:b5:c2:23:0a:e4), tid:0, ssn:0, winSize:64
I (5886) esp-x509-crt-bundle: Certificate validated
I (6886) HTTP_CLIENT: HTTP_EVENT_ON_CONNECTED
I (6896) HTTP_CLIENT: HTTP_EVENT_HEADER_SENT
I (7196) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Server, value=nginx
I (7196) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Date, value=Thu, 12 Sep 2024 12:04:26 GMT
I (7196) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Content-Type, value=text/html; charset=utf-8
I (7206) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Transfer-Encoding, value=chunked
I (7216) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Connection, value=keep-alive
I (7226) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Expires, value=Thu, 19 Nov 1981 08:52:00 GMT
I (7236) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Cache-Control, value=no-store, no-cache, must-revalidate
I (7246) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Pragma, value=no-cache
I (7246) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Set-Cookie, value=PHPSESSID=j9qs7orag5p0dhmc1f2n7mnqfr; path=/
I (7266) HTTP_CLIENT: HTTP_EVENT_ON_HEADER, key=Location, value=record_list.php
I (7286) HTTP_CLIENT: HTTP_EVENT_REDIRECT
E (7286) HTTP_CLIENT: Error parse url record_list.php
I (7296) HTTP_CLIENT: HTTP_EVENT_ON_FINISH
I (7296) HTTP_CLIENT: HTTPS Status = 302, content_length = 18446744073709551615
I (7306) HTTP_CLIENT: HTTP_EVENT_DISCONNECTED
I (7316) HTTP_CLIENT: Finish http example

More Information.

http_parser_parse_url() function expects a schema when the location starts with a letter and therefore fails on the symbol _

kayzen9 commented 1 month ago

try https://github.com/espressif/esp-idf/issues/13459#issuecomment-2019815901 @CarlosDerSeher

CarlosDerSeher commented 1 month ago

I've changed toplevel CMakeLists to this

# The following five lines of boilerplate have to be in your project's
# CMakeLists in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.16)

set(EXTRA_COMPONENT_DIRS $ENV{IDF_PATH}/examples/common_components/protocol_examples_common)

if(${IDF_TARGET} STREQUAL "linux")
    list(APPEND EXTRA_COMPONENT_DIRS "$ENV{IDF_PATH}/tools/mocks/freertos/"
                    "$ENV{IDF_PATH}/examples/protocols/linux_stubs/esp_stubs")
    set(COMPONENTS main)
endif()

include($ENV{IDF_PATH}/tools/cmake/project.cmake)

project(esp_http_client_example)

idf_component_get_property(http_parser http_parser COMPONENT_LIB)
target_compile_definitions(${http_parser} PRIVATE "-DHTTP_PARSER_STRICT=0")

but the problem still persists.

mahavirj commented 1 month ago

@CarlosDerSeher

http_parser_parse_url() returns with sdead on the character and therefore http_client can't parse the url.

If your problem is indeed related to above parsing failure then DHTTP_PARSER_STRICT=0 should have helped here. You can try putting some debug prints under HTTP_PARSER_STRICT in file components/http_parser/http_parser.c to see that the build part is correct here.

If you still run into any issues then please share a minimal self sufficient application that we can use on our side to recreate the issue.

CarlosDerSeher commented 1 month ago

Ok I'll see if I find the time to investigate further. I switched to Arduino framework though and there I don't see this problem, so for me this is solved. Thanks for your time sir.

nileshkale123 commented 1 month ago

So, I am closing this issue now. Thanks.

espressif / esp-idf