nginx 1.19.0 *ERROR : peer closed connection in SSL handshake (104: Connection reset by peer) while SSL handshaking to upstream

[LearnitRm]

Hi

We are using nginx proxy v1.19.0 for outbound connectivity to client endpoint.

we see intermittent connectivity issues to client endpoint : *164817 peer closed connection in SSL handshake (104: Connection reset by peer) while SSL handshaking to upstream. My nginx configuration : data: egress-service-meshproxy.conf: |

server {

  listen 9080;

  server_name www.services.com;

  proxy_set_header X-Real-IP $remote_addr;
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header X-NginX-Proxy true;

  proxy_cache_bypass $http_upgrade;
  proxy_redirect off;

  proxy_ssl_protocols TLSv1.2 TLSv1.3;
  proxy_ssl_ciphers   HIGH:!aNULL:!MD5;

  proxy_read_timeout 10s;
  proxy_connect_timeout 10s;

  # this doesn't seem to work well of "on" -- 502 upstream drop from on reused connections
  proxy_http_version 1.1;
  proxy_set_header Connection "";
  proxy_ssl_session_reuse off;

  #proxy_ssl_name off;
  proxy_ssl_server_name on;

  proxy_ssl_verify on;
  proxy_ssl_verify_depth 3;

  location / {

    proxy_ssl_certificate /deployment/secrets/egress-service-prod/tls.crt;
    proxy_ssl_certificate_key /deployment/secrets/egress-service-prod/tls.key;
    #proxy_ssl_trusted_certificate /deployment/secrets/egress-service-prod/ca.crt;
    proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;

    proxy_pass https://www.services.com:443;
  }
}

nginx-server-default.conf: |+ server {

  listen 9080 default_server;
  listen [::]:9080 default_server;

  root /usr/share/nginx/html;

  index index.html;

  # Proxy everything we know about to static content
  location /api/v1/irp/health {
    add_header Content-Type text/plain;
    return 200 '{ "status": "OK" }';
  }
  location /api/v1/irp/actuator/health {
    add_header Content-Type text/plain;
    return 200 '{ "status": "OK" }';
  }
  location / {
    add_header Content-Type text/plain;
    return 200 '{ "status": "OK, no content here, use the  services hostname to access SSL reverse proxy!" }';
  }

}

nginx.conf: |+

pcre_jit on;
user  nginx;
worker_processes  1;
error_log  /var/log/nginx/error.log debug;
pid        /var/run/nginx.pid;
events {
    worker_connections        2048;
    accept_mutex              off;
    multi_accept              off;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format main '{"time": "$time_local","status": "$status","request_time": $request_time, "host": "$http_host", "port": "$server_port", "request_uri": "$uri", "x_et_request_id":"$http_x_et_request_id","x_et_response_code": "$upstream_http_x_et_response_code"}';      
    access_log  /var/log/nginx/access.log  main;
    error_log  /var/log/nginx/error.log;
    sendfile                  on;
    tcp_nopush                on;
    tcp_nodelay               on;
    client_max_body_size      10m;
    keepalive_timeout         60;
    #ssl_prefer_server_ciphers on;
    #use                      epoll;
    gzip  on;

    include /deployment/config/nginx-server-default.conf;
    include /deployment/config/egress-service-meshproxy-*.conf;
}

template-nginx-server.conf: |-

server {

  listen 9080;

  server_name ${MESH_HOSTNAME};

  proxy_set_header X-Real-IP $remote_addr;
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header X-NginX-Proxy true;  

  proxy_cache_bypass $http_upgrade;
  proxy_redirect off;

  proxy_ssl_protocols TLSv1.2 TLSv1.3;
  proxy_ssl_ciphers   HIGH:!aNULL:!MD5;

  proxy_read_timeout 10s;
  proxy_connect_timeout 10s;  

  # this doesn't seem to work well of "on" -- 502 upstream drop from  on reused connections
  proxy_http_version 1.1;
  proxy_set_header Connection "";  
  proxy_ssl_session_reuse off;

  #proxy_ssl_name off;
  proxy_ssl_server_name on;
  proxy_ssl_verify on;
  proxy_ssl_verify_depth 3;

  location / {

    proxy_ssl_certificate /deployment/secrets/payaas-ipccpaas-com/tls.crt;
    proxy_ssl_certificate_key /deployment/secrets/payaas-ipccpaas-com/tls.key;
    #proxy_ssl_trusted_certificate /deployment/secrets/payaas-ipccpaas-com/ca.crt;
    proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
    proxy_pass https://${MESH_HOSTNAME};
  }
}

How can I know if the issue is with ciphers/protocols/ssl-certificate etc ? 
Is there any issue with V1.9 

I tried to add $ssl-ciphers, $ssl-protocol to the log-format , but it is just giving me blank fields. 

please help. 

[tippexs]

Hi @LearnitRm thanks for reporting this to us but as this is a bug tracker for NGINX Unit. Please reach out to the NGINX Mailinglist or the Community Slack to get support for NGINX. Information can be found here https://nginx.org/

[xiaoyierle]

关键是 proxy_ssl_protocols 和 proxy_ssl_session_reuse