search

kubernetes / ingress-nginx

Ingress NGINX Controller for Kubernetes
https://kubernetes.github.io/ingress-nginx/
Apache License 2.0
17.34k stars 8.22k forks source link

connection refused when access nginx-ingress controller with https #411

Closed mqliang closed 7 years ago

mqliang commented 7 years ago

@aledbf

I deploy nginx-ingress-controller and use TLS termination to secure an Ingress as this tutorial does. Howe, https connection was refused by nginx-ingress controller:

Ingress yaml is as follows:

[root@c1v41 ~]# kubectl get ingress foo -o yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  annotations:
    ingress.alpha.k8s.io/loadbalancer-name: lb
  creationTimestamp: 2017-03-08T07:05:11Z
  generation: 1
  name: foo
  namespace: default
  resourceVersion: "35034658"
  selfLink: /apis/extensions/v1beta1/namespaces/default/ingresses/foo
  uid: 9244814c-03cd-11e7-9b29-525400d8fcb5
spec:
  rules:
  - host: foo.bar.com
    http:
      paths:
      - backend:
          serviceName: echoheaders-x
          servicePort: 80
        path: /
  tls:
  - hosts:
    - foo.bar.com
    secretName: foo-secret
status:
  loadBalancer:
    ingress:
    - ip: 192.168.16.182

Can access through service IP

[root@c1v41 ~]# kubectl get svc echoheaders-x
NAME            CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
echoheaders-x   10.254.31.223   <none>        80/TCP    19h
[root@c1v41 ~]# kubectl get endpoints echoheaders-x
NAME            ENDPOINTS           AGE
echoheaders-x   10.100.53.40:8080   19h
[root@c1v41 ~]# curl 10.254.31.223:80
CLIENT VALUES:
client_address=10.100.15.0
command=GET
real path=/
query=nil
request_version=1.1
request_uri=http://10.254.31.223:8080/

SERVER VALUES:
server_version=nginx: 1.10.0 - lua: 10001

HEADERS RECEIVED:
accept=*/*
host=10.254.31.223
user-agent=curl/7.29.0
BODY:
-no body in request-[root@c1v41 ~]#

Can not access through Ingress: Connection refused

[root@c1v41 ~]# curl https://192.168.16.182 -H 'Host:foo.bar.com' -k -v
* About to connect() to 192.168.16.182 port 443 (#0)
*   Trying 192.168.16.182...
* Connection refused
* Failed connect to 192.168.16.182:443; Connection refused
* Closing connection 0
curl: (7) Failed connect to 192.168.16.182:443; Connection refused

nginx config:

kubectl exec -it sh into nginx-ingress-controller Pod and cat /etc/nginx/nginx.conf: 

# cat /etc/nginx/nginx.conf

daemon off;

worker_processes 4;
pid /run/nginx.pid;

worker_rlimit_nofile 261120;
events {
    multi_accept        on;
    worker_connections  16384;
    use                 epoll;
}

http {
    real_ip_header      X-Forwarded-For;
    set_real_ip_from    0.0.0.0/0;
    real_ip_recursive   on;

    geoip_country       /etc/nginx/GeoIP.dat;
    geoip_city          /etc/nginx/GeoLiteCity.dat;
    geoip_proxy_recursive on;
    # lua section to return proper error codes when custom pages are used
    lua_package_path '.?.lua;./etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/lua-resty-http/lib/?.lua;';
    init_by_lua_block {
        require("error_page")
    }

    sendfile            on;
    aio                 threads;
    tcp_nopush          on;
    tcp_nodelay         on;
    log_subrequest      on;

    reset_timedout_connection on;

    keepalive_timeout 75s;

    client_header_buffer_size       1k;
    large_client_header_buffers     4 8k;

    types_hash_max_size             2048;
    server_names_hash_max_size      512;
    server_names_hash_bucket_size   64;
    map_hash_bucket_size            64;

    include /etc/nginx/mime.types;
    default_type text/html;
    gzip on;
    gzip_comp_level 5;
    gzip_http_version 1.1;
    gzip_min_length 256;
    gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component;
    gzip_proxied any;

    server_tokens on;

    log_format upstreaminfo '$remote_addr - [$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';

    map $request_uri $loggable {
        default 1;
    }

    access_log /var/log/nginx/access.log upstreaminfo if=$loggable;
    error_log  /var/log/nginx/error.log notice;

    resolver 192.168.16.1 114.114.114.114 valid=30s;

    # Retain the default nginx handling of requests without a "Connection" header
    map $http_upgrade $connection_upgrade {
        default          upgrade;
        ''               close;
    }

    # trust http_x_forwarded_proto headers correctly indicate ssl offloading
    map $http_x_forwarded_proto $pass_access_scheme {
        default          $http_x_forwarded_proto;
        ''               $scheme;
    }

    map $http_x_forwarded_port $pass_server_port {
       default           $http_x_forwarded_port;
       ''                $server_port;
    }

    # map port 442 to 443 for header X-Forwarded-Port
    map $pass_server_port $pass_port {
        442              443;
        default          $pass_server_port;
    }

    # Map a response error watching the header Content-Type
    map $http_accept $httpAccept {
        default          html;
        application/json json;
        application/xml  xml;
        text/plain       text;
    }

    map $httpAccept $httpReturnType {
        default          text/html;
        json             application/json;
        xml              application/xml;
        text             text/plain;
    }

    server_name_in_redirect off;
    port_in_redirect        off;

    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

    # turn on session caching to drastically improve performance
    ssl_session_cache builtin:1000 shared:SSL:10m;
    ssl_session_timeout 10m;

    # allow configuring ssl session tickets
    ssl_session_tickets on;

    # slightly reduce the time-to-first-byte
    ssl_buffer_size 4k;

    # allow configuring custom ssl ciphers
    ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
    ssl_prefer_server_ciphers on;

    # In case of errors try the next upstream server before returning an error
    proxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;

    upstream default-echoheaders-x-80 {
        least_conn;
        server 10.100.53.40:8080 max_fails=0 fail_timeout=0;
    }
    upstream upstream-default-backend {
        least_conn;
        server 10.100.63.37:80 max_fails=0 fail_timeout=0;
    }
    server {
        server_name _;
        listen [::]:80 ipv6only=off default_server reuseport backlog=511;
        listen [::]:443  default_server reuseport backlog=511 ssl http2;
        # PEM sha: 91ecca52df8408b78c72f5aa779c909d2d8b4706
        ssl_certificate                         /ingress-controller/ssl/system-snake-oil-certificate.pem;
        ssl_certificate_key                     /ingress-controller/ssl/system-snake-oil-certificate.pem;

        more_set_headers                        "Strict-Transport-Security: max-age=15724800; includeSubDomains; preload";
        location / {
            set $proxy_upstream_name "upstream-default-backend";

            port_in_redirect off;
            client_max_body_size                    "1m";

            proxy_set_header Host                   $host;

            # Pass the extracted client certificate to the backend

            # Pass Real IP
            proxy_set_header X-Real-IP              $remote_addr;

            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;
            proxy_set_header                        Connection        $connection_upgrade;

            proxy_set_header X-Forwarded-For        $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host       $host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";

            # Custom headers

            proxy_connect_timeout                   5s;
            proxy_send_timeout                      60s;
            proxy_read_timeout                      60s;

            proxy_redirect                          off;
            proxy_buffering                         off;
            proxy_buffer_size                       "4k";

            proxy_http_version                      1.1;

            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;

            proxy_pass http://upstream-default-backend;
        }
        # health checks in cloud providers require the use of port 80
        location /healthz {
            access_log off;
            return 200;
        }

        # this is required to avoid error if nginx is being monitored
        # with an external software (like sysdig)
        location /nginx_status {
            allow 127.0.0.1;
            allow ::1;
            deny all;

            access_log off;
            stub_status on;
        }
    }

    server {
        server_name foo.bar.com;
        listen [::]:80;
        listen [::]:443  ssl http2;
        # PEM sha: 2f6fdfac98bdfb70cdb54d4d05a7c1dd887c7692
        ssl_certificate                         /ingress-controller/ssl/default-foo-secret.pem;
        ssl_certificate_key                     /ingress-controller/ssl/default-foo-secret.pem;

        more_set_headers                        "Strict-Transport-Security: max-age=15724800; includeSubDomains; preload";
        location / {
            set $proxy_upstream_name "default-echoheaders-x-80";

            port_in_redirect off;

            # enforce ssl on server side
            if ($scheme = http) {
                return 301 https://$host$request_uri;
            }

            client_max_body_size                    "1m";

            proxy_set_header Host                   $host;

            # Pass the extracted client certificate to the backend

            # Pass Real IP
            proxy_set_header X-Real-IP              $remote_addr;

            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;
            proxy_set_header                        Connection        $connection_upgrade;

            proxy_set_header X-Forwarded-For        $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host       $host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;

            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";

            # Custom headers

            proxy_connect_timeout                   5s;
            proxy_send_timeout                      60s;
            proxy_read_timeout                      60s;

            proxy_redirect                          off;
            proxy_buffering                         off;
            proxy_buffer_size                       "4k";

            proxy_http_version                      1.1;

            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;

            proxy_pass http://default-echoheaders-x-80;
        }
    }

    # default server, used for NGINX healthcheck and access to nginx stats
    server {
        # Use the port 18080 (random value just to avoid known ports) as default port for nginx.
        # Changing this value requires a change in:
        # https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx/command.go#L104
        listen [::]:18080 ipv6only=off default_server reuseport backlog=511;

        location /healthz {
            access_log off;
            return 200;
        }
        location /nginx_status {
            access_log off;
            stub_status on;
        }

        # this location is used to extract nginx metrics
        # using prometheus.
        # TODO: enable extraction for vts module.
        location /internal_nginx_status {
            allow 127.0.0.1;
            allow ::1;
            deny all;

            access_log off;
            stub_status on;
        }

        location / {
            set $proxy_upstream_name "upstream-default-backend";
            proxy_pass             http://upstream-default-backend;
        }

    }

    # default server for services without endpoints
    server {
        listen 8181;
        set $proxy_upstream_name "-";

        location / {
            return 503;
        }
    }
}

stream {
    # TCP services

    # UDP services
}
#
mqliang commented 7 years ago

cc @ddysher @superxi911

gianrubio commented 7 years ago

@mqliang looks like you have a network issue, not related to ingress.

Just to certify if nginx is correctly exposing the port 443, run the curl command inside the container and paste the output. 

$ kubectl exec -ti nginx-ingress-controller-q7m97 bash
root@ hosts:/# curl localhost:80
curl: (52) Empty reply from server
root@hosts:/# curl localhost:443
curl: (56) Recv failure: Connection reset by peer
root@ hosts:/#

Also your service is just exposing the 80 port, you should add port 443.

[root@c1v41 ~]# kubectl get svc echoheaders-x
NAME            CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
echoheaders-x   10.254.31.223   <none>        80/TCP    19h
mqliang commented 7 years ago

@gianrubio

nginx does correctly exposing 443. 

#
#
# curl localhost:80

<html>
    <head>
        <meta http-equiv="content-type" content="text/html; charset=UTF-8">
        <title>page not found</title>
    </head>

    <body>
        <h1>Page Not Found</h1>
    </body>
</html>
# curl localhost:443
<html>
<head><title>400 The plain HTTP request was sent to HTTPS port</title></head>
<body bgcolor="white">
<center><h1>400 Bad Request</h1></center>
<center>The plain HTTP request was sent to HTTPS port</center>
<hr><center>nginx/1.11.10</center>
</body>
</html>
#
# curl https://127.0.0.1 -H 'Host:foo.bar.com' -k -v
* Rebuilt URL to: https://127.0.0.1/
*   Trying 127.0.0.1...
* connect to 127.0.0.1 port 443 failed: Connection refused
* Failed to connect to 127.0.0.1 port 443: Connection refused
* Closing connection 0
curl: (7) Failed to connect to 127.0.0.1 port 443: Connection refused

Seenetstat output:

# netstat -lnp | grep 443
tcp6       0      0 :::443                  :::*                    LISTEN      14/nginx.conf
tcp6       0      0 :::443                  :::*                    LISTEN      14/nginx.conf
tcp6       0      0 :::443                  :::*                    LISTEN      14/nginx.conf
tcp6       0      0 :::443                  :::*                    LISTEN      14/nginx.conf
# ps -ef
UID        PID  PPID  C STIME TTY          TIME CMD
root         1     0  0 Mar08 ?        00:00:00 /sbin/dumb-init -- /nginx-ingress-controller --default-backend-service=default/default-http-backend
root         6     1  1 Mar08 ?        00:20:13 /nginx-ingress-controller --default-backend-service=default/default-http-backend
root        14     6  0 Mar08 ?        00:00:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
root      6665     0  0 09:20 ?        00:00:00 sh
root      7402  6665  0 09:33 ?        00:00:00 ps -ef
root     27253     0  0 02:18 ?        00:00:00 sh
nobody   30617    14  0 03:31 ?        00:00:00 nginx: worker process
nobody   30618    14  0 03:31 ?        00:00:00 nginx: worker process
nobody   30619    14  0 03:31 ?        00:00:00 nginx: worker process
nobody   30620    14  0 03:31 ?        00:00:00 nginx: worker process
mqliang commented 7 years ago

@gianrubio It's very strange that curl 127.0.0.1:443 succeeded but curl https://127.0.0.1 was "connection refused".

mqliang commented 7 years ago

Also your service is just exposing the 80 port, you should add port 443.

It's not a big problem, sine Ingress rule is 

  rules:
  - host: foo.bar.com
    http:
      paths:
      - backend:
          serviceName: echoheaders-x
          servicePort: 80
        path: /

So, nginx will terminate the https and forward request to the 80 port of backend service

gianrubio commented 7 years ago

@mqliang I got the issue, your listen port 443 it's just listening on ipv6. The master branch include a fix to this, the flag is ipv6only=off so the right listen for the default server is

listen [::]:443 default_server ipv6only=off reuseport backlog=511 ssl http2;

What ingress version are you running? Could you build a custom ingress, pointing to the master branch?

mqliang commented 7 years ago

@gianrubio It works after build the latest master branch. Great thanks and close this.

zhang8473 commented 6 years ago

Still have the same problem in release 0.9.0.beta.15