search

hashicorp / consul-helm

Helm chart to install Consul and other associated components.
Mozilla Public License 2.0
419 stars 386 forks source link

Consul Ingress Gateway: CrashLoopBackOff #965

Closed d3bt3ch closed 3 years ago

d3bt3ch commented 3 years ago

Consul Ingress Gateway CrashLoopBackOff

Helm Chart Values

## Global
## Tags: https://hub.docker.com/r/hashicorp/consul/tags
## Tags: https://hub.docker.com/r/hashicorp/consul-k8s/tags
## Tags: https://hub.docker.com/r/envoyproxy/envoy-alpine/tags
##
global:
  enabled: false

  name: consul

  datacenter: "dc1"

  domain: "drive.mydomain.local"

  image: "hashicorp/consul:1.9.5"

  imageK8S: "hashicorp/consul-k8s:0.25.0"

  imageEnvoy: "envoyproxy/envoy-alpine:v1.18.3"

  imagePullSecrets:
    - name: docker-reg-cred

  enablePodSecurityPolicies: false

  gossipEncryption:
    secretName: "consul-gossip-encryption"
    secretKey: "key"

  tls:
    enabled: false
    enableAutoEncrypt: false

    serverAdditionalDNSSANs: []

    serverAdditionalIPSANs: []

    verify: true

    httpsOnly: true

    caCert:
      secretName: null
      secretKey: null

    caKey:
      secretName: null
      secretKey: null

  # [Enterprise Only]
  enableConsulNamespaces: false

  acls:
    manageSystemACLs: false

    bootstrapToken:
      secretName: "consul-bootstrap-token"
      secretKey: "key"

    createReplicationToken: false

    replicationToken:
      secretName: null
      secretKey: null

  federation:
    enabled: false
    createFederationSecret: false

  metrics:
    enabled: false
    enableAgentMetrics: false

    agentMetricsRetentionTime: 1m

    enableGatewayMetrics: true

  consulSidecarContainer:
    resources:
      limits:
        cpu: 20m
        memory: 50Mi
      requests:
        cpu: "20m"
        memory: "25Mi"

  openshift:
    enabled: false

## Server
##
server:
  enabled: false

## External Server(s)
externalServers:
  enabled: true

  hosts:
    - "10.3.1.5"

  httpsPort: 8501

  tlsServerName: null

  useSystemRoots: false

  k8sAuthMethodHost: "https://E11CD5F1479D76C6CD2D381AD42C15C8.gr7.us-west-2.eks.amazonaws.com:443"

## Client
##
client:
  enabled: true

  join:
    - "10.3.1.5"

  # dataDirectoryHostPath: "/opt/consul"

  grpc: true

  exposeGossipPorts: false

  resources:
    limits:
      cpu: 100m
      memory: 100Mi
    requests:
      cpu: 100m
      memory: 100Mi

  securityContext:
    runAsNonRoot: true
    runAsGroup: 1000
    runAsUser: 100
    fsGroup: 1000

  extraConfig: |
    {}

  dnsPolicy: "ClusterFirst"

  hostNetwork: false

  # [Enterprise Only]
  snapshotAgent:
    enabled: false

## DNS
##
dns:
  enabled: false

## UI
ui:
  enabled: false

## Sync Catalog
##
syncCatalog:
  enabled: true

  default: true

  toConsul: true

  toK8S: true

  k8sPrefix: null

  k8sAllowNamespaces: ["*"]

  k8sDenyNamespaces: ["kube-system", "kube-public"]

  # # [Enterprise Only]
  # consulNamespaces:
  #   consulDestinationNamespace: "default"
  #   mirroringK8S: false
  #   mirroringK8SPrefix: ""

  addK8SNamespaceSuffix: true

  consulPrefix: null

  k8sTag: "k8s"

  consulNodeName: "k8s-sync"

  syncClusterIPServices: true

  nodePortSyncType: ExternalFirst

  aclSyncToken:
    secretName: "consul-acl-sync-token"
    secretKey: "key"

  nodeSelector: |
    "mydomain.co/nodegroup": "ng-sys"

  affinity: null

  tolerations: null

  resources:
    limits:
      cpu: "50m"
      memory: "50Mi"
    requests:
      cpu: "50m"
      memory: "50Mi"

  logLevel: info

  consulWriteInterval: null

## ConnectInject
##
connectInject:
  enabled: true

  default: false

  transparentProxy:
    defaultEnabled: true

  metrics:
    defaultEnabled: "-"
    defaultEnableMerging: false
    defaultMergedMetricsPort: 20100
    defaultPrometheusScrapePort: 20200
    defaultPrometheusScrapePath: "/metrics"

  envoyExtraArgs: null

  logLevel: info

  resources:
    limits:
      cpu: 50m
      memory: 50Mi
    requests:
      cpu: 50m
      memory: 50Mi

  namespaceSelector: null

  k8sAllowNamespaces: ["*"]

  k8sDenyNamespaces: []

  # [Enterprise Only]
  consulNamespaces:
    consulDestinationNamespace: "default"
    mirroringK8S: false
    mirroringK8SPrefix: ""

  nodeSelector: |
    "mydomain.co/nodegroup": "ng-sys"

  affinity: null

  tolerations: null

  aclBindingRuleSelector: "serviceaccount.name!=default"

  overrideAuthMethodName: ""

  aclInjectToken:
    secretName: "consul-acl-inject-token"
    secretKey: "key"

  sidecarProxy:
    resources:
      limits:
        cpu: 100m
        memory: 100Mi
      requests:
        cpu: 100m
        memory: 100Mi

  initContainer:
    resources:
      limits:
        cpu: 50m
        memory: 150Mi
      requests:
        cpu: 50m
        memory: 25Mi

## Controller
controller:
  enabled: true

  replicas: 2

  logLevel: info

  resources:
    limits:
      cpu: 100m
      memory: 50Mi
    requests:
      cpu: 100m
      memory: 50Mi

  nodeSelector: |
    "mydomain.co/nodegroup": "ng-sys"

  affinity: |
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              app: {{ template "consul.name" . }}
              release: "{{ .Release.Name }}"
              component: controller
          topologyKey: kubernetes.io/hostname

  aclToken:
    secretName: "consul-acl-token"
    secretKey: "key"

## Mesh Gateways
##
meshGateway:
  enabled: false

## IngressGateways
##
ingressGateways:
  enabled: true

  defaults:
    replicas: 2

    service:
      type: ClusterIP

      ports:
        - port: 8080
          nodePort: null
        - port: 8443
          nodePort: null

      # annotations: |
      #   "service.beta.kubernetes.io/aws-load-balancer-type": "nlb-ip"
      #   "service.beta.kubernetes.io/aws-load-balancer-internal": "true"
      #   "service.beta.kubernetes.io/aws-load-balancer-proxy-protocol": "*"
      #   "service.beta.kubernetes.io/aws-load-balancer-ip-address-type": "ipv4"
      #   "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": "false"
      #   "service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": "preserve_client_ip.enabled=true"

      additionalSpec: null

    resources:
      limits:
        cpu: 100m
        memory: 100Mi
      requests:
        cpu: 100m
        memory: 100Mi

    initCopyConsulContainer:
      resources:
        limits:
          cpu: 50m
          memory: 150Mi
        requests:
          cpu: 50m
          memory: 25Mi

    affinity: |
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                app: {{ template "consul.name" . }}
                release: "{{ .Release.Name }}"
                component: ingress-gateway
            topologyKey: kubernetes.io/hostname

    tolerations: null

    nodeSelector: |
      mydomain.co/nodegroup: ng-sys

    priorityClassName: ""

    annotations:
      null
      # consul.hashicorp.com/connect-inject: true

    # [Enterprise Only]
    consulNamespace: "default"

  gateways:
    - name: ingress-gateway

## TerminatingGateways
##
terminatingGateways:
  enabled: false

  defaults:
    replicas: 3

    extraVolumes: []

    resources:
      limits:
        cpu: "100m"
        memory: "100Mi"
      requests:
        cpu: "100m"
        memory: "100Mi"

    initCopyConsulContainer:
      resources:
        limits:
          cpu: "50m"
          memory: "150Mi"
        requests:
          cpu: "50m"
          memory: "25Mi"

    affinity: |
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                app: {{ template "consul.name" . }}
                release: "{{ .Release.Name }}"
                component: terminating-gateway
            topologyKey: kubernetes.io/hostname

    nodeSelector: |
      "mydomain.co/nodegroup": "ng-sys"

    annotations: null

    # [Enterprise Only]
    consulNamespace: "default"

  gateways:
    - name: terminating-gateway

## Prometheus
prometheus:
  enabled: false

# Tests
tests:
  enabled: true

The Pod Description:

Name:         consul-ingress-gateway-575c97997f-9d6q6
Namespace:    consul
Priority:     0
Node:         ip-10-3-3-212.us-west-2.compute.internal/10.3.3.212
Start Time:   Mon, 24 May 2021 01:23:44 +0530
Labels:       app=consul
              chart=consul-helm
              component=ingress-gateway
              heritage=Helm
              ingress-gateway-name=consul-ingress-gateway
              pod-template-hash=575c97997f
              release=consul
Annotations:  consul.hashicorp.com/connect-inject: false
              kubernetes.io/psp: eks.privileged
Status:       Running
IP:           10.3.3.129
IPs:
  IP:           10.3.3.129
Controlled By:  ReplicaSet/consul-ingress-gateway-575c97997f
Init Containers:
  copy-consul-bin:
    Container ID:  docker://a3f2958ad2d3fe78edfde63ed7d28b410c60a8aed397cfbc24267a558c663692
    Image:         hashicorp/consul:1.9.5
    Image ID:      docker-pullable://hashicorp/consul@sha256:35f1bdb6c516a4fae6e4b056b0d4e9ddd0a3874efc43fc0dc8db49ef2b5d4442
    Port:          <none>
    Host Port:     <none>
    Command:
      cp
      /bin/consul
      /consul-bin/consul
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 24 May 2021 01:23:48 +0530
      Finished:     Mon, 24 May 2021 01:23:50 +0530
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     50m
      memory:  150Mi
    Requests:
      cpu:        50m
      memory:     25Mi
    Environment:  <none>
    Mounts:
      /consul-bin from consul-bin (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from consul-ingress-gateway-token-7x7zp (ro)
  service-init:
    Container ID:  docker://8733bc52327123195754f0bf77849086ece7cf2cba00dbeb3ab979fe8a0002a5
    Image:         hashicorp/consul-k8s:0.25.0
    Image ID:      docker-pullable://hashicorp/consul-k8s@sha256:66a1dfd964e9a8fe2477803462fd08cb83744a65f2b8083e1c51c580f6930c7d
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -ec
      consul-k8s service-address \
        -k8s-namespace=consul \
        -name=consul-ingress-gateway \
        -output-file=/tmp/address.txt
      WAN_ADDR="$(cat /tmp/address.txt)"
      WAN_PORT=8080

      cat > /consul/service/service.hcl << EOF
      service {
        kind = "ingress-gateway"
        name = "ingress-gateway"
        id = "${POD_NAME}"
        port = ${WAN_PORT}
        address = "${WAN_ADDR}"
        tagged_addresses {
          lan {
            address = "${POD_IP}"
            port = 21000
          }
          wan {
            address = "${WAN_ADDR}"
            port = ${WAN_PORT}
          }
        }
        proxy {
          config {
            envoy_gateway_no_default_bind = true
            envoy_gateway_bind_addresses {
              all-interfaces {
                address = "0.0.0.0"
              }
            }
          }
        }
        checks = [
          {
            name = "Ingress Gateway Listening"
            interval = "10s"
            tcp = "${POD_IP}:21000"
            deregister_critical_service_after = "6h"
          }
        ]
      }
      EOF

      /consul-bin/consul services register \
        /consul/service/service.hcl

    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Mon, 24 May 2021 01:23:54 +0530
      Finished:     Mon, 24 May 2021 01:23:57 +0530
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     50m
      memory:  50Mi
    Requests:
      cpu:     50m
      memory:  50Mi
    Environment:
      HOST_IP:            (v1:status.hostIP)
      POD_IP:             (v1:status.podIP)
      POD_NAME:          consul-ingress-gateway-575c97997f-9d6q6 (v1:metadata.name)
      CONSUL_HTTP_ADDR:  http://$(HOST_IP):8500
    Mounts:
      /consul-bin from consul-bin (rw)
      /consul/service from consul-service (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from consul-ingress-gateway-token-7x7zp (ro)
Containers:
  ingress-gateway:
    Container ID:  docker://4f7ab4f474d369bd8af91413102bc96abdad0d7dd88587c5fbf9666e446dfe78
    Image:         envoyproxy/envoy-alpine:v1.18.3
    Image ID:      docker-pullable://envoyproxy/envoy-alpine@sha256:6225750f76e1e995690ce4a512aaff3ba5d9d70426926ec821b867346a703c0e
    Ports:         21000/TCP, 8080/TCP, 8443/TCP
    Host Ports:    0/TCP, 0/TCP, 0/TCP
    Command:
      /consul-bin/consul
      connect
      envoy
      -gateway=ingress
      -proxy-id=$(POD_NAME)
      -address=$(POD_IP):21000
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    1
      Started:      Mon, 24 May 2021 01:30:13 +0530
      Finished:     Mon, 24 May 2021 01:30:16 +0530
    Ready:          False
    Restart Count:  6
    Limits:
      cpu:     100m
      memory:  100Mi
    Requests:
      cpu:      100m
      memory:   100Mi
    Liveness:   tcp-socket :21000 delay=30s timeout=5s period=10s #success=1 #failure=3
    Readiness:  tcp-socket :21000 delay=10s timeout=5s period=10s #success=1 #failure=3
    Environment:
      HOST_IP:            (v1:status.hostIP)
      POD_IP:             (v1:status.podIP)
      POD_NAME:          consul-ingress-gateway-575c97997f-9d6q6 (v1:metadata.name)
      CONSUL_HTTP_ADDR:  http://$(HOST_IP):8500
      CONSUL_GRPC_ADDR:  $(HOST_IP):8502
    Mounts:
      /consul-bin from consul-bin (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from consul-ingress-gateway-token-7x7zp (ro)
  consul-sidecar:
    Container ID:  docker://6500273e1c5f017814fa6cf5fd9c7607b165ff514fdb78de95967eb76cf92465
    Image:         hashicorp/consul-k8s:0.25.0
    Image ID:      docker-pullable://hashicorp/consul-k8s@sha256:66a1dfd964e9a8fe2477803462fd08cb83744a65f2b8083e1c51c580f6930c7d
    Port:          <none>
    Host Port:     <none>
    Command:
      consul-k8s
      consul-sidecar
      -service-config=/consul/service/service.hcl
      -consul-binary=/consul-bin/consul
    State:          Running
      Started:      Mon, 24 May 2021 01:23:59 +0530
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     20m
      memory:  50Mi
    Requests:
      cpu:     20m
      memory:  25Mi
    Environment:
      HOST_IP:            (v1:status.hostIP)
      POD_IP:             (v1:status.podIP)
      CONSUL_HTTP_ADDR:  http://$(HOST_IP):8500
    Mounts:
      /consul-bin from consul-bin (rw)
      /consul/service from consul-service (ro)
      /var/run/secrets/kubernetes.io/serviceaccount from consul-ingress-gateway-token-7x7zp (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  consul-bin:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:
    SizeLimit:  <unset>
  consul-service:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  consul-ingress-gateway-token-7x7zp:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  consul-ingress-gateway-token-7x7zp
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  mydomain.co/nodegroup=ng-sys
Tolerations:     node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                 node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                     From               Message
  ----     ------     ----                    ----               -------
  Normal   Scheduled  7m46s                   default-scheduler  Successfully assigned consul/consul-ingress-gateway-575c97997f-9d6q6 to ip-10-3-3-212.us-west-2.compute.internal
  Normal   Created    7m42s                   kubelet            Created container copy-consul-bin
  Normal   Started    7m42s                   kubelet            Started container copy-consul-bin
  Normal   Pulled     7m42s                   kubelet            Container image "hashicorp/consul:1.9.5" already present on machine
  Normal   Started    7m36s                   kubelet            Started container service-init
  Normal   Pulled     7m36s                   kubelet            Container image "hashicorp/consul-k8s:0.25.0" already present on machine
  Normal   Created    7m36s                   kubelet            Created container service-init
  Normal   Created    7m31s                   kubelet            Created container consul-sidecar
  Normal   Started    7m31s                   kubelet            Started container consul-sidecar
  Normal   Pulled     7m31s                   kubelet            Container image "hashicorp/consul-k8s:0.25.0" already present on machine
  Normal   Started    7m6s (x3 over 7m31s)    kubelet            Started container ingress-gateway
  Normal   Pulled     6m35s (x4 over 7m32s)   kubelet            Container image "envoyproxy/envoy-alpine:v1.18.3" already present on machine
  Normal   Created    6m35s (x4 over 7m32s)   kubelet            Created container ingress-gateway
  Warning  BackOff    2m41s (x25 over 7m25s)  kubelet            Back-off restarting failed container
thisisnotashwin commented 3 years ago

hey @debjitk,

would you happen to have the logs from the failing container? does it fail before starting up? it is a little difficult to identify what could be the potential problem here without the logs unfortunately

d3bt3ch commented 3 years ago

@thisisnotashwin The issue is fixed. According to logs there was configuration issue with envoy v1.18.x. Once I downgraded the envoy to v1.16.x everything is working fine.