Envoy pods fail to start if kubernetes-route-as header / subdomain consists only of numbers

[scp-mb]

As per title, envoy pods do not start correctly using just numbers with the kubernetes-route-as header, e.g. with the annotation routing.visualstudio.io/route-on-header=kubernetes-route-as=6514 on a pod the following log entry is seen during startup of the envoy pods (note: there is also another service already set up under review-1496 in these logs, that one works fine and can be ignored).

Some information has been redacted where required

Unable to parse JSON as proto (INVALID_ARGUMENT:(route_config.virtual_hosts[4].routes[0].match.headers[0].exact_match): invalid value 6514 for type TYPE_STRING): {
  "admin": {
    "access_log_path": "/tmp/admin_access.log"
  },
  "static_resources": {
    "clusters": [
      {
        "type": "strict_dns",
        "load_assignment": {
          "cluster_name": "service_original_clone_80_80",
          "endpoints": [
            {
              "lb_endpoints": [
                {
                  "endpoint": {
                    "address": {
                      "socket_address": {
                        "address": "gatewayapi-cloned-routing-svc.redacted-namespace",
                        "port_value": 80
                      }
                    }
                  }
                }
              ]
            }
          ]
        },
        "name": "service_original_clone_80_80",
        "http_protocol_options": {},
        "connect_timeout": "1.00s"
      },
      {
        "http_protocol_options": {},
        "connect_timeout": "1.00s",
        "type": "static",
        "load_assignment": {
          "cluster_name": "service_debug_withHeader_kubernetes-route-as_6514_80_80",
          "endpoints": [
            {
              "lb_endpoints": [
                {
                  "endpoint": {
                    "address": {
                      "socket_address": {
                        "address": "10.240.0.252",
                        "port_value": 80
                      }
                    }
                  }
                }
              ]
            }
          ]
        },
        "name": "service_debug_withHeader_kubernetes-route-as_6514_80_80"
      }
    ],
    "listeners": [
      {
        "name": "listener_80_80",
        "filter_chains": [
          {
            "filter_chain_match": {
              "application_protocols": [
                "http/1.0",
                "http/1.1",
                "h2c"
              ]
            },
            "filters": [
              {
                "typed_config": {
                  "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
                  "route_config": {
                    "name": "listener_80_80_route",
                    "virtual_hosts": [
                      {
                        "routes": [
                          {
                            "route": {
                              "cluster": "service_debug_withHeader_kubernetes-route-as_6514_80_80",
                              "timeout": "0s",
                              "idle_timeout": "0s"
                            },
                            "match": {
                              "prefix": "/"
                            },
                            "request_headers_to_add": [
                              {
                                "header": {
                                  "value": 6514,
                                  "key": "kubernetes-route-as"
                                },
                                "append": false
                              }
                            ]
                          }
                        ],
                        "domains": [
                          "6514.api.subdomain2.company.net"
                        ],
                        "name": "listener_80_80_route_ingress_withDomain_6514.api.subdomain2.company.net"
                      },
                      {
                        "routes": [
                          {
                            "match": {
                              "prefix": "/"
                            },
                            "request_headers_to_add": [
                              {
                                "header": {
                                  "value": "review-1496",
                                  "key": "kubernetes-route-as"
                                },
                                "append": false
                              }
                            ],
                            "route": {
                              "cluster": "service_original_clone_80_80",
                              "timeout": "0s",
                              "idle_timeout": "0s"
                            }
                          }
                        ],
                        "domains": [
                          "review-1496.api.subdomain2.company.net"
                        ],
                        "name": "listener_80_80_route_ingress_withDomain_review-1496.api.subdomain2.company.net"
                      },
                      {
                        "routes": [
                          {
                            "request_headers_to_add": [
                              {
                                "header": {
                                  "value": 6514,
                                  "key": "kubernetes-route-as"
                                },
                                "append": false
                              }
                            ],
                            "route": {
                              "cluster": "service_debug_withHeader_kubernetes-route-as_6514_80_80",
                              "timeout": "0s",
                              "idle_timeout": "0s"
                            },
                            "match": {
                              "prefix": "/"
                            }
                          }
                        ],
                        "domains": [
                          "6514.api.subdomain1.company.net"
                        ],
                        "name": "listener_80_80_route_ingress_withDomain_6514.api.subdomain1.company.net"
                      },
                      {
                        "routes": [
                          {
                            "route": {
                              "cluster": "service_original_clone_80_80",
                              "timeout": "0s",
                              "idle_timeout": "0s"
                            },
                            "match": {
                              "prefix": "/"
                            },
                            "request_headers_to_add": [
                              {
                                "append": false,
                                "header": {
                                  "value": "review-1496",
                                  "key": "kubernetes-route-as"
                                }
                              }
                            ]
                          }
                        ],
                        "domains": [
                          "review-1496.api.subdomain1.company.net"
                        ],
                        "name": "listener_80_80_route_ingress_withDomain_review-1496.api.subdomain1.company.net"
                      },
                      {
                        "routes": [
                          {
                            "route": {
                              "idle_timeout": "0s",
                              "cluster": "service_debug_withHeader_kubernetes-route-as_6514_80_80",
                              "timeout": "0s"
                            },
                            "match": {
                              "prefix": "/",
                              "headers": [
                                {
                                  "exact_match": 6514,
                                  "name": "kubernetes-route-as"
                                }
                              ]
                            }
                          },
                          {
                            "route": {
                              "cluster": "service_original_clone_80_80",
                              "timeout": "0s",
                              "idle_timeout": "0s"
                            },
                            "match": {
                              "prefix": "/"
                            }
                          }
                        ],
                        "domains": [
                          "*"
                        ],
                        "name": "listener_80_80_route_default"
                      }
                    ]
                  },
                  "codec_type": "auto",
                  "http_filters": [
                    {
                      "name": "envoy.filters.http.router"
                    }
                  ],
                  "stat_prefix": "listener_80_80"
                },
                "name": "envoy.http_connection_manager"
              }
            ]
          }
        ],
        "listener_filters": [
          {
            "name": "envoy.filters.listener.http_inspector"
          }
        ],
        "address": {
          "socket_address": {
            "address": "0.0.0.0",
            "port_value": 80
          }
        }
      }
    ]
  }
}

And the configmap for the relevant envoy pod:

static_resources:
  listeners:
  - name: listener_80_80
    address:
      socket_address:
        address: 0.0.0.0
        port_value: 80
    listener_filters:
    - name: envoy.filters.listener.http_inspector
    filter_chains:
    - filter_chain_match:
        application_protocols:
        - http/1.0
        - http/1.1
        - h2c
      filters:
      - name: envoy.http_connection_manager
        typed_config:
          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          codec_type: auto
          stat_prefix: listener_80_80
          route_config:
            name: listener_80_80_route
            virtual_hosts:
            - name: listener_80_80_route_ingress_withDomain_6514.api.subdomain2.company.net
              domains:
              - 6514.api.subdomain2.company.net
              routes:
              - match:
                  prefix: /
                route:
                  cluster: service_debug_withHeader_kubernetes-route-as_6514_80_80
                  timeout: 0s
                  idle_timeout: 0s
                request_headers_to_add:
                - header:
                    key: kubernetes-route-as
                    value: 6514
                  append: false
            - name: listener_80_80_route_ingress_withDomain_review-1496.api.subdomain2.company.net
              domains:
              - review-1496.api.subdomain2.company.net
              routes:
              - match:
                  prefix: /
                route:
                  cluster: service_original_clone_80_80
                  timeout: 0s
                  idle_timeout: 0s
                request_headers_to_add:
                - header:
                    key: kubernetes-route-as
                    value: review-1496
                  append: false
            - name: listener_80_80_route_ingress_withDomain_6514.api.subdomain1.company.net
              domains:
              - 6514.api.subdomain1.company.net
              routes:
              - match:
                  prefix: /
                route:
                  cluster: service_debug_withHeader_kubernetes-route-as_6514_80_80
                  timeout: 0s
                  idle_timeout: 0s
                request_headers_to_add:
                - header:
                    key: kubernetes-route-as
                    value: 6514
                  append: false
            - name: listener_80_80_route_ingress_withDomain_review-1496.api.subdomain1.company.net
              domains:
              - review-1496.api.subdomain1.company.net
              routes:
              - match:
                  prefix: /
                route:
                  cluster: service_original_clone_80_80
                  timeout: 0s
                  idle_timeout: 0s
                request_headers_to_add:
                - header:
                    key: kubernetes-route-as
                    value: review-1496
                  append: false
            - name: listener_80_80_route_default
              domains:
              - '*'
              routes:
              - match:
                  headers:
                  - name: kubernetes-route-as
                    exact_match: 6514
                  prefix: /
                route:
                  cluster: service_debug_withHeader_kubernetes-route-as_6514_80_80
                  timeout: 0s
                  idle_timeout: 0s
              - match:
                  prefix: /
                route:
                  cluster: service_original_clone_80_80
                  timeout: 0s
                  idle_timeout: 0s
          http_filters:
          - name: envoy.filters.http.router
  clusters:
  - name: service_original_clone_80_80
    connect_timeout: 1.00s
    type: strict_dns
    load_assignment:
      cluster_name: service_original_clone_80_80
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: gatewayapi-cloned-routing-svc.redacted-namespace
                port_value: 80
    http_protocol_options: {}
  - name: service_debug_withHeader_kubernetes-route-as_6514_80_80
    connect_timeout: 1.00s
    type: static
    load_assignment:
      cluster_name: service_debug_withHeader_kubernetes-route-as_6514_80_80
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 10.240.0.252
                port_value: 80
    http_protocol_options: {}
admin:
  access_log_path: /tmp/admin_access.log

[amsoedal]

Hi @scp-mb, is there a reason not to use an alphanumeric string in this case? If this is a limitation with Envoy, there's not much we can do in the short term besides guarding against numeric-only values.

[scp-mb]

No hard requirement, but as part of working around #289 I shortened the release name to just match our internal work item number.

I presume the envoy configmap is generated by the routingmanager pod? If so, would a fix not be as simple as just adding quotes around the value within the yaml to force it to be interpreted as a string?

[amsoedal]

Fair enough. Will log a bug on our side to investigate