Closed Patrick0308 closed 1 year ago
hello. 目前我也在尝试用envoy作为global-sidecar的poc,目前遇到流量到global-sidecar后无法分发到正确des的问题。
可以看到global-sidecar envoy的boostrap配置如下:
2023-06-29T07:57:24.661009Z info Envoy command: [-c etc/istio/proxy/envoy-rev0.json --restart-epoch 0 --drain-time-s 45 --parent-shutdown-time-s 60 --service-cluster global-sidecar.istio-mesh-operator --service-node router~10.215.15.24~istio-global-sidecar-ingressgateway-de6f2906-6fbb49dd45-ddk64.istio-mesh-operator~istio-mesh-operator.svc.cluster.local --local-address-ip-version v4 --bootstrap-version 3 --log-format %Y-%m-%dT%T.%fZ %l envoy %n %v -l warning --component-log-level misc:error --config-yaml {
"static_resources": {
"listeners": [
{
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 20000
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"codec_type": "AUTO",
"stat_prefix": "health",
"use_remote_address": true,
"route_config": {
"name": "health",
"virtual_hosts": [
{
"name": "all",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 200,
"body": {
"inline_string": "success"
}
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
]
}
}
]
}
]
},
{
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 80
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"codec_type": "AUTO",
"stat_prefix": "egress_http",
"common_http_protocol_options": {
"idle_timeout": "840s"
},
"use_remote_address": true,
"route_config": {
"name": "proxy",
"request_headers_to_remove": [
"Slime-Orig-Dest"
],
"virtual_hosts": [
{
"name": "all",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "original_dst_cluster"
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.lua",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua",
"inlineCode": "function insert_namespace(host, namespace) if string.find(host, '%.') then return host end if not namespace or namespace == '' then return host end local idx = string.find(host, ':') if idx then local prefix = string.sub(host, 1, idx-1) local suffix = string.sub(host, idx) return prefix .. \".\" .. namespace .. suffix end return host .. '.' .. namespace end\nfunction envoy_on_request(request_handle) local sourceNs = request_handle:headers():get(\"Slime-Source-Ns\") local reqHost = request_handle:headers():get(\":authority\") reqHost = insert_namespace(reqHost, sourceNs) request_handle:headers():replace(\":authority\", reqHost) end"
}
},
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
]
}
}
]
}
]
}
],
"clusters": [
{
"cleanupInterval": "8640000s",
"connectTimeout": "10s",
"name": "original_dst_cluster",
"type": "ORIGINAL_DST",
"lbPolicy": "CLUSTER_PROVIDED",
"circuitBreakers": {
"thresholds": [
{
"maxConnections": 4294967295,
"maxPendingRequests": 4294967295,
"maxRequests": 4294967295,
"maxRetries": 4294967295
}
]
},
"protocolSelection": "USE_DOWNSTREAM_PROTOCOL",
"filters": [
{
"name": "istio.metadata_exchange",
"typedConfig": {
"@type": "type.googleapis.com/udpa.type.v1.TypedStruct",
"typeUrl": "type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange",
"value": {
"protocol": "istio-peer-exchange"
}
}
}
]
},
{
"name": "lazyload-accesslog-source",
"type": "STRICT_DNS",
"connect_timeout": "5s",
"http2_protocol_options": {},
"dns_lookup_family": "V4_ONLY",
"load_assignment": {
"cluster_name": "lazyload-accesslog-source",
"endpoints": [{
"lb_endpoints": [{
"endpoint": {
"address": {
"socket_address": {
"address": "lazyload.istio-mesh-operator",
"port_value": 8082
}
}
}
}]
}]
},
"respect_dns_ttl": true
}
]
}
}
--concurrency 2]
2023-06-29T15:57:24.706297Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T15:57:24.706374Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T15:57:24.706928Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T15:57:24.706972Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T07:57:24.771976Z info xdsproxy connected to upstream XDS server: istiod.istio-system.svc:15012
2023-06-29T07:57:25.285948Z warn Envoy proxy is NOT ready: config not received from Pilot (is Pilot running?): cds updates: 1 successful, 0 rejected; lds updates: 0 successful, 0 rejected
2023-06-29T07:57:27.186564Z info Initialization took 6.650089338s
2023-06-29T07:57:27.186611Z info Envoy proxy is ready
用一个简单的bookinfo实例触发流量,查看global-sidecar中envoy日志,这一条是从productpage到details
2023-06-29T16:12:52.904530Z debug envoy http [C2] new stream
2023-06-29T16:12:52.904749Z debug envoy http [C2][S13502177400256495316] request headers complete (end_stream=true):
':authority', 'details:9080'
':path', '/details/0'
':method', 'GET'
'user-agent', 'curl/7.64.0'
'accept-encoding', 'gzip, deflate'
'accept', '*/*'
'x-request-id', 'b28fb898-fce1-4349-bdbf-96e33945a51d'
'x-forwarded-proto', 'http'
'x-envoy-peer-metadata', 'Ch8KDkFQUF9DT05UQUlORVJTEg0aC3Byb2R1Y3RwYWdlChoKCkNMVVNURVJfSUQSDBoKS3ViZXJuZXRlcwoaCg1JU1RJT19WRVJTSU9OEgkaBzEuOS1kZXYKrAMKBkxBQkVMUxKhAyqeAwoUCgNhcHASDRoLcHJvZHVjdHBhZ2UKGQoMaXN0aW8uaW8vcmV2EgkaB2RlZmF1bHQKIQoRcG9kLXRlbXBsYXRlLWhhc2gSDBoKNzZjOWM5NDRkZgoRCgdwcm9qZWN0EgYaBHRlc3QKGgoHcmVsZWFzZRIPGg0zMTAxLWJvb2tpbmZvCiQKGXNlY3VyaXR5LmlzdGlvLmlvL3Rsc01vZGUSBxoFaXN0aW8KMAofc2VydmljZS5pc3Rpby5pby9jYW5vbmljYWwtbmFtZRINGgtwcm9kdWN0cGFnZQorCiNzZXJ2aWNlLmlzdGlvLmlvL2Nhbm9uaWNhbC1yZXZpc2lvbhIEGgJ2MQoWCg5zeW0tYXBpdmVyc2lvbhIEGgJ2MwobCgdzeW0tYXBwEhAaDnByb2R1Y3RwYWdlLXYxCh4KFHN5bS1tYW5hZ2VyLWluamVjdGVkEgYaBHRydWUKLgoEdXVpZBImGiQxYTFjOWU5MC1mYmQ3LTQzOWQtOGQyMi1lNjMzNjU2MTllYzEKDwoHdmVyc2lvbhIEGgJ2MQoaCgdNRVNIX0lEEg8aDWNsdXN0ZXIubG9jYWwKKQoETkFNRRIhGh9wcm9kdWN0cGFnZS12MS03NmM5Yzk0NGRmLWpsOWZ6CigKCU5BTUVTUEFDRRIbGhl0ZXN0LXpvdXJ1aS1kby1ub3QtZGVsZXRlCmQKBU9XTkVSElsaWWt1YmVybmV0ZXM6Ly9hcGlzL2FwcHMvdjEvbmFtZXNwYWNlcy90ZXN0LXpvdXJ1aS1kby1ub3QtZGVsZXRlL2RlcGxveW1lbnRzL3Byb2R1Y3RwYWdlLXYxChcKEVBMQVRGT1JNX01FVEFEQVRBEgIqAAohCg1XT1JLTE9BRF9OQU1FEhAaDnByb2R1Y3RwYWdlLXYx'
'x-envoy-peer-metadata-id', 'sidecar~10.215.52.22~productpage-v1-76c9c944df-jl9fz.test-zourui-do-not-delete~test-zourui-do-not-delete.svc.cluster.local'
'slime-source-ns', 'test-zourui-do-not-delete'
'x-b3-traceid', 'ba9628c7f6326fa28e70ef6a57c2f828'
'x-b3-spanid', '1960393687a785f1'
'x-b3-parentspanid', '206171532c0a960e'
'x-b3-sampled', '0'
'slime-orig-dest', '10.212.168.62:9080'
'content-length', '0'
2023-06-29T16:12:52.904765Z debug envoy http [C2][S13502177400256495316] request end stream
2023-06-29T16:12:52.904930Z debug envoy lua coroutine finished
2023-06-29T16:12:52.904961Z debug envoy router [C2][S13502177400256495316] cluster 'original_dst_cluster' match for URL '/details/0'
2023-06-29T16:12:52.905003Z warning envoy upstream original_dst_load_balancer: No downstream connection or no original_dst.
2023-06-29T16:12:52.905011Z debug envoy upstream no healthy host for HTTP connection pool
2023-06-29T16:12:52.905082Z debug envoy http [C2][S13502177400256495316] Sending local reply with details no_healthy_upstream
2023-06-29T16:12:52.905168Z debug envoy http [C2][S13502177400256495316] encoding headers via codec (end_stream=false):
':status', '503'
'content-length', '19'
'content-type', 'text/plain'
'date', 'Thu, 29 Jun 2023 08:12:52 GMT'
'server', 'envoy'
2023-06-29T16:12:52.927247Z debug envoy http [C2] new stream
2023-06-29T16:12:52.927327Z debug envoy http [C2][S427227805447913891] request headers complete (end_stream=true):
':authority', 'reviews:9080'
':path', '/reviews/0'
':method', 'GET'
'user-agent', 'curl/7.64.0'
'accept-encoding', 'gzip, deflate'
'accept', '*/*'
'x-request-id', 'b28fb898-fce1-4349-bdbf-96e33945a51d'
'x-forwarded-proto', 'http'
'x-envoy-peer-metadata', 'Ch8KDkFQUF9DT05UQUlORVJTEg0aC3Byb2R1Y3RwYWdlChoKCkNMVVNURVJfSUQSDBoKS3ViZXJuZXRlcwoaCg1JU1RJT19WRVJTSU9OEgkaBzEuOS1kZXYKrAMKBkxBQkVMUxKhAyqeAwoUCgNhcHASDRoLcHJvZHVjdHBhZ2UKGQoMaXN0aW8uaW8vcmV2EgkaB2RlZmF1bHQKIQoRcG9kLXRlbXBsYXRlLWhhc2gSDBoKNzZjOWM5NDRkZgoRCgdwcm9qZWN0EgYaBHRlc3QKGgoHcmVsZWFzZRIPGg0zMTAxLWJvb2tpbmZvCiQKGXNlY3VyaXR5LmlzdGlvLmlvL3Rsc01vZGUSBxoFaXN0aW8KMAofc2VydmljZS5pc3Rpby5pby9jYW5vbmljYWwtbmFtZRINGgtwcm9kdWN0cGFnZQorCiNzZXJ2aWNlLmlzdGlvLmlvL2Nhbm9uaWNhbC1yZXZpc2lvbhIEGgJ2MQoWCg5zeW0tYXBpdmVyc2lvbhIEGgJ2MwobCgdzeW0tYXBwEhAaDnByb2R1Y3RwYWdlLXYxCh4KFHN5bS1tYW5hZ2VyLWluamVjdGVkEgYaBHRydWUKLgoEdXVpZBImGiQxYTFjOWU5MC1mYmQ3LTQzOWQtOGQyMi1lNjMzNjU2MTllYzEKDwoHdmVyc2lvbhIEGgJ2MQoaCgdNRVNIX0lEEg8aDWNsdXN0ZXIubG9jYWwKKQoETkFNRRIhGh9wcm9kdWN0cGFnZS12MS03NmM5Yzk0NGRmLWpsOWZ6CigKCU5BTUVTUEFDRRIbGhl0ZXN0LXpvdXJ1aS1kby1ub3QtZGVsZXRlCmQKBU9XTkVSElsaWWt1YmVybmV0ZXM6Ly9hcGlzL2FwcHMvdjEvbmFtZXNwYWNlcy90ZXN0LXpvdXJ1aS1kby1ub3QtZGVsZXRlL2RlcGxveW1lbnRzL3Byb2R1Y3RwYWdlLXYxChcKEVBMQVRGT1JNX01FVEFEQVRBEgIqAAohCg1XT1JLTE9BRF9OQU1FEhAaDnByb2R1Y3RwYWdlLXYx'
'x-envoy-peer-metadata-id', 'sidecar~10.215.52.22~productpage-v1-76c9c944df-jl9fz.test-zourui-do-not-delete~test-zourui-do-not-delete.svc.cluster.local'
'slime-source-ns', 'test-zourui-do-not-delete'
'x-b3-traceid', 'ba9628c7f6326fa28e70ef6a57c2f828'
'x-b3-spanid', 'c3d09b54e38054f8'
'x-b3-parentspanid', '206171532c0a960e'
'x-b3-sampled', '0'
'slime-orig-dest', '10.212.168.96:9080'
'content-length', '0'
2023-06-29T16:12:52.927336Z debug envoy http [C2][S427227805447913891] request end stream
2023-06-29T16:12:52.927403Z debug envoy lua coroutine finished
2023-06-29T16:12:52.927421Z debug envoy router [C2][S427227805447913891] cluster 'original_dst_cluster' match for URL '/reviews/0'
2023-06-29T16:12:52.927431Z warning envoy upstream original_dst_load_balancer: No downstream connection or no original_dst.
2023-06-29T16:12:52.927434Z debug envoy upstream no healthy host for HTTP connection pool
2023-06-29T16:12:52.927453Z debug envoy http [C2][S427227805447913891] Sending local reply with details no_healthy_upstream
2023-06-29T16:12:52.927478Z debug envoy http [C2][S427227805447913891] encoding headers via codec (end_stream=false):
':status', '503'
'content-length', '19'
'content-type', 'text/plain'
'date', 'Thu, 29 Jun 2023 08:12:52 GMT'
'server', 'envoy'
2023-06-29T16:12:52.951242Z debug envoy http [C2] new stream
2023-06-29T16:12:52.951297Z debug envoy http [C2][S2282158311367245600] request headers complete (end_stream=true):
':authority', 'reviews:9080'
':path', '/reviews/0'
':method', 'GET'
'user-agent', 'curl/7.64.0'
'accept-encoding', 'gzip, deflate'
'accept', '*/*'
'x-request-id', 'b28fb898-fce1-4349-bdbf-96e33945a51d'
'x-forwarded-proto', 'http'
'x-envoy-peer-metadata', 'Ch8KDkFQUF9DT05UQUlORVJTEg0aC3Byb2R1Y3RwYWdlChoKCkNMVVNURVJfSUQSDBoKS3ViZXJuZXRlcwoaCg1JU1RJT19WRVJTSU9OEgkaBzEuOS1kZXYKrAMKBkxBQkVMUxKhAyqeAwoUCgNhcHASDRoLcHJvZHVjdHBhZ2UKGQoMaXN0aW8uaW8vcmV2EgkaB2RlZmF1bHQKIQoRcG9kLXRlbXBsYXRlLWhhc2gSDBoKNzZjOWM5NDRkZgoRCgdwcm9qZWN0EgYaBHRlc3QKGgoHcmVsZWFzZRIPGg0zMTAxLWJvb2tpbmZvCiQKGXNlY3VyaXR5LmlzdGlvLmlvL3Rsc01vZGUSBxoFaXN0aW8KMAofc2VydmljZS5pc3Rpby5pby9jYW5vbmljYWwtbmFtZRINGgtwcm9kdWN0cGFnZQorCiNzZXJ2aWNlLmlzdGlvLmlvL2Nhbm9uaWNhbC1yZXZpc2lvbhIEGgJ2MQoWCg5zeW0tYXBpdmVyc2lvbhIEGgJ2MwobCgdzeW0tYXBwEhAaDnByb2R1Y3RwYWdlLXYxCh4KFHN5bS1tYW5hZ2VyLWluamVjdGVkEgYaBHRydWUKLgoEdXVpZBImGiQxYTFjOWU5MC1mYmQ3LTQzOWQtOGQyMi1lNjMzNjU2MTllYzEKDwoHdmVyc2lvbhIEGgJ2MQoaCgdNRVNIX0lEEg8aDWNsdXN0ZXIubG9jYWwKKQoETkFNRRIhGh9wcm9kdWN0cGFnZS12MS03NmM5Yzk0NGRmLWpsOWZ6CigKCU5BTUVTUEFDRRIbGhl0ZXN0LXpvdXJ1aS1kby1ub3QtZGVsZXRlCmQKBU9XTkVSElsaWWt1YmVybmV0ZXM6Ly9hcGlzL2FwcHMvdjEvbmFtZXNwYWNlcy90ZXN0LXpvdXJ1aS1kby1ub3QtZGVsZXRlL2RlcGxveW1lbnRzL3Byb2R1Y3RwYWdlLXYxChcKEVBMQVRGT1JNX01FVEFEQVRBEgIqAAohCg1XT1JLTE9BRF9OQU1FEhAaDnByb2R1Y3RwYWdlLXYx'
'x-envoy-peer-metadata-id', 'sidecar~10.215.52.22~productpage-v1-76c9c944df-jl9fz.test-zourui-do-not-delete~test-zourui-do-not-delete.svc.cluster.local'
'slime-source-ns', 'test-zourui-do-not-delete'
'x-b3-traceid', 'ba9628c7f6326fa28e70ef6a57c2f828'
'x-b3-spanid', 'cb1d49a8a184423f'
'x-b3-parentspanid', '206171532c0a960e'
'x-b3-sampled', '0'
'slime-orig-dest', '10.212.168.96:9080'
'content-length', '0'
2023-06-29T16:12:52.951307Z debug envoy http [C2][S2282158311367245600] request end stream
2023-06-29T16:12:52.951343Z debug envoy lua coroutine finished
2023-06-29T16:12:52.951354Z debug envoy router [C2][S2282158311367245600] cluster 'original_dst_cluster' match for URL '/reviews/0'
2023-06-29T16:12:52.951361Z warning envoy upstream original_dst_load_balancer: No downstream connection or no original_dst.
2023-06-29T16:12:52.951375Z debug envoy upstream no healthy host for HTTP connection pool
2023-06-29T16:12:52.951395Z debug envoy http [C2][S2282158311367245600] Sending local reply with details no_healthy_upstream
2023-06-29T16:12:52.951442Z debug envoy http [C2][S2282158311367245600] encoding headers via codec (end_stream=false):
':status', '503'
'content-length', '19'
'content-type', 'text/plain'
'date', 'Thu, 29 Jun 2023 08:12:52 GMT'
'server', 'envoy'
再查看productpage处envoy日志:
{
"x_forwarded_for": null,
"start_time": "2023-06-29T08:12:52.929+0000",
"method": "GET",
"requested_server_name": null,
"istio_policy_status": null,
"duration": 1,
"protocol": "HTTP/1.1",
"route_name": null,
"upstream_transport_failure_reason": null,
"response_flags": "-",
"path": "/reviews/0",
"user_agent": "curl/7.64.0",
"bytes_sent": 19,
"request_id": "b28fb898-fce1-4349-bdbf-96e33945a51d",
"upstream_local_address": "10.215.52.22:33616",
"upstream_host": "10.215.15.24:80",
"downstream_remote_address": "10.215.52.22:44750",
"downstream_local_address": "10.212.168.96:9080",
"response_code_details": "via_upstream",
"upstream_service_time": "0",
"response_code": 503,
"tracing_id": null,
"upstream_cluster": "outbound|80||istio-global-sidecar-ingressgateway-a1b89d20.istio-mesh-operator.svc.cluster.local",
"authority": "reviews:9080",
"bytes_received": 0
}
{
"upstream_service_time": "0",
"user_agent": "curl/7.64.0",
"request_id": "b28fb898-fce1-4349-bdbf-96e33945a51d",
"response_code": 503,
"duration": 1,
"method": "GET",
"downstream_remote_address": "10.215.52.22:44752",
"tracing_id": null,
"x_forwarded_for": null,
"response_flags": "-",
"downstream_local_address": "10.212.168.96:9080",
"istio_policy_status": null,
"bytes_sent": 19,
"protocol": "HTTP/1.1",
"response_code_details": "via_upstream",
"route_name": null,
"start_time": "2023-06-29T08:12:52.953+0000",
"upstream_local_address": "10.215.52.22:33616",
"path": "/reviews/0",
"upstream_cluster": "outbound|80||istio-global-sidecar-ingressgateway-a1b89d20.istio-mesh-operator.svc.cluster.local",
"upstream_transport_failure_reason": null,
"requested_server_name": null,
"bytes_received": 0,
"authority": "reviews:9080",
"upstream_host": "10.215.15.24:80"
}
{
"istio_policy_status": null,
"start_time": "2023-06-29T08:12:52.894+0000",
"user_agent": "curl/7.64.0",
"protocol": "HTTP/1.1",
"response_code": 200,
"downstream_remote_address": "10.248.41.74:0",
"upstream_service_time": "61",
"bytes_received": 0,
"upstream_local_address": "127.0.0.1:52520",
"route_name": "default",
"request_id": "b28fb898-fce1-4349-bdbf-96e33945a51d",
"authority": "10.215.52.9:804",
"path": "/productpage",
"upstream_host": "127.0.0.1:9080",
"duration": 62,
"tracing_id": null,
"response_flags": "-",
"requested_server_name": null,
"x_forwarded_for": "10.248.41.74",
"upstream_transport_failure_reason": null,
"bytes_sent": 3769,
"upstream_cluster": "inbound|9080||",
"method": "GET",
"downstream_local_address": "10.215.52.22:9080",
"response_code_details": "via_upstream"
}
这个报错可能是什么原因呢?另外这里可以直接用passthrough替换cluster 'original_dst_cluster'吗?
hello. 目前我也在尝试用envoy作为global-sidecar的poc,目前遇到流量到global-sidecar后无法分发到正确des的问题。
lua添加上去就是为了解决这个短域名问题
@YonkaFang 我们可以先合入
一些chart的调整以及使用的调整,我来跟进
用 envoy 替换的 gs 的方案,有计划支持 mtls 流量吗
hello. 目前我也在尝试用envoy作为global-sidecar的poc,目前遇到流量到global-sidecar后无法分发到正确des的问题。
lua添加上去就是为了解决这个短域名问题
感谢回复,这个问题已经解决了。另外想请问下,用 envoy 作为 gs 的方案有计划支持 mtls 流量吗?
lazyload using envoy as global sidecar to support h1/h2. See details on #373.
Close #373
TODO: