berylshow
commented
2 years ago proxyDefaults
$ kubectl get proxyDefaults -n jaeger NAME SYNCED AGE global True 129m
The Jaeger configuration information distributed can also be seen in the file in the app sidecar envoy container envoy-bootstarp.yaml
{ "admin": { "access_log_path": "/dev/null", "address": { "socket_address": { "address": "127.0.0.1", "port_value": 19000 } } }, "node": { "cluster": "api", "id": "api-deployment-v1-7f9869c476-hr7dl-api-sidecar-proxy", "metadata": { "namespace": "default", "envoy_version": "1.18.3" } }, "static_resources": { "clusters": [ { "name": "local_agent", "ignore_health_on_host_removal": false, "connect_timeout": "1s", "type": "STATIC", "http2_protocol_options": {}, "loadAssignment": { "clusterName": "local_agent", "endpoints": [ { "lbEndpoints": [ { "endpoint": { "address": { "socket_address": { "address": "10.111.4.252", "port_value": 8502 } } } } ] } ] } }, { "connect_timeout": "3.000s", "dns_lookup_family": "V4_ONLY", "lb_policy": "ROUND_ROBIN", "load_assignment": { "cluster_name": "jaeger_9411", "endpoints": [ { "lb_endpoints": [ { "endpoint": { "address": { "socket_address": { "address": "jaeger-collector", "port_value": 9411, "protocol": "TCP" } } } } ] } ] }, "name": "jaeger_9411", "type": "STRICT_DNS" } ] }, "stats_config": { "stats_tags": [ { "regex": "^cluster\.(?:passthrough~)?((?:([^.]+)~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.destination.custom_hash" }, { "regex": "^cluster\.(?:passthrough~)?((?:[^.]+~)?(?:([^.]+)\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.destination.service_subset" }, { "regex": "^cluster\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\.)?([^.]+)\.[^.]+\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.destination.service" }, { "regex": "^cluster\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.([^.]+)\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.destination.namespace" }, { "regex": "^cluster\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.([^.]+)\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.destination.datacenter" }, { "regex": "^cluster\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.([^.]+)\.[^.]+\.consul\.)", "tag_name": "consul.destination.routing_type" }, { "regex": "^cluster\.(?:passthrough~)?((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.([^.]+)\.consul\.)", "tag_name": "consul.destination.trust_domain" }, { "regex": "^cluster\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+)\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.destination.target" }, { "regex": "^cluster\.(?:passthrough~)?(((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.[^.]+)\.consul\.)", "tag_name": "consul.destination.full_target" }, { "regex": "^(?:tcp|http)\.upstream\.(([^.]+)(?:\.[^.]+)?\.[^.]+\.)", "tag_name": "consul.upstream.service" }, { "regex": "^(?:tcp|http)\.upstream\.([^.]+(?:\.[^.]+)?\.([^.]+)\.)", "tag_name": "consul.upstream.datacenter" }, { "regex": "^(?:tcp|http)\.upstream\.([^.]+(?:\.([^.]+))?\.[^.]+\.)", "tag_name": "consul.upstream.namespace" }, { "regex": "^cluster\.((?:([^.]+)~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.custom_hash" }, { "regex": "^cluster\.((?:[^.]+~)?(?:([^.]+)\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.service_subset" }, { "regex": "^cluster\.((?:[^.]+~)?(?:[^.]+\.)?([^.]+)\.[^.]+\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.service" }, { "regex": "^cluster\.((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.([^.]+)\.[^.]+\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.namespace" }, { "regex": "^cluster\.((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.([^.]+)\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.datacenter" }, { "regex": "^cluster\.((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.([^.]+)\.[^.]+\.consul\.)", "tag_name": "consul.routing_type" }, { "regex": "^cluster\.((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.([^.]+)\.consul\.)", "tag_name": "consul.trust_domain" }, { "regex": "^cluster\.(((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+)\.[^.]+\.[^.]+\.consul\.)", "tag_name": "consul.target" }, { "regex": "^cluster\.(((?:[^.]+~)?(?:[^.]+\.)?[^.]+\.[^.]+\.[^.]+\.[^.]+\.[^.]+)\.consul\.)", "tag_name": "consul.full_target" }, { "tag_name": "local_cluster", "fixed_value": "api" }, { "tag_name": "consul.source.service", "fixed_value": "api" }, { "tag_name": "consul.source.namespace", "fixed_value": "default" }, { "tag_name": "consul.source.datacenter", "fixed_value": "dc1" } ], "use_all_default_tags": true }, "tracing": { "http": { "name": "envoy.tracers.zipkin", "typedConfig": { "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig", "collector_cluster": "jaeger_9411", "collector_endpoint_version": "HTTP_JSON", "collector_endpoint": "/api/v1/spans", "shared_span_context": false } } }, "dynamic_resources": { "lds_config": { "ads": {}, "resource_api_version": "V3" }, "cds_config": { "ads": {}, "resource_api_version": "V3" }, "ads_config": { "api_type": "DELTA_GRPC", "transport_api_version": "V3", "grpc_services": { "initial_metadata": [ { "key": "x-consul-token", "value": "" } ], "envoy_grpc": { "cluster_name": "local_agent" } } } } }
service.hcl
services { id = "api-deployment-v1-7f9869c476-hr7dl-api" name = "api" address = "10.222.0.80" port = 9090 tags = ["v1"] meta = { version = "1" pod-name = "api-deployment-v1-7f9869c476-hr7dl" k8s-namespace = "jaeger" } }
services { id = "api-deployment-v1-7f9869c476-hr7dl-api-sidecar-proxy" name = "api-sidecar-proxy" kind = "connect-proxy" address = "10.222.0.80" port = 20000 tags = ["v1"] meta = { version = "1" pod-name = "api-deployment-v1-7f9869c476-hr7dl" k8s-namespace = "jaeger" }
proxy { destination_service_name = "api" destination_service_id = "api-deployment-v1-7f9869c476-hr7dl-api" local_service_address = "127.0.0.1" local_service_port = 9090 upstreams { destination_type = "service" destination_name = "payment" local_bind_port = 21001 } }
checks { name = "Proxy Public Listener" tcp = "10.222.0.80:20000" interval = "10s" deregister_critical_service_after = "10m" }
checks { name = "Destination Alias" alias_service = "api-deployment-v1-7f9869c476-hr7dl-api" } }
ndhanushkodi
Question I want to use envoy combined with Jaeger trace, using proxy-defaults.yaml,Jaeger tracer is configured in yaml file. No span data created by envoy was found in Jaeger. Can you help me look at this problem？Thanks !
Helm Configuration
configuration information of consul global: enabled: true name: consul datacenter: dc1 image: "hashicorp/consul:1.10.0" imageK8S: "hashicorp/consul-k8s:0.24.0" imageEnvoy: "envoyproxy/envoy-alpine:v1.17.4"
server: enabled: true
use 1 server
replicas: 1 storage: 20Gi storageClass: alicloud-disk-ssd ui: enabled: true
connectInject: enabled: true
inject an envoy sidecar into every new pod, except for those with annotations that prevent injection
default: true k8sAllowNamespaces: ["jaeger"] logLevel: debug client: enabled: true dnsPolicy: ClusterFirstWithHostNet hostNetwork: true
controller: enabled: true
configuration information of proxy-defaults.yaml
apiVersion: consul.hashicorp.com/v1alpha1 kind: ProxyDefaults metadata: name: global spec: config: envoy_extra_static_clusters_json: '{"connect_timeout":"3.000s","dns_lookup_family":"V4_ONLY","lb_policy":"ROUND_ROBIN","load_assignment":{"cluster_name":"jaeger_9411","endpoints":[{"lb_endpoints":[{"endpoint":{"address":{"socket_address":{"address":"jaeger-collector","port_value":9411,"protocol":"TCP"}}}}]}]},"name":"jaeger_9411","type":"STRICT_DNS"}' envoy_tracing_json: '{"http":{"name":"envoy.tracers.zipkin","typedConfig":{"@type":"type.googleapis.com/envoy.config.trace.v3.ZipkinConfig","collector_cluster":"jaeger_9411","collector_endpoint_version":"HTTP_JSON","collector_endpoint":"/api/v1/spans","shared_span_context":false}}}'
Jaeger svc:
$ kubectl get svc -n jaeger NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE consul-connect-injector-svc ClusterIP 10.221.143.35 443/TCP 133m
consul-controller-webhook ClusterIP 10.221.124.139 443/TCP 133m
consul-dns ClusterIP 10.221.217.208 53/TCP,53/UDP 133m
consul-server ClusterIP None 8500/TCP,8301/TCP,8301/UDP,8302/TCP,8302/UDP,8300/TCP,8600/TCP,8600/UDP 133m
consul-ui ClusterIP 10.221.105.51 80/TCP 133m
jaeger-collector ClusterIP 10.221.100.56 14250/TCP,14268/TCP,9411/TCP,14269/TCP 28d
jaeger-query ClusterIP 10.221.90.191 80/TCP,16687/TCP 28d
web ClusterIP 10.221.84.19 9090/TCP 127m
Logs
###web $ kubectl logs -f web-deployment-5f6ffdcd94-bw2kv-c envoy-sidecar -n jaeger [2022-01-17 09:48:10.549][1][info][config] [source/server/configuration_impl.cc:91] loading 2 cluster(s) [2022-01-17 09:48:10.616][1][info][config] [source/server/configuration_impl.cc:95] loading 0 listener(s) [2022-01-17 09:48:10.616][1][info][config] [source/server/configuration_impl.cc:107] loading stats configuration [2022-01-17 09:48:10.616][1][info][main] [source/server/server.cc:732] starting main dispatch loop [2022-01-17 09:48:10.617][1][info][runtime] [source/common/runtime/runtime_impl.cc:425] RTDS has finished initialization [2022-01-17 09:48:10.617][1][info][upstream] [source/common/upstream/cluster_manager_impl.cc:187] cm init: initializing cds [2022-01-17 09:48:10.617][1][warning][main] [source/server/server.cc:610] there is no configured limit to the number of allowed active connections. Set a limit via the runtime key overload.global_downstream_max_connections [2022-01-17 09:48:10.621][1][info][upstream] [source/common/upstream/cds_api_impl.cc:71] cds: add 2 cluster(s), remove 0 cluster(s) [2022-01-17 09:48:10.665][1][info][upstream] [source/common/upstream/cds_api_impl.cc:86] cds: add/update cluster 'api.default.dc1.internal.ada4c01e-5f59-0995-4f1e-17a73a07ace8.consul' [2022-01-17 09:48:10.693][1][info][upstream] [source/common/upstream/cds_api_impl.cc:86] cds: add/update cluster 'local_app' [2022-01-17 09:48:10.693][1][info][upstream] [source/common/upstream/cluster_manager_impl.cc:167] cm init: initializing secondary clusters [2022-01-17 09:48:10.694][1][info][upstream] [source/common/upstream/cluster_manager_impl.cc:191] cm init: all clusters initialized [2022-01-17 09:48:10.694][1][info][main] [source/server/server.cc:713] all clusters initialized. initializing init manager [2022-01-17 09:48:10.700][1][info][upstream] [source/server/lds_api.cc:79] lds: add/update listener 'public_listener:10.222.0.82:20000' [2022-01-17 09:48:10.700][1][info][upstream] [source/server/lds_api.cc:79] lds: add/update listener 'api:127.0.0.1:21001' [2022-01-17 09:48:10.700][1][info][config] [source/server/listener_manager_impl.cc:888] all dependencies initialized. starting workers [2022-01-17 10:03:10.709][1][info][main] [source/server/drain_manager_impl.cc:70] shutting down parent after drain
Api $ kubectl logs -f api-deployment-v1-7f9869c476-hr7dl -c envoy-sidecar -n jaeger
Current understanding and Expected behavior
I hope I can see the span information created by envoy in the front UI of Jaeger
Environment details
Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.0", GitCommit:"9e991415386e4cf155a24b1da15becaa390438d8", GitTreeState:"clean", BuildDate:"2020-03-25T14:58:59Z", GoVersion:"go1.13.8", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"20+", GitVersion:"v1.20.4-aliyun.1", GitCommit:"66492d8", GitTreeState:"", BuildDate:"2021-08-23T11:21:01Z", GoVersion:"go1.15.8", Compiler:"gc", Platform:"linux/amd64"}
Additional Context
https://www.youtube.com/watch?v=FO67TZHq5-M https://github.com/hashicorp/consul-service-mesh-for-developers