barkbay
commented
2 months ago Could you please provide the manifests you are using, this would help me reproduce. Thanks!
Open up2neck opened 2 months ago
barkbay
commented
2 months ago Could you please provide the manifests you are using, this would help me reproduce. Thanks!
up2neck
commented
2 months ago Could you please provide the manifests you are using, this would help me reproduce. Thanks!
It's slightly sanitized from sensitive data: affinity, specific labels, but cross-resource refers are persisted "as-is"
apiVersion: apm.k8s.elastic.co/v1
kind: ApmServer
metadata:
labels:
module/name: apm-server
package/name: intake
package/overlay: base
package/version: "2"
name: apm-server-v2
namespace: apm-sandbox
spec:
config:
apm-server:
auth:
anonymous:
allow_agent:
- rum-js
- rum-js-dpeo
- js-base
- java
- dotnet
- php
- opentelemetry/cpp
- python
- otlp
- go
- opentelemetry
- opentelemetry/webjs
- opentelemetry/js
- opentelemetry/go
- opentelemetry/java
- opentelemetry/nodejs
- opentelemetry/dotnet
- nodejs
- '@microlabs/otel-workers-sdk/js'
enabled: true
rate_limit:
event_limit: 8000
ip_limit: 1000
api_key:
enabled: false
limit: 100
capture_personal_data: true
default_service_environment: undefined
expvar.enabled: false
host: 0.0.0.0:8200
idle_timeout: 45s
max_connections: 0
max_event_size: 307200
max_header_size: 1048576
pprof.enabled: false
read_timeout: 30s
rum:
allow_headers:
- x-requested-with
- access-control-request-private-network
- access-control-allow-origin
- xmlhttprequest
- request-origin
allow_origins:
- '*'
enabled: true
exclude_from_grouping: ^/webpack
library_pattern: node_modules|bower_components|~
shutdown_timeout: 30s
ssl:
supported_protocols:
- TLSv1.2
- TLSv1.3
write_timeout: 30s
logging.level: warning
monitoring.elasticsearch: {}
count: 2
elasticsearchRef:
name: elasticsearch-v1
http:
service:
metadata:
labels:
module/name: apm-server
package/name: intake
package/version: "2"
spec:
ports:
- appProtocol: HTTPS
name: https
port: 8200
protocol: TCP
targetPort: 8200
tls:
certificate: {}
selfSignedCertificate:
subjectAltNames:
- dns: apm-server
kibanaRef:
name: kibana-v1
podTemplate:
metadata:
creationTimestamp: null
labels:
module/name: apm-server
package/name: intake
package/version: "2"
spec:
containers:
- env:
- name: ELASTIC_APM_GLOBAL_LABELS
value: project=dummy
- name: ELASTIC_APM_CAPTURE_BODY
value: all
- name: ELASTICSEARCH_HOST
value: https://elasticsearch:9200
name: apm-server
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 1
memory: 1Gi
topologySpreadConstraints:
- labelSelector:
matchLabels:
module/name: apm-server
package/name: intake
package/version: "2"
maxSkew: 1
nodeAffinityPolicy: Honor
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
version: 8.14.3
---
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
name: kibana-v1
namespace: apm-sandbox
spec:
config:
elasticsearch.requestHeadersWhitelist:
- authorization
elasticsearch.requestTimeout: 60000
elasticsearch.shardTimeout: 60000
server:
customResponseHeaders:
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-XSS-Protection: 1; mode=block
telemetry.optIn: false
xpack.fleet.agentPolicies:
- id: eck-fleet-server
monitoring_enabled:
- logs
- metrics
name: Fleet Server on ECK policy
namespace: default
package_policies:
- id: fleet_server-1
name: fleet_server-1
package:
name: fleet_server
unenroll_timeout: 900
xpack.fleet.agents.fleet_server.hosts:
- https://fleet:8220
xpack.fleet.outputs:
- config:
ssl.verification_mode: none
hosts:
- https://elasticsearch:9200
id: fleet-default-output
is_default: "true"
is_default_monitoring: "true"
name: default
type: elasticsearch
xpack.fleet.packages:
- name: system
version: latest
- name: elastic_agent
version: latest
- name: fleet_server
version: latest
- name: apm
version: latest
- name: kubernetes
version: latest
- name: cloudflare
version: latest
- name: synthetics
version: latest
- name: cloudflare_logpush
version: latest
- name: gcp_pubsub
version: latest
xpack.reporting.roles.enabled: false
xpack.spaces.maxSpaces: 1000
xpack.task_manager.max_workers: 100
xpack.task_manager.monitored_stats_health_verbose_log.enabled: true
count: 3
elasticsearchRef:
name: elasticsearch-v1
enterpriseSearchRef: {}
http:
service:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
- name: https
port: 5601
protocol: TCP
targetPort: 5601
tls:
certificate: {}
selfSignedCertificate:
subjectAltNames:
- dns: kibana
monitoring:
logs: {}
metrics: {}
podTemplate:
metadata:
creationTimestamp: null
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
common.k8s.elastic.co/type: kibana
package/name: elastic-stack-eck
package/type: component
package/version: "1"
topologyKey: kubernetes.io/hostname
containers:
- name: kibana
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
version: 8.14.3
---
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
name: elasticsearch-v1
spec:
auth: {}
http:
service:
metadata: {}
spec: {}
tls:
certificate: {}
selfSignedCertificate:
subjectAltNames:
- dns: elasticsearch
monitoring:
logs: {}
metrics: {}
nodeSets:
- config:
node.roles:
- master
- remote_cluster_client
xpack.security.authc:
anonymous:
roles: monitoring_user
username: anon
count: 3
name: master
podTemplate:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
containers:
- name: elasticsearch
readinessProbe:
httpGet:
port: 9200
scheme: HTTPS
resources:
limits:
cpu: 2
memory: 10Gi
requests:
cpu: 1
memory: 10Gi
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: standard-rwo
- config:
node.roles:
- remote_cluster_client
- data_content
- data_hot
- ingest
- transform
xpack.security.authc:
anonymous:
roles: monitoring_user
username: anon
count: 4
name: data
podTemplate:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
containers:
- name: elasticsearch
readinessProbe:
httpGet:
port: 9200
scheme: HTTPS
resources:
limits:
cpu: 7
memory: 54Gi
requests:
cpu: 6
memory: 54Gi
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Ti
storageClassName: premium-rwo
- config:
node.roles:
- remote_cluster_client
- data_cold
- data_warm
xpack.security.authc:
anonymous:
roles: monitoring_user
username: anon
count: 3
name: data-cold
podTemplate:
metadata:
labels:
package/name: elastic-stack-eck
package/type: component
package/version: "1"
spec:
containers:
- name: elasticsearch
readinessProbe:
httpGet:
port: 9200
scheme: HTTPS
resources:
limits:
cpu: 4
memory: 16Gi
requests:
cpu: 2
memory: 16Gi
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3096Gi
storageClassName: standard-rwo
podDisruptionBudget:
metadata: {}
spec:
maxUnavailable: 1
selector:
matchLabels:
common.k8s.elastic.co/type: elasticsearch
package/name: elastic-stack-eck
package/type: component
package/version: "1"
transport:
service:
metadata: {}
spec: {}
tls:
certificate: {}
certificateAuthorities: {}
updateStrategy:
changeBudget:
maxUnavailable: 1
version: 8.14.3
When APM Server is deployed with ECK it has no sufficient permissions for Kibana API:
Transaction JSON
``` { "_index": ".ds-traces-apm-epm_paas-2024.08.24-000002", "_id": "x61jsZEB26iDE9PXHEpU", "_version": 1, "_score": 0, "_source": { "parent": { "id": "154a123cf3b4f8ce" }, "agent": { "name": "go", "version": "2.6.0" }, "process": { "args": [ "apm-server", "run", "-e", "-c", "config/config-secret/apm-server.yml" ], "pid": 1, "title": "apm-server" }, "destination": { "address": "kibana-v1-kb-http.apm-sandbox.svc", "port": 5601 }, "processor": { "event": "span" }, "url": { "original": "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm" }, "labels": { "project": "epm-paas" }, "cloud": { "availability_zone": "europe-west3-a", "instance": { "name": "gke-epm-iass-elastic-europe-w-generic-c4e5a328-nebn", "id": "3933813931648332798" }, "provider": "gcp", "project": { "id": "or2-ms-epm-iass-elastic-t1iylu" }, "region": "europe-west3" }, "observer": { "hostname": "apm-server-v2-apm-server-56c7746446-m7dzp", "type": "apm-server", "version": "8.14.3" }, "trace": { "id": "154a123cf3b4f8ce0fb856d2d80a0416" }, "@timestamp": "2024-09-02T06:19:00.318Z", "data_stream": { "namespace": "epm_paas", "type": "traces", "dataset": "apm" }, "service": { "node": { "name": "apm-server-v2-apm-server-56c7746446-m7dzp" }, "environment": "sandbox-latest", "name": "apm-server", "runtime": { "name": "gc", "version": "go1.22.5" }, "language": { "name": "go", "version": "go1.22.5" }, "version": "8.14.3", "target": { "name": "kibana-v1-kb-http.apm-sandbox.svc:5601", "type": "http" } }, "host": { "hostname": "apm-server-v2-apm-server-56c7746446-m7dzp", "os": { "platform": "linux" }, "name": "apm-server-v2-apm-server-56c7746446-m7dzp", "architecture": "amd64" }, "http": { "response": { "status_code": 403 } }, "event": { "agent_id_status": "missing", "ingested": "2024-09-02T06:19:08Z", "success_count": 0, "outcome": "failure" }, "transaction": { "id": "154a123cf3b4f8ce" }, "span": { "duration": { "us": 102475 }, "representative_count": 1, "stacktrace": [ { "exclude_from_grouping": false, "library_frame": true, "filename": "span.go", "line": { "number": 442 }, "function": "(*Span).End", "module": "go.elastic.co/apm/v2" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "client.go", "line": { "number": 198 }, "function": "(*responseBody).endSpan", "module": "go.elastic.co/apm/module/apmhttp/v2" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "client.go", "line": { "number": 187 }, "function": "(*responseBody).Read", "module": "go.elastic.co/apm/module/apmhttp/v2" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "client.go", "line": { "number": 963 }, "function": "(*cancelTimerBody).Read", "module": "net/http" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "io.go", "line": { "number": 712 }, "function": "ReadAll", "module": "io" }, { "exclude_from_grouping": false, "filename": "checkintegration.go", "line": { "number": 94 }, "function": "checkIntegrationInstalledKibana", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "checkintegration.go", "line": { "number": 57 }, "function": "checkIntegrationInstalled", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 629 }, "function": "(*Runner).waitReady.func3", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 638 }, "function": "(*Runner).waitReady.func4", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "waitready.go", "line": { "number": 59 }, "function": "waitReady", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 644 }, "function": "(*Runner).waitReady", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 331 }, "function": "(*Runner).Run.func4", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "errgroup.go", "line": { "number": 78 }, "function": "(*Group).Go.func1", "module": "golang.org/x/sync/errgroup" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "asm_amd64.s", "line": { "number": 1695 }, "function": "goexit", "module": "runtime" } ], "subtype": "http", "destination": { "service": { "resource": "kibana-v1-kb-http.apm-sandbox.svc:5601", "name": "https://kibana-v1-kb-http.apm-sandbox.svc:5601", "type": "external" } }, "name": "GET kibana-v1-kb-http.apm-sandbox.svc:5601", "id": "b9dd9b517374b4cf", "type": "external" }, "timestamp": { "us": 1725257940318438 } }, "fields": { "host.hostname": [ "apm-server-v2-apm-server-56c7746446-m7dzp" ], "url.original.text": [ "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm" ], "process.pid": [ 1 ], "service.language.name": [ "go" ], "cloud.availability_zone": [ "europe-west3-a" ], "process.title.text": [ "apm-server" ], "transaction.id": [ "154a123cf3b4f8ce" ], "processor.event": [ "span" ], "labels.project": [ "epm-paas" ], "agent.name": [ "go" ], "destination.address": [ "kibana-v1-kb-http.apm-sandbox.svc" ], "host.name": [ "apm-server-v2-apm-server-56c7746446-m7dzp" ], "event.agent_id_status": [ "missing" ], "http.response.status_code": [ 403 ], "event.outcome": [ "failure" ], "cloud.region": [ "europe-west3" ], "service.runtime.version": [ "go1.22.5" ], "span.id": [ "b9dd9b517374b4cf" ], "data_stream.type": [ "traces" ], "span.type": [ "external" ], "host.architecture": [ "amd64" ], "cloud.provider": [ "gcp" ], "timestamp.us": [ 1725257940318438 ], "observer.type": [ "apm-server" ], "observer.version": [ "8.14.3" ], "agent.version": [ "2.6.0" ], "parent.id": [ "154a123cf3b4f8ce" ], "span.destination.service.name": [ "https://kibana-v1-kb-http.apm-sandbox.svc:5601" ], "process.title": [ "apm-server" ], "span.representative_count": [ 1 ], "span.destination.service.type": [ "external" ], "span.name": [ "GET kibana-v1-kb-http.apm-sandbox.svc:5601" ], "destination.port": [ 5601 ], "service.node.name": [ "apm-server-v2-apm-server-56c7746446-m7dzp" ], "cloud.instance.id": [ "3933813931648332798" ], "trace.id": [ "154a123cf3b4f8ce0fb856d2d80a0416" ], "span.duration.us": [ 102475 ], "span.stacktrace": [ { "exclude_from_grouping": false, "library_frame": true, "filename": "span.go", "line": { "number": 442 }, "function": "(*Span).End", "module": "go.elastic.co/apm/v2" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "client.go", "line": { "number": 198 }, "function": "(*responseBody).endSpan", "module": "go.elastic.co/apm/module/apmhttp/v2" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "client.go", "line": { "number": 187 }, "function": "(*responseBody).Read", "module": "go.elastic.co/apm/module/apmhttp/v2" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "client.go", "line": { "number": 963 }, "function": "(*cancelTimerBody).Read", "module": "net/http" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "io.go", "line": { "number": 712 }, "function": "ReadAll", "module": "io" }, { "exclude_from_grouping": false, "filename": "checkintegration.go", "line": { "number": 94 }, "function": "checkIntegrationInstalledKibana", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "checkintegration.go", "line": { "number": 57 }, "function": "checkIntegrationInstalled", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 629 }, "function": "(*Runner).waitReady.func3", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 638 }, "function": "(*Runner).waitReady.func4", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "waitready.go", "line": { "number": 59 }, "function": "waitReady", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 644 }, "function": "(*Runner).waitReady", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "beater.go", "line": { "number": 331 }, "function": "(*Runner).Run.func4", "module": "github.com/elastic/apm-server/internal/beater" }, { "exclude_from_grouping": false, "filename": "errgroup.go", "line": { "number": 78 }, "function": "(*Group).Go.func1", "module": "golang.org/x/sync/errgroup" }, { "exclude_from_grouping": false, "library_frame": true, "filename": "asm_amd64.s", "line": { "number": 1695 }, "function": "goexit", "module": "runtime" } ], "event.success_count": [ 0 ], "service.target.type": [ "http" ], "service.environment": [ "sandbox-latest" ], "service.name": [ "apm-server" ], "data_stream.namespace": [ "epm_paas" ], "service.runtime.name": [ "gc" ], "process.args": [ "apm-server", "run", "-e", "-c", "config/config-secret/apm-server.yml" ], "span.subtype": [ "http" ], "service.target.name": [ "kibana-v1-kb-http.apm-sandbox.svc:5601" ], "observer.hostname": [ "apm-server-v2-apm-server-56c7746446-m7dzp" ], "event.ingested": [ "2024-09-02T06:19:08.000Z" ], "url.original": [ "https://kibana-v1-kb-http.apm-sandbox.svc:5601/api/fleet/epm/packages/apm" ], "@timestamp": [ "2024-09-02T06:19:00.318Z" ], "service.version": [ "8.14.3" ], "host.os.platform": [ "linux" ], "data_stream.dataset": [ "apm" ], "service.language.version": [ "go1.22.5" ], "span.destination.service.resource": [ "kibana-v1-kb-http.apm-sandbox.svc:5601" ], "cloud.instance.name": [ "gke-epm-iass-elastic-europe-w-generic-c4e5a328-nebn" ], "cloud.project.id": [ "or2-ms-epm-iass-elastic-t1iylu" ] } } ```