# Default values for sck-otel.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
nameOverride: ""
fullnameOverride: ""
# Splunk Data-to-Everything Platform configuration.
# Fields `endpoint` and `token` are required to collect and send logs to Splunk Cloud or Splunk Enterprise.
splunkPlatform:
# URL to a Splunk instance to send data to. ie) "http://X.X.X.X:8088/services/collector"
endpoint: "http://host.minikube.internal:8088/services/collector"
# Splunk HTTP Event Collector token.
token: ***REDACTED***
# Optional. Name of the Splunk index targeted.
index: "sck_otel"
# Optional. Name of the Splunk metrics index targeted. Required if sending metric data to Splunk platform.
metrics_index: "sck_otel_metrics"
# Optional. Default value for `source` field.
source: "kubernetes"
# Optional. Default value for `sourcetype` field. For container logs, it will be container name.
sourcetype:
# Maximum HTTP connections to use simultaneously when sending data. Defaults to 200.
max_connections: 200
# Whether to disable gzip compression over HTTP. Defaults to true.
disable_compression: true
# HTTP timeout when sending data. Defaults to 10s.
timeout: 10s
# Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to true.
insecure_skip_verify: true
# The PEM-format CA certificate for this client.
# NOTE: The content of the certificate itself should be used here, not the file path.
# The certificate will be stored as a secret in kubernetes.
clientCert:
# The private key for this client.
# NOTE: The content of the key itself should be used here, not the file path.
# The key will be stored as a secret in kubernetes.
clientKey:
# The PEM-format CA certificate file.
# NOTE: The content of the file itself should be used here, not the file path.
# The file will be stored as a secret in kubernetes.
caFile:
# Options to disable particular telemetry data types that will be sent to Splunk Platform.
logsEnabled: true
metricsEnabled: true
# Splunk Observability configuration.
# Fields `realm` and `accessToken` are required to collect and send telemetry data to Splunk Observability Cloud.
splunkObservability:
# Splunk Observability realm to send telemetry data to.
realm:
# Splunk Observability org access token.
accessToken:
# Optional. Splunk Observability ingest URL, default: "https://ingest.<realm>.signalfx.com".
ingestUrl:
# Optional. Splunk Observability API URL, default: "https://api.<realm>.signalfx.com".
apiUrl:
# Options to disable particular telemetry data types that will be sent to Splunk Observability.
logsEnabled: true
metricsEnabled: true
tracesEnabled: true
# If provided, `k8s.cluster.name` attribute will be set uniformly across all the telemetry.
clusterName:
# If provided, `deployment.environment` attribute will be set on traces and logs when it's missed.
environment:
# Metadata to be set on the telemetry data from Kubernetes objects.
# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sprocessor.
k8sMetadata:
# Boolean for enriching all telemetry with k8s metadata (labels, annotations, etc)
enabled: true
# This option is to fetch metadata from extra annotations.
annotations: []
labels:
- key: app
# Cloud provider, if any, the collector is running on. Leave empty for none/other.
# This option adds cloud specific metadata to the telemetry.
# https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor
# - "aws" (Amazon Web Services)
# - "gcp" (Google Cloud Platform)
# - "azure" (Microsoft Azure)
cloudProvider: ""
# Kubernetes distribution being run. Leave empty for none/other. Similar to cloudProvider.
# - "eks" (Amazon Elastic Kubernetes Service)
# - "gke" (Google Google Kubernetes Engine)
# - "aks" (Azure Kubernetes Service)
k8sDistro: ""
# List of key/value pairs to set additional metadata set uniformly across the collected telemetry.
# Can be used to define things such as cloud_account_id, cloud_account_region, etc.
# Should be set using https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/resourceprocessor.
customMetadata: {}
# cloud_account_id: "1234567890"
# Automatic detection of additional telemetry sources.
# For now it has effect on Splunk Observability only,
# but can it be used with Splunk Platform in future as well?
autodetect:
# Set to `true` if you want the agent to scrape
# prometheus metrics from pods that have prometheus-style annotations like
# "prometheus.io/scrape".
prometheus: false
# Set to `true` in Istio environment to enable logs and metrics correlation
# with traces pushed by Istio.
istio: false
# Configurations for container logs
containerLogs:
enabled: true
# Container runtime. One of `docker`, `cri-o`, or `containerd`
# Automatically discovered if not set.
containerRuntime: ""
# Path of container log files, default /var/log/pods/*/*/*.log
# The file format is /var/log/pods/<namespace_name>_<pod_name>_<pod_uid>/<container_name>/<run_id>.log
path: /var/log/pods/*/*/*.log
# Paths of logfiles to exclude. object type is array:
# i.e) to exclude `kube-system` namespace,
# excludePaths: [ /var/log/pods/kube-system_*/*/*.log ]
excludePaths: []
# - /var/log/pods/kube-system_*.log (to exclude `kube-system` namespace)
# Boolean for ingesting the agent's own log
excludeAgentLogs: true
# Extra operators for container logs. https://github.com/open-telemetry/opentelemetry-log-collection/blob/main/docs/operators/README.md#what-operators-are-available
extraOperators: []
# Multiline logs currently support only container name which is not unique within the cluster.
# Can we also support k8s object owner name (deployment, daemonset, statefulset) along with namespace?
multilineConfigs: []
# Example configuration for processing and handling multiline logs/ java stack trace from test container "buttercup-app" by specifying a regular expression (first_entry_regex) to parse first entry in a multiline log series.
# Sample java stack trace/ multiline log from container "buttercup-app"
# .........
# Exception in thread "main" java.lang.NumberFormatException: For input string: "3.1415"
# at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
# at java.lang.Integer.parseInt(Integer.java:580)
# at ExampleCli.parseNumericArgument(ExampleCli.java:47)
# at ExampleCli.parseCliOptions(ExampleCli.java:27)
# at ExampleCli.main(ExampleCli.java:11)
# .........
# Sample configuration to handle multiline java stack trace from buttercup-app container
# multilineConfigs:
# - containerName: buttercup-app
# first_entry_regex: ^.+Exception[^\n]++(\s+at.*)+
# Specify configuration for any files other than container log files
# ref: https://github.com/open-telemetry/opentelemetry-log-collection/blob/main/docs/operators/file_input.md
extraFileLogs: {}
# Sample configuration to collect Audit logs. Please note hostPath can vary depending on the audit-policy.yaml configuration.
# extraFileLogs:
# filelog/audit-log:
# include: [/var/log/kubernetes/apiserver/audit.log]
# start_at: beginning
# include_file_path: true
# include_file_name: false
# resource:
# service.name: /var/log/kubernetes/apiserver/audit.log
# host.name: 'EXPR(env("K8S_NODE_NAME"))'
# com.splunk.sourcetype: kube:apiserver-audit
# agent:
# extraVolumeMounts:
# - name: audit-log
# mountPath: /var/log/kubernetes/apiserver
# extraVolumes:
# - name: audit-log
# hostPath:
# path: /var/log/kubernetes/apiserver
# Persistent storage path for storing logs checkpoints
checkpointPath: "/var/lib/otel_pos"
# Configuration specific to OTel agent - a daemonset responsible for collecting telemetry data from each k8s node.
agent:
enabled: true
podSecurityContext:
runAsUser: 10001
runAsGroup: 10001
securityContext:
runAsUser: 0
# OTel agent extra daemonset annotations.
annotations: {}
# OTel agent extra pods annotations.
podAnnotations: {}
# OTel agent extra daemonset labels.
labels: {}
# OTel agent extra pod labels.
podLabels: {}
# Pods scheduling configurations
nodeSelector: {}
affinity: {}
# This allows the daemonset to be deployed on master nodes,
# so that we can also collect logs and metrics from those nodes.
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
# Extra environment variables to be set in the agent container.
extraEnvs: []
# Extra volumes to be mounted to the agent container.
# Can be used to mount any host volumes, secrets, csi drivers, etc.
# It's more flexible than having extraHostPathMounts, secretMounts and others.
extraVolumes: []
extraVolumeMounts: []
# Need to be adjusted based on log EPS of a node.
resources:
limits:
cpu: 256m
memory: 512Mi
# Ports exposed on the agent as host ports
# No ports exposed for splunkPlatform, only enabled if splunkObservability enabled/
# Can be disabled with `enabled` option
ports:
otlp:
containerPort: 4317
hostPort: 4317
protocol: TCP
enabled: true
sfx-forwarder:
containerPort: 9080
hostPort: 9080
protocol: TCP
enabled: true
zipkin:
containerPort: 9411
hostPort: 9411
protocol: TCP
enabled: true
jaeger-thrift:
containerPort: 14268
hostPort: 14268
protocol: TCP
enabled: true
jaeger-grpc:
containerPort: 14250
hostPort: 14250
protocol: TCP
enabled: true
signalfx:
containerPort: 9943
hostPort: 9943
protocol: TCP
enabled: true
# Configuration override that will be merged into the agent's default config
# Example - this will replace `splunk_hec` with `logging` exporter for container logs
# config:
# exporters:
# logging/container:
# loglevel: info
# service:
# pipelines:
# logs/container:
# exporters:
# - logging/container
config: {}
image:
repository: quay.io/signalfx/splunk-otel-collector
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
# OpenTelemetry Collector executable
command:
name: otelcol
extraArgs: []
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
# Create or use existing secret if name is empty default name is used
secret:
create: true
name:
# Configuration for additional OTel collector deployment fetching data
# from the Kubernetes API and sending it to Observability Cloud as metrics and
# to Splunk Platform as Kubernetes objects (to be added later).
clusterDataCollector:
enabled: true
# Need to be adjusted based on size of monitored cluster
resources:
limits:
cpu: 256m
memory: 512Mi
# Pods scheduling configurations
nodeSelector: {}
tolerations: []
affinity: {}
podSecurityContext: {}
# OTel collector deployment annotations.
annotations: {}
# OTel collector pod annotations.
podAnnotations: {}
# k8s cluster receiver extra pod labels
podLabels: {}
# Extra environment variables to be set in the OTel Cluster Receiver container
extraEnvs: []
# Extra volumes to be mounted to the k8s cluster receiver container.
extraVolumes: []
extraVolumeMounts: []
# Configuration override that will be merged into the collector's default config
config: {}
# (To Be Implemented)
# OpenTelemetry "gateway" k8s deployment configuration.
# This is an optional deployment that can be used to forward telemetry data
# from the agents to a Splunk backend (Observability/Cloud/Enterprise).
# There are several use-cases for that:
# 1. Deployment of the gateway on specific nodes that can exclusively talk to internet
# 2. Move k8s metadata enrichment from the agent to reduce overall load on
# the Kubernetes API.
# 3. Apply bigger batching and reduce number of calls to a Splunk backend.
# If enabled, the agent and the k8sDataCollector are automatically configured
# to send data through the gateway deployment.
# Usually it's recommended to enable the gateway in large kubernetes clusters.
gateway:
enabled: false
# Number of collector replicas
replicaCount: 3
# Default resource limits are high assuming that the gateway is enabled
# in a big cluster with 25+ nodes.
resources:
limits:
cpu: 4
memory: 8Gi
# The ports exposed by the collector container.
# Any port can be disabled using `enabled` config option.
ports:
otlp:
containerPort: 4317
protocol: TCP
enabled: true
jaeger-thrift:
containerPort: 14268
protocol: TCP
enabled: # true for splunkObservability and if tracesEnabled=true
jaeger-grpc:
containerPort: 14250
protocol: TCP
enabled: # true for splunkObservability and if tracesEnabled=true
zipkin:
containerPort: 9411
protocol: TCP
enabled: # true for splunkObservability and if tracesEnabled=true
signalfx:
containerPort: 9943
protocol: TCP
enabled: # true for splunkObservability
http-forwarder:
containerPort: 6060
protocol: TCP
enabled: # true for splunkObservability
# Pods scheduling configurations
nodeSelector: {}
tolerations: []
affinity: {}
podSecurityContext: {}
# OTel collector deployment annotations.
annotations: {}
# OTel collector pods annotations.
podAnnotations: {}
# OTel collector extra pod labels
podLabels: {}
# Extra environment variables to be set in the standalone OTel collector container
extraEnvs: []
# Extra volumes to be mounted to the OTel Collector container.
extraVolumes: []
extraVolumeMounts: []
# Configuration override that will be merged into the collector's default config
config: {}
# Kubernetes service config.
service:
# Service type
type: ClusterIP
# Service annotations
annotations: {}
Steps to reproduce
Set
metricsEnabled: true
and
metrics_index: "sck_otel_metrics"
and we see errors in both cluster receiver and daemonset agent
Other information
Error Log from cluster receiver pod
2021/09/28 18:00:23 main.go:189: Set config to /conf/relay.yaml
2021/09/28 18:00:23 main.go:272: Set ballast to 168 MiB
2021/09/28 18:00:23 main.go:286: Set memory limit to 460 MiB
2021-09-28T18:00:23.893Z info service/collector.go:303 Starting otelcol... {"Version": "v0.33.1", "NumCPU": 2}
2021-09-28T18:00:23.893Z info service/collector.go:242 Loading configuration...
2021-09-28T18:00:23.969Z info service/collector.go:258 Applying configuration...
2021-09-28T18:00:23.969Z info builder/exporters_builder.go:264 Exporter was built. {"kind": "exporter", "name": "splunk_hec/platformMetrics"}
2021-09-28T18:00:23.969Z info memorylimiter/memorylimiter.go:105 Memory limiter configured {"kind": "processor", "name": "memory_limiter", "limit_mib": 460, "spike_limit_mib": 92, "check_interval": 5}
2021-09-28T18:00:23.969Z info builder/pipelines_builder.go:214 Pipeline was built. {"pipeline_name": "metrics", "pipeline_datatype": "metrics"}
2021-09-28T18:00:23.969Z info memorylimiter/memorylimiter.go:105 Memory limiter configured {"kind": "processor", "name": "memory_limiter", "limit_mib": 460, "spike_limit_mib": 92, "check_interval": 5}
2021-09-28T18:00:23.969Z info builder/pipelines_builder.go:214 Pipeline was built. {"pipeline_name": "metrics/collector", "pipeline_datatype": "metrics"}
2021-09-28T18:00:23.969Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "prometheus/k8s_cluster_receiver", "datatype": "metrics"}
2021-09-28T18:00:23.970Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "k8s_cluster", "datatype": "metrics"}
2021-09-28T18:00:23.970Z info service/service.go:143 Starting extensions...
2021-09-28T18:00:23.971Z info builder/extensions_builder.go:54 Extension is starting... {"kind": "extension", "name": "health_check"}
2021-09-28T18:00:23.971Z info healthcheckextension/healthcheckextension.go:41 Starting health_check extension {"kind": "extension", "name": "health_check", "config": {"Port":0,"TCPAddr":{"Endpoint":"0.0.0.0:13133"}}}
2021-09-28T18:00:23.971Z info builder/extensions_builder.go:60 Extension started. {"kind": "extension", "name": "health_check"}
2021-09-28T18:00:23.971Z info service/service.go:188 Starting exporters...
2021-09-28T18:00:23.971Z info builder/exporters_builder.go:93 Exporter is starting... {"kind": "exporter", "name": "splunk_hec/platformMetrics"}
2021-09-28T18:00:23.971Z info builder/exporters_builder.go:98 Exporter started. {"kind": "exporter", "name": "splunk_hec/platformMetrics"}
2021-09-28T18:00:23.971Z info service/service.go:193 Starting processors...
2021-09-28T18:00:23.971Z info builder/pipelines_builder.go:52 Pipeline is starting... {"pipeline_name": "metrics", "pipeline_datatype": "metrics"}
2021-09-28T18:00:23.971Z info builder/pipelines_builder.go:63 Pipeline is started. {"pipeline_name": "metrics", "pipeline_datatype": "metrics"}
2021-09-28T18:00:23.971Z info builder/pipelines_builder.go:52 Pipeline is starting... {"pipeline_name": "metrics/collector", "pipeline_datatype": "metrics"}
2021-09-28T18:00:23.971Z info internal/resourcedetection.go:126 began detecting resource information {"kind": "processor", "name": "resourcedetection"}
2021-09-28T18:00:23.971Z info internal/resourcedetection.go:139 detected resource information {"kind": "processor", "name": "resourcedetection", "resource": {"host.name":"local-k8s-sck-otel-k8s-cluster-receiver-5f8bcf865d-whwrr","os.type":"LINUX"}}
2021-09-28T18:00:23.971Z info builder/pipelines_builder.go:63 Pipeline is started. {"pipeline_name": "metrics/collector", "pipeline_datatype": "metrics"}
2021-09-28T18:00:23.971Z info service/service.go:198 Starting receivers...
2021-09-28T18:00:23.971Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "prometheus/k8s_cluster_receiver"}
2021-09-28T18:00:23.971Z info builder/receivers_builder.go:76 Receiver started. {"kind": "receiver", "name": "prometheus/k8s_cluster_receiver"}
2021-09-28T18:00:23.971Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "k8s_cluster"}
Error: cannot setup pipelines: cannot start receivers: platformMetrics exporter does not implement MetadataExporter
2021/09/28 18:00:23 main.go:95: application run finished with error: cannot setup pipelines: cannot start receivers: platformMetrics exporter does not implement MetadataExporter
Error log from agent daemonset pod
2021/09/28 18:00:01 main.go:189: Set config to /conf/relay.yaml
2021/09/28 18:00:01 main.go:272: Set ballast to 168 MiB
2021/09/28 18:00:01 main.go:286: Set memory limit to 460 MiB
2021-09-28T18:00:01.685Z info service/collector.go:303 Starting otelcol... {"Version": "v0.33.1", "NumCPU": 2}
2021-09-28T18:00:01.685Z info service/collector.go:242 Loading configuration...
2021-09-28T18:00:01.871Z info service/collector.go:258 Applying configuration...
2021-09-28T18:00:01.873Z info builder/exporters_builder.go:264 Exporter was built. {"kind": "exporter", "name": "splunk_hec/platformMetrics"}
2021-09-28T18:00:01.873Z info builder/exporters_builder.go:264 Exporter was built. {"kind": "exporter", "name": "splunk_hec/platform"}
2021-09-28T18:00:01.875Z info kube/client.go:95 k8s filtering {"kind": "processor", "name": "k8s_tagger", "labelSelector": "", "fieldSelector": "spec.nodeName=minikube"}
2021-09-28T18:00:01.876Z info memorylimiter/memorylimiter.go:105 Memory limiter configured {"kind": "processor", "name": "memory_limiter", "limit_mib": 460, "spike_limit_mib": 92, "check_interval": 5}
2021-09-28T18:00:01.876Z info builder/pipelines_builder.go:214 Pipeline was built. {"pipeline_name": "logs/container", "pipeline_datatype": "logs"}
2021-09-28T18:00:01.876Z info memorylimiter/memorylimiter.go:105 Memory limiter configured {"kind": "processor", "name": "memory_limiter", "limit_mib": 460, "spike_limit_mib": 92, "check_interval": 5}
2021-09-28T18:00:01.876Z info builder/pipelines_builder.go:214 Pipeline was built. {"pipeline_name": "metrics/agent", "pipeline_datatype": "metrics"}
2021-09-28T18:00:01.876Z info memorylimiter/memorylimiter.go:105 Memory limiter configured {"kind": "processor", "name": "memory_limiter", "limit_mib": 460, "spike_limit_mib": 92, "check_interval": 5}
2021-09-28T18:00:01.877Z info builder/pipelines_builder.go:214 Pipeline was built. {"pipeline_name": "metrics", "pipeline_datatype": "metrics"}
2021-09-28T18:00:01.877Z info builder/receivers_builder.go:110 Ignoring receiver as it is not used by any pipeline {"kind": "receiver", "name": "zipkin"}
2021-09-28T18:00:01.877Z info builder/receivers_builder.go:110 Ignoring receiver as it is not used by any pipeline {"kind": "receiver", "name": "jaeger"}
2021-09-28T18:00:01.877Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "receiver_creator", "datatype": "metrics"}
2021-09-28T18:00:01.877Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "signalfx", "datatype": "metrics"}
2021-09-28T18:00:01.877Z info builder/receivers_builder.go:110 Ignoring receiver as it is not used by any pipeline {"kind": "receiver", "name": "otlp"}
2021-09-28T18:00:01.877Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "prometheus/agent", "datatype": "metrics"}
2021-09-28T18:00:01.967Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "kubeletstats", "datatype": "metrics"}
2021-09-28T18:00:01.967Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "hostmetrics", "datatype": "metrics"}
2021-09-28T18:00:01.969Z info builder/receivers_builder.go:227 Receiver was built. {"kind": "receiver", "name": "filelog", "datatype": "logs"}
2021-09-28T18:00:01.969Z info service/service.go:143 Starting extensions...
2021-09-28T18:00:01.969Z info builder/extensions_builder.go:54 Extension is starting... {"kind": "extension", "name": "file_storage"}
2021-09-28T18:00:01.969Z info builder/extensions_builder.go:60 Extension started. {"kind": "extension", "name": "file_storage"}
2021-09-28T18:00:01.969Z info builder/extensions_builder.go:54 Extension is starting... {"kind": "extension", "name": "k8s_observer"}
2021-09-28T18:00:01.969Z info builder/extensions_builder.go:60 Extension started. {"kind": "extension", "name": "k8s_observer"}
2021-09-28T18:00:01.969Z info builder/extensions_builder.go:54 Extension is starting... {"kind": "extension", "name": "zpages"}
2021-09-28T18:00:01.969Z info zpagesextension/zpagesextension.go:40 Register Host's zPages {"kind": "extension", "name": "zpages"}
2021-09-28T18:00:01.970Z info zpagesextension/zpagesextension.go:53 Starting zPages extension {"kind": "extension", "name": "zpages", "config": {"TCPAddr":{"Endpoint":"localhost:55679"}}}
2021-09-28T18:00:01.970Z info builder/extensions_builder.go:60 Extension started. {"kind": "extension", "name": "zpages"}
2021-09-28T18:00:01.970Z info builder/extensions_builder.go:54 Extension is starting... {"kind": "extension", "name": "health_check"}
2021-09-28T18:00:01.970Z info healthcheckextension/healthcheckextension.go:41 Starting health_check extension {"kind": "extension", "name": "health_check", "config": {"Port":0,"TCPAddr":{"Endpoint":"0.0.0.0:13133"}}}
2021-09-28T18:00:01.970Z info builder/extensions_builder.go:60 Extension started. {"kind": "extension", "name": "health_check"}
2021-09-28T18:00:01.970Z info service/service.go:188 Starting exporters...
2021-09-28T18:00:01.970Z info builder/exporters_builder.go:93 Exporter is starting... {"kind": "exporter", "name": "splunk_hec/platformMetrics"}
2021-09-28T18:00:01.970Z info builder/exporters_builder.go:98 Exporter started. {"kind": "exporter", "name": "splunk_hec/platformMetrics"}
2021-09-28T18:00:01.970Z info builder/exporters_builder.go:93 Exporter is starting... {"kind": "exporter", "name": "splunk_hec/platform"}
2021-09-28T18:00:01.971Z info builder/exporters_builder.go:98 Exporter started. {"kind": "exporter", "name": "splunk_hec/platform"}
2021-09-28T18:00:01.971Z info service/service.go:193 Starting processors...
2021-09-28T18:00:01.971Z info builder/pipelines_builder.go:52 Pipeline is starting... {"pipeline_name": "logs/container", "pipeline_datatype": "logs"}
2021-09-28T18:00:01.971Z info internal/resourcedetection.go:126 began detecting resource information {"kind": "processor", "name": "resourcedetection"}
2021-09-28T18:00:01.971Z info internal/resourcedetection.go:139 detected resource information {"kind": "processor", "name": "resourcedetection", "resource": {"host.name":"local-k8s-sck-otel-l5lxq","os.type":"LINUX"}}
2021-09-28T18:00:01.971Z info builder/pipelines_builder.go:63 Pipeline is started. {"pipeline_name": "logs/container", "pipeline_datatype": "logs"}
2021-09-28T18:00:01.971Z info builder/pipelines_builder.go:52 Pipeline is starting... {"pipeline_name": "metrics/agent", "pipeline_datatype": "metrics"}
2021-09-28T18:00:01.971Z info builder/pipelines_builder.go:63 Pipeline is started. {"pipeline_name": "metrics/agent", "pipeline_datatype": "metrics"}
2021-09-28T18:00:01.971Z info builder/pipelines_builder.go:52 Pipeline is starting... {"pipeline_name": "metrics", "pipeline_datatype": "metrics"}
2021-09-28T18:00:01.971Z info builder/pipelines_builder.go:63 Pipeline is started. {"pipeline_name": "metrics", "pipeline_datatype": "metrics"}
2021-09-28T18:00:01.971Z info service/service.go:198 Starting receivers...
2021-09-28T18:00:01.971Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "receiver_creator"}
2021-09-28T18:00:01.971Z info builder/receivers_builder.go:76 Receiver started. {"kind": "receiver", "name": "receiver_creator"}
2021-09-28T18:00:01.971Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "signalfx"}
2021-09-28T18:00:01.971Z info builder/receivers_builder.go:76 Receiver started. {"kind": "receiver", "name": "signalfx"}
2021-09-28T18:00:01.971Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "prometheus/agent"}
2021-09-28T18:00:01.972Z info builder/receivers_builder.go:76 Receiver started. {"kind": "receiver", "name": "prometheus/agent"}
2021-09-28T18:00:01.972Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "kubeletstats"}
2021-09-28T18:00:01.972Z info builder/receivers_builder.go:76 Receiver started. {"kind": "receiver", "name": "kubeletstats"}
2021-09-28T18:00:01.972Z info builder/receivers_builder.go:71 Receiver is starting... {"kind": "receiver", "name": "hostmetrics"}
Error: cannot setup pipelines: cannot start receivers: open /hostfs/proc/stat: no such file or directory
2021/09/28 18:00:01 main.go:95: application run finished with error: cannot setup pipelines: cannot start receivers: open /hostfs/proc/stat: no such file or directory
rockb1017
commented
2 years ago
Describe the bug Enabling metrics for Splunk Platform dosen't work
Environment Chart Version: latest develop Cloud Provider: minikube k8s 1.20
Values.yaml:Steps to reproduce Set metricsEnabled: true and metrics_index: "sck_otel_metrics" and we see errors in both cluster receiver and daemonset agent
Other information Error Log from cluster receiver pod
Error log from agent daemonset pod