Unable to send any logs to splunk enterprise 8.1.9 using OTEL

[bala151187]

Describe the issue you're reporting

We are currently migrating from splunk connect for kubernetes to Splunk OpenTelemetry Collector for Kubernetes using helm and after installing OTEL we don't see any logs in splunk dashboard. below is the OTEL yaml . can you please let us know wat we are doing wrong

exporters:
  splunk_hec/platform_logs:
    disable_compression: true
    endpoint: <REDACTED>
    idle_conn_timeout: 10s
    index: <REDACTED>
    max_connections: 200
    profiling_data_enabled: false
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_elapsed_time: 300s
      max_interval: 30s
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 5000
    source: kubernetes
    splunk_app_name: splunk-otel-collector
    splunk_app_version: 0.81.0
    timeout: 10s
    tls:
      insecure_skip_verify: false
    token: ${SPLUNK_PLATFORM_HEC_TOKEN}
extensions:
  file_storage:
    directory: /var/addon/splunk/otel_pos
  health_check: null
  k8s_observer:
    auth_type: serviceAccount
    node: ${K8S_NODE_NAME}
  memory_ballast:
    size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
  zpages: null
processors:
  batch: null
  filter/logs:
    logs:
      exclude:
        match_type: strict
        resource_attributes:
        - key: splunk.com/exclude
          value: "true"
  k8sattributes:
    extract:
      annotations:
      - from: pod
        key: splunk.com/sourcetype
      - from: namespace
        key: splunk.com/exclude
        tag_name: splunk.com/exclude
      - from: pod
        key: splunk.com/exclude
        tag_name: splunk.com/exclude
      - from: namespace
        key: splunk.com/index
        tag_name: com.splunk.index
      - from: pod
        key: splunk.com/index
        tag_name: com.splunk.index
      labels:
      - key: app
      metadata:
      - k8s.namespace.name
      - k8s.node.name
      - k8s.pod.name
      - k8s.pod.uid
      - container.id
      - container.image.name
      - container.image.tag
    filter:
      node_from_env_var: K8S_NODE_NAME
    pod_association:
    - sources:
      - from: resource_attribute
        name: k8s.pod.uid
    - sources:
      - from: resource_attribute
        name: k8s.pod.ip
    - sources:
      - from: resource_attribute
        name: ip
    - sources:
      - from: connection
    - sources:
      - from: resource_attribute
        name: host.name
  memory_limiter:
    check_interval: 2s
    limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
  resource:
    attributes:
    - action: insert
      key: k8s.node.name
      value: ${K8S_NODE_NAME}
    - action: upsert
      key: k8s.cluster.name
      value: contain-eks-lab
  resource/add_agent_k8s:
    attributes:
    - action: insert
      key: k8s.pod.name
      value: ${K8S_POD_NAME}
    - action: insert
      key: k8s.pod.uid
      value: ${K8S_POD_UID}
    - action: insert
      key: k8s.namespace.name
      value: ${K8S_NAMESPACE}
  resource/logs:
    attributes:
    - action: upsert
      from_attribute: k8s.pod.annotations.splunk.com/sourcetype
      key: com.splunk.sourcetype
    - action: delete
      key: k8s.pod.annotations.splunk.com/sourcetype
    - action: delete
      key: splunk.com/exclude
  resourcedetection:
    detectors:
    - env
    - eks
    - system
    override: true
    timeout: 10s
receivers:
  filelog:
    encoding: utf-8
    exclude:
    - /var/log/pods/splunk_splunk-app-splunk-otel-collector*_*/otel-collector/*.log
    fingerprint_size: 1kb
    force_flush_period: "0"
    include:
    - /var/log/pods/*/*/*.log
    include_file_name: false
    include_file_path: true
    max_concurrent_files: 1024
    max_log_size: 1MiB
    operators:
    - id: parser-containerd
      regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
      timestamp:
        layout: '%Y-%m-%dT%H:%M:%S.%LZ'
        parse_from: attributes.time
      type: regex_parser
    - combine_field: attributes.log
      combine_with: ""
      id: containerd-recombine
      is_last_entry: attributes.logtag == 'F'
      max_log_size: 1048576
      output: handle_empty_log
      source_identifier: attributes["log.file.path"]
      type: recombine
    - field: attributes.log
      id: handle_empty_log
      if: attributes.log == nil
      type: add
      value: ""
    - parse_from: attributes["log.file.path"]
      regex: ^\/var\/log\/pods\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
      type: regex_parser
    - from: attributes.uid
      to: resource["k8s.pod.uid"]
      type: move
    - from: attributes.restart_count
      to: resource["k8s.container.restart_count"]
      type: move
    - from: attributes.container_name
      to: resource["k8s.container.name"]
      type: move
    - from: attributes.namespace
      to: resource["k8s.namespace.name"]
      type: move
    - from: attributes.pod_name
      to: resource["k8s.pod.name"]
      type: move
    - field: resource["com.splunk.sourcetype"]
      type: add
      value: EXPR("kube:container:"+resource["k8s.container.name"])
    - from: attributes.stream
      to: attributes["log.iostream"]
      type: move
    - from: attributes["log.file.path"]
      to: resource["com.splunk.source"]
      type: move
    - from: attributes.log
      id: clean-up-log-record
      to: body
      type: move
    poll_interval: 200ms
    retry_on_failure:
      enabled: true
    start_at: beginning
    storage: file_storage
  fluentforward:
    endpoint: 0.0.0.0:8006
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
  prometheus/agent:
    config:
      scrape_configs:
      - job_name: otel-agent
        scrape_interval: 10s
        static_configs:
        - targets:
          - ${K8S_POD_IP}:8889
service:
  extensions:
  - file_storage
  - health_check
  - k8s_observer
  - memory_ballast
  - zpages
  pipelines:
    logs:
      exporters:
      - splunk_hec/platform_logs
      processors:
      - memory_limiter
      - k8sattributes
      - filter/logs
      - batch
      - resourcedetection
      - resource
      - resource/logs
      receivers:
      - filelog
      - fluentforward
      - otlp
  telemetry:
    logs:
      level: debug
    metrics:
      address: 0.0.0.0:8889

[omrozowicz-splunk]

Hey, what are the agent's logs? Do you see any errors?

[atoulme]

Closing as inactive. @bala151187 please open a support case to receive further guidance. Thanks!