Missing metrics CPU and memory metrics

[benzch]

Hi,

I am missing the metrics: container_memory_working_set_bytes and container_cpu_usage_seconds_total. Tried to follow https://github.com/signalfx/splunk-otel-collector-chart/discussions/700 to add them from the exlucde_metric list with no luck. Attaching agent configmap

apiVersion: v1
data:
  relay: |
    exporters:
      otlp:
        endpoint: splunk-otel-collector:4317
        tls:
          insecure: true
      signalfx:
        access_token: ${SPLUNK_OBSERVABILITY_ACCESS_TOKEN}
        api_url: http://splunk-otel-collector:6060
        correlation: null
        include_metrics:
        - metric_names:
          - /^.*$/
        ingest_url: http://splunk-otel-collector:9943
        sync_host_metadata: true
    extensions:
      health_check: null
      k8s_observer:
        auth_type: serviceAccount
        node: ${K8S_NODE_NAME}
      memory_ballast:
        size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
      zpages: null
    processors:
      attributes/istio:
        actions:
        - action: delete
          key: source_cluster
        - action: delete
          key: destination_cluster
        - action: delete
          key: source_canonical_service
        - action: delete
          key: destination_canonical_service
        - action: delete
          key: source_canonical_revision
        - action: delete
          key: destination_canonical_revision
        include:
          match_type: regexp
          metric_names:
          - istio_.*
      batch: null
      groupbyattrs/logs:
        keys:
        - com.splunk.source
        - com.splunk.sourcetype
        - container.id
        - fluent.tag
        - istio_service_name
        - k8s.container.name
        - k8s.namespace.name
        - k8s.pod.name
        - k8s.pod.uid
      k8sattributes:
        extract:
          annotations:
          - from: pod
            key: splunk.com/sourcetype
          - from: namespace
            key: splunk.com/exclude
            tag_name: splunk.com/exclude
          - from: pod
            key: splunk.com/exclude
            tag_name: splunk.com/exclude
          - from: namespace
            key: splunk.com/index
            tag_name: com.splunk.index
          - from: pod
            key: splunk.com/index
            tag_name: com.splunk.index
          labels:
          - key: app
          metadata:
          - k8s.namespace.name
          - k8s.node.name
          - k8s.pod.name
          - k8s.pod.uid
          - container.id
          - container.image.name
          - container.image.tag
        filter:
          node_from_env_var: K8S_NODE_NAME
        passthrough: true
        pod_association:
        - sources:
          - from: resource_attribute
            name: k8s.pod.uid
        - sources:
          - from: resource_attribute
            name: k8s.pod.ip
        - sources:
          - from: resource_attribute
            name: ip
        - sources:
          - from: connection
        - sources:
          - from: resource_attribute
            name: host.name
      memory_limiter:
        check_interval: 2s
        limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
      resource:
        attributes:
        - action: insert
          key: k8s.node.name
          value: ${K8S_NODE_NAME}
        - action: upsert
          key: k8s.cluster.name
          value: CASB-eks01-dev
      resource/add_agent_k8s:
        attributes:
        - action: insert
          key: k8s.pod.name
          value: ${K8S_POD_NAME}
        - action: insert
          key: k8s.pod.uid
          value: ${K8S_POD_UID}
        - action: insert
          key: k8s.namespace.name
          value: ${K8S_NAMESPACE}
      resourcedetection:
        detectors:
        - env
        - eks
        - ec2
        - system
        override: true
        timeout: 10s
    receivers:
      hostmetrics:
        collection_interval: 10s
        scrapers:
          cpu: null
          disk: null
          filesystem: null
          load: null
          memory: null
          network: null
          paging: null
          processes: null
      jaeger:
        protocols:
          grpc:
            endpoint: 0.0.0.0:14250
          thrift_http:
            endpoint: 0.0.0.0:14268
      kubeletstats:
        auth_type: serviceAccount
        collection_interval: 10s
        endpoint: ${K8S_NODE_IP}:10250
        extra_metadata_labels:
        - container.id
        metric_groups:
        - container
        - pod
        - node
        - volume
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
      prometheus/agent:
        config:
          scrape_configs:
          - job_name: otel-agent
            scrape_interval: 10s
            static_configs:
            - targets:
              - ${K8S_POD_IP}:8889
      receiver_creator:
        receivers:
          prometheus_simple:
            config:
              endpoint: '`endpoint`:`"prometheus.io/port" in annotations ? annotations["prometheus.io/port"]
                : 9090`'
              metrics_path: '`"prometheus.io/path" in annotations ? annotations["prometheus.io/path"]
                : "/metrics"`'
            rule: type == "pod" && annotations["prometheus.io/scrape"] == "true"
        watch_observers:
        - k8s_observer
      signalfx:
        endpoint: 0.0.0.0:9943
      smartagent/signalfx-forwarder:
        listenAddress: 0.0.0.0:9080
        type: signalfx-forwarder
      zipkin:
        endpoint: 0.0.0.0:9411
    service:
      extensions:
      - health_check
      - k8s_observer
      - memory_ballast
      - zpages
      pipelines:
        metrics:
          exporters:
          - otlp
          processors:
          - memory_limiter
          - batch
          - attributes/istio
          - resourcedetection
          - resource
          receivers:
          - hostmetrics
          - kubeletstats
          - otlp
          - receiver_creator
          - signalfx
        metrics/agent:
          exporters:
          - signalfx
          processors:
          - memory_limiter
          - batch
          - resource/add_agent_k8s
          - resourcedetection
          - resource
          receivers:
          - prometheus/agent
        traces:
          exporters:
          - otlp
          - signalfx
          processors:
          - memory_limiter
          - k8sattributes
          - batch
          - resourcedetection
          - resource
          receivers:
          - otlp
          - jaeger
          - smartagent/signalfx-forwarder
          - zipkin
      telemetry:
        metrics:
          address: 0.0.0.0:8889

[benzch]

@jvoravong maybe you can help?, I saw you answered https://github.com/signalfx/splunk-otel-collector-chart/discussions/700

[atoulme]

@benzch please consider following up with a support case to receive timely responses. Here is a link to get started.