Open newmoon88 opened 1 year ago
Hi, I deployed my application on Azure Aks (it's a simple python hello world that logs to stdout). And I used splunk for connect with the following yaml file:
`global: logLevel: info splunk: hec: host: port: token: mytoken protocol: https endpoint: /endpoint/ fullUrl: https://fullurl indexName: kube-logs insecureSSL: clientCert: clientKey: caFile: indexRouting: consume_chunk_on_4xx_errors: kubernetes: clusterName: "cluster_name" prometheus_enabled: monitoring_agent_enabled: monitoring_agent_index_name: monitoring_agent_bind_address: metrics: service: enabled: true headless: true serviceMonitor: enabled: false
metricsPort: 24231 interval: "" scrapeTimeout: "10s" additionalLabels: { }
splunk-kubernetes-logging: enabled: true logLevel:
namespace:
fluentd: path: /var/log/containers/*.log exclude_path:
containers: path: /var/log pathDest: /var/lib/docker/containers logFormatType: cri logFormat: refreshInterval: removeBlankEvents: true localTime: false enableStatWatcher: true
k8sMetadata: podLabels:
release watch: true cache_ttl: 3600 propagate_namespace_labels: false
sourcetypePrefix: "kube"
rbac: create: true openshiftPrivilegedSccBinding: false
serviceAccount: create: true name:
podSecurityPolicy: create: false apparmor_security: true apiGroup: policy
splunk: hec: host: port: token: protocol: endpoint: fullUrl: indexName: insecureSSL: clientCert: clientKey: caFile: consume_chunk_on_4xx_errors: gzip_compression: ingest_api: serviceClientIdentifier: serviceClientSecretKey: tokenEndpoint: ingestAuthHost: ingestAPIHost: tenant: eventsEndpoint: debugIngestAPI:
secret: create: true name:
journalLogPath: /run/log/journal
charEncodingUtf8: false
customLogs: default: from: pod: helloworld5-5dc446c649-zhvq5 multiline: firstline: /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+\s+\w+\s+[/ flushInterval: 30 logs: docker: from: journald: unit: docker.service sourcetype: kube:docker kubelet: &glog from: journald: unit: kubelet.service multiline: firstline: /^\w[0-1]\d[0-3]\d/ sourcetype: kube:kubelet default: from: pod: helloworld5-5dc446c649-zhvq5 etcd: from: pod: etcd-server container: etcd-container etcd-minikube: from: pod: etcd-minikube container: etcd etcd-events: from: pod: etcd-server-events container: etcd-container kube-apiserver: <<: glog from: pod: kube-apiserver sourcetype: kube:kube-apiserver kube-scheduler: <<: glog from: pod: kube-scheduler sourcetype: kube:kube-scheduler kube-controller-manager: <<: glog from: pod: kube-controller-manager sourcetype: kube:kube-controller-manager kube-proxy: <<: glog from: pod: kube-proxy sourcetype: kube:kube-proxy kubedns: <<: glog from: pod: kube-dns sourcetype: kube:kubedns dnsmasq: <<: glog from: pod: kube-dns sourcetype: kube:dnsmasq dns-sidecar: <<: glog from: pod: kube-dns container: sidecar sourcetype: kube:kubedns-sidecar dns-controller: <<: glog from: pod: dns-controller sourcetype: kube:dns-controller kube-dns-autoscaler: <<: *glog from: pod: kube-dns-autoscaler container: autoscaler sourcetype: kube:kube-dns-autoscaler kube-audit: from: file: path: /var/log/kube-apiserver-audit.log timestampExtraction: format: "%Y-%m-%dT%H:%M:%SZ" sourcetype: kube:apiserver-audit
image: registry: docker.io name: splunk/fluentd-hec tag: 1.3.2 pullPolicy: IfNotPresent usePullSecret: false pullSecretName:
environmentVar:
podAnnotations:
extraLabels:
resources: requests: cpu: 100m memory: 200Mi
bufferChunkKeys:
sendAllMetadata: false
tolerations:
nodeSelector: kubernetes.io/os: linux
affinity: {}
extraVolumes: [] extraVolumeMounts: []
priorityClassName:
kubernetes: clusterName: securityContext: false
customMetadata:
customMetadataAnnotations:
customFilters: {}
indexFields: []
rollingUpdate:
splunk-kubernetes-objects: enabled: true logLevel:
rbac: create: true
serviceAccount: create: true name: usePullSecrets: false
kubernetes: url: insecureSSL: false clientCert: clientKey: caFile: bearerTokenFile: secretDir: clusterName:
objects: core: v1:
name: events mode: watch
checkpointFile: name: kubernetes-objects.pos volume:
splunk: hec: host: token: protocol: port: fullUrl: indexName: insecureSSL: clientCert: clientKey: caFile: caPath: indexRouting: consume_chunk_on_4xx_errors:
image: registry: docker.io name: splunk/kube-objects tag: 1.2.2 pullPolicy: IfNotPresent usePullSecret: false pullSecretName:
buffer: "@type": memory total_limit_size: 600m chunk_limit_size: 20m chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block retry_max_times: 10 retry_type: periodic retry_wait: 30
tolerations: []
customFilters: {} indexFields: []
splunk-kubernetes-metrics: enabled: true logLevel:
splunk: hec: host: port: token: protocol: fullUrl: indexName: insecureSSL: clientCert: clientKey: caFile: consume_chunk_on_4xx_errors:
image: registry: docker.io name: splunk/k8s-metrics tag: 1.2.2 pullPolicy: IfNotPresent usePullSecret: false pullsecretName:
imageAgg: registry: docker.io name: splunk/k8s-metrics-aggr tag: 1.2.2 pullPolicy: IfNotPresent usePullSecret: false pullsecretName:
environmentVarAgg:
podAnnotationsAgg:
extraLabelsAgg:
resources: fluent: limits: cpu: 200m memory: 300Mi requests: cpu: 200m memory: 300Mi buffer: "@type": memory total_limit_size: 400m chunk_limit_size: 10m chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block retry_max_times: 10 retry_type: periodic retry_wait: 30
aggregatorBuffer: "@type": memory total_limit_size: 400m chunk_limit_size: 10m chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block retry_max_times: 10 retry_type: periodic retry_wait: 30
metricsInterval: 15s
aggregatorNodeSelector: kubernetes.io/os: linux
key: node-role.kubernetes.io/master effect: NoSchedule
aggregatorTolerations: {}
priorityClassNameAgg:
kubernetes: kubeletAddress: kubeletPort: kubeletPortAggregator: useRestClientSSL: true insecureSSL: false caFile: bearerTokenFile: secretDir: clusterName:
customFiltersAggr: {}
rollingUpdate: ` Unfortunately I am not able to see my python script logs in Splunk. Any hint?
Thank you Best Regards Marta
Hi, I deployed my application on Azure Aks (it's a simple python hello world that logs to stdout). And I used splunk for connect with the following yaml file:
`global: logLevel: info splunk: hec: host: port: token: mytoken protocol: https endpoint: /endpoint/ fullUrl: https://fullurl indexName: kube-logs insecureSSL: clientCert: clientKey: caFile: indexRouting: consume_chunk_on_4xx_errors: kubernetes: clusterName: "cluster_name" prometheus_enabled: monitoring_agent_enabled: monitoring_agent_index_name: monitoring_agent_bind_address: metrics: service: enabled: true headless: true serviceMonitor: enabled: false
splunk-kubernetes-logging: enabled: true logLevel:
namespace:
fluentd: path: /var/log/containers/*.log
exclude_path:
containers: path: /var/log pathDest: /var/lib/docker/containers logFormatType: cri logFormat: refreshInterval: removeBlankEvents: true localTime: false enableStatWatcher: true
k8sMetadata: podLabels:
release watch: true cache_ttl: 3600 propagate_namespace_labels: false
sourcetypePrefix: "kube"
rbac: create: true openshiftPrivilegedSccBinding: false
serviceAccount: create: true name:
podSecurityPolicy: create: false apparmor_security: true apiGroup: policy
splunk: hec: host: port: token: protocol: endpoint: fullUrl: indexName: insecureSSL: clientCert: clientKey: caFile: consume_chunk_on_4xx_errors: gzip_compression: ingest_api: serviceClientIdentifier: serviceClientSecretKey: tokenEndpoint: ingestAuthHost: ingestAPIHost: tenant: eventsEndpoint: debugIngestAPI:
secret: create: true name:
journalLogPath: /run/log/journal
charEncodingUtf8: false
customLogs: default: from: pod: helloworld5-5dc446c649-zhvq5 multiline: firstline: /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+\s+\w+\s+[/ flushInterval: 30 logs: docker: from: journald: unit: docker.service sourcetype: kube:docker kubelet: &glog from: journald: unit: kubelet.service multiline: firstline: /^\w[0-1]\d[0-3]\d/ sourcetype: kube:kubelet default: from: pod: helloworld5-5dc446c649-zhvq5 etcd: from: pod: etcd-server container: etcd-container etcd-minikube: from: pod: etcd-minikube container: etcd etcd-events: from: pod: etcd-server-events container: etcd-container kube-apiserver: <<: glog from: pod: kube-apiserver sourcetype: kube:kube-apiserver kube-scheduler: <<: glog from: pod: kube-scheduler sourcetype: kube:kube-scheduler kube-controller-manager: <<: glog from: pod: kube-controller-manager sourcetype: kube:kube-controller-manager kube-proxy: <<: glog from: pod: kube-proxy sourcetype: kube:kube-proxy kubedns: <<: glog from: pod: kube-dns sourcetype: kube:kubedns dnsmasq: <<: glog from: pod: kube-dns sourcetype: kube:dnsmasq dns-sidecar: <<: glog from: pod: kube-dns container: sidecar sourcetype: kube:kubedns-sidecar dns-controller: <<: glog from: pod: dns-controller sourcetype: kube:dns-controller kube-dns-autoscaler: <<: *glog from: pod: kube-dns-autoscaler container: autoscaler sourcetype: kube:kube-dns-autoscaler kube-audit: from: file: path: /var/log/kube-apiserver-audit.log timestampExtraction: format: "%Y-%m-%dT%H:%M:%SZ" sourcetype: kube:apiserver-audit
image: registry: docker.io name: splunk/fluentd-hec tag: 1.3.2 pullPolicy: IfNotPresent usePullSecret: false pullSecretName:
environmentVar:
podAnnotations:
extraLabels:
resources: requests: cpu: 100m memory: 200Mi
bufferChunkKeys:
sendAllMetadata: false
tolerations:
nodeSelector: kubernetes.io/os: linux
affinity: {}
extraVolumes: [] extraVolumeMounts: []
priorityClassName:
kubernetes: clusterName: securityContext: false
customMetadata:
customMetadataAnnotations:
customFilters: {}
indexFields: []
rollingUpdate:
splunk-kubernetes-objects: enabled: true logLevel:
namespace:
rbac: create: true
serviceAccount: create: true name: usePullSecrets: false
podSecurityPolicy: create: false apparmor_security: true apiGroup: policy
priorityClassName:
kubernetes: url: insecureSSL: false clientCert: clientKey: caFile: bearerTokenFile: secretDir: clusterName:
objects: core: v1:
name: events mode: watch
checkpointFile: name: kubernetes-objects.pos volume:
splunk: hec: host: token: protocol: port: fullUrl: indexName: insecureSSL: clientCert: clientKey: caFile: caPath: indexRouting: consume_chunk_on_4xx_errors:
secret: create: true name:
image: registry: docker.io name: splunk/kube-objects tag: 1.2.2 pullPolicy: IfNotPresent usePullSecret: false pullSecretName:
environmentVar:
podAnnotations:
extraLabels:
resources: requests: cpu: 100m memory: 200Mi
buffer: "@type": memory total_limit_size: 600m chunk_limit_size: 20m chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block retry_max_times: 10 retry_type: periodic retry_wait: 30
nodeSelector: kubernetes.io/os: linux
tolerations: []
affinity: {}
customFilters: {} indexFields: []
splunk-kubernetes-metrics: enabled: true logLevel:
namespace:
rbac: create: true
serviceAccount: create: true name: usePullSecrets: false
podSecurityPolicy: create: false apparmor_security: true apiGroup: policy
splunk: hec: host: port: token: protocol: fullUrl: indexName: insecureSSL: clientCert: clientKey: caFile: consume_chunk_on_4xx_errors:
secret: create: true name:
image: registry: docker.io name: splunk/k8s-metrics tag: 1.2.2 pullPolicy: IfNotPresent usePullSecret: false pullsecretName:
imageAgg: registry: docker.io name: splunk/k8s-metrics-aggr tag: 1.2.2 pullPolicy: IfNotPresent usePullSecret: false pullsecretName:
environmentVar:
environmentVarAgg:
podAnnotations:
podAnnotationsAgg:
extraLabels:
extraLabelsAgg:
resources: fluent: limits: cpu: 200m memory: 300Mi requests: cpu: 200m memory: 300Mi buffer: "@type": memory total_limit_size: 400m chunk_limit_size: 10m chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block retry_max_times: 10 retry_type: periodic retry_wait: 30
aggregatorBuffer: "@type": memory total_limit_size: 400m chunk_limit_size: 10m chunk_limit_records: 10000 flush_interval: 5s flush_thread_count: 1 overflow_action: block retry_max_times: 10 retry_type: periodic retry_wait: 30
metricsInterval: 15s
nodeSelector: kubernetes.io/os: linux
aggregatorNodeSelector: kubernetes.io/os: linux
tolerations:
key: node-role.kubernetes.io/master effect: NoSchedule
aggregatorTolerations: {}
priorityClassName:
priorityClassNameAgg:
affinity: {}
kubernetes: kubeletAddress: kubeletPort: kubeletPortAggregator: useRestClientSSL: true insecureSSL: false caFile: bearerTokenFile: secretDir: clusterName:
customFilters: {}
customFiltersAggr: {}
rollingUpdate: ` Unfortunately I am not able to see my python script logs in Splunk. Any hint?
Thank you Best Regards Marta