Closed ghost closed 2 years ago
We are trying to add pod_name to the s3 path, but the config is not able to read the pod_name from metadata. Can anyone please help. it just adding ${record["kubernetes"]["pod_name"]}
pods-fluent.conf: |-
@type tail @id in_tail_container_logs path /var/log/containers/*.log pos_file /var/log/containers/fluentd/fluentd.pos exclude_path ["/var/log/containers/fluent*"] tag kubernetes.* path_key log_path read_from_head true <parse> @type kubernetes @type "#{ENV['FLUENT_CONTAINER_TAIL_PARSER_TYPE'] || 'json'}" time_format %Y-%m-%dT%H:%M:%S.%NZ </parse> </source> <filter kubernetes.**> @type kubernetes_metadata kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}" bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || false}" ca_file "#{ENV['KUBERNETES_CA_FILE']}" cache_size 1000 skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}" skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}" skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}" skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}" watch false merge_json_log false use_journal false </filter> <filter kubernetes.**> @type record_transformer enable_ruby true <record> pod ${record["kubernetes"]["pod_name"]} </record> </filter>
output-fluent.conf: |- <match **> @type copy
<store> @type s3 @log_level debug s3_bucket "#{ENV['s3_bucket']}" s3_region "#{ENV['Region']}" path k8s-nonsensitive time_slice_format %Y/%m/%d/%H time_slice_wait 10m s3_object_key_format "logs/test/${record[\"kubernetes\"][\"pod_name\"]}/log_%{index}.txt" time_slice_format %Y/%m/%d/%H <instance_profile_credentials> ip_address 169.254.169.254 port 80 </instance_profile_credentials> <buffer tag,time> @type file flush_mode interval flush_interval 10s path /var/log/fluent/s3 timekey 300 # 1 hour partition timekey_wait 1m timekey_use_utc true # use utc chunk_limit_size 100M </buffer> <format> @type json </format> </store> </match>
fluentd log : {"log":"2021-12-02 13:35:13 XXXX ::1 GET /WebApplications/ApplicationServer/ServiceListener.asmx - 443 - ::1 Mozilla/5.0+(Windows+NT;+Windows+NT+10.0;+en-US)+WindowsPowerShell/5.1.17763.1971 - 200 0 0 3626 203 9\r\r\n","stream":"stdout","log_path":"/var/log/containers/web-test-XXXXXX.log","docker":{"container_id":"XXXXXXX"},"kubernetes":{"container_name":"web-test","namespace_name":".orphaned","pod_name":"web-test-XXXXXXX","orphaned_namespace":"web-strelka-dev","namespace_id":"orphaned"},"service":"web-test-8bf54dfc6-b476z"} {"message":"out_s3: write chunk 5d229de7f9afb249d77f1325402b14d4 with metadata #<struct Fluent::Plugin::Buffer::Metadata timekey=1638452100, tag=\"fluent.info\", variables=nil, seq=0> to s3://issuing-apps-logs-devsyntheticdigitalinnov-eu-central-1-tsys/logs/test/${record[\"kubernetes\"][\"pod_name\"]}/log_5.txt"}
Please refer your question directly to fluentd forums as this is for the kubernetes metadata plugin specifically
We are trying to add pod_name to the s3 path, but the config is not able to read the pod_name from metadata. Can anyone please help. it just adding ${record["kubernetes"]["pod_name"]}
pods-fluent.conf: |-