search

cromono / log-consumer

0 stars 0 forks source link

Filebeats #2

Closed copolio closed 3 years ago

copolio commented 3 years ago

In GitLab by @gm2202983 on Jun 10, 2021, 09:09

설치 방법

# 설치
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.13.2-linux-x86_64.tar.gz
tar xzvf filebeat-7.13.2-linux-x86_64.tar.gz

# 설정
cd filebeat-7.13.2-linux-x86_64/

# syslog만 받아보고 싶은 경우 이하 명령어 실행. 그것이 싫은 경우엔 filebeat 폴더내에 logproducer.yml 생성.
curl -0 logproducer.yml https://gitlab.gabia.com/mentoring/newbie/2021.05/mentoring-cron/log-consumer/uploads/5b8aff3f8e0999240355ca784d199c1c/logproducer.yml

# 실행
./filebeat -e -c logproducer.yml &

설정 파일

logproducer.yml

filebeat.inputs:
  - type: log
    enabled: true

    paths:
      - /var/log/syslog
    fields:
      log_topic: "syslog"
    encoding: utf-8
    include_lines: ["CRON"]
  # Syslog를 제외한 로그의 경우, 서버에 로그가 쌓이는 방식에 맞춰 설정
  # - type: log
  #   enabled: true
  # 개발자가 남긴 Syslog 파일 혹은 저장경로
  #   paths:
  #     - /var/log/*
  #   fields:
  #     log_topic: "sysdevlog"
  #   encoding: utf-8
  # - type: log
  #   enabled: true
  # 개발자가 남긴 로그 파일 혹은 저장경로
  #   paths:
  #     - /tmp/check_test
  #   fields:
  #     log_topic: "devlog"
  #   encoding: utf-8

output.kafka:
  hosts: ["182.162.142.151:9093"]
  topic: "%{[fields.log_topic]}"
  partition.round_robin:
    reachable_only: false

  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

  codec.json:
    pretty: true

# ================================= Processors =================================
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  # - add_cloud_metadata: ~
  # - add_docker_metadata: ~
  # - add_kubernetes_metadata: ~

예시 로그

{
  "@timestamp": "2021-06-16T06:21:01.681Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.13.2"
  },
  "agent": {
    "id": "255f5de9-99b8-4512-bd96-b75d69ddbcc4",
    "name": "DESKTOP-Q5DGAKN",
    "type": "filebeat",
    "version": "7.13.2",
    "hostname": "DESKTOP-Q5DGAKN",
    "ephemeral_id": "36319a9d-d2ea-4357-960e-3da320941073"
  },
  "ecs": {
    "version": "1.8.0"
  },
  "log": {
    "offset": 93394,
    "file": {
      "path": "/var/log/syslog"
    }
  },
  "message": "Jun 16 15:21:01 DESKTOP-Q5DGAKN CRON[6626]: (CRON) info (No MTA installed, discarding output)",
  "input": {
    "type": "log"
  },
  "fields": {
    "log_topic": "syslog"
  },
  "host": {
    "ip": [
      "172.29.45.55",
      "fe80::215:5dff:fee2:2ca4"
    ],
    "mac": [
      "4e:cb:8b:29:18:5d",
      "6e:24:f3:c4:d7:47",
      "00:15:5d:e2:2c:a4"
    ],
    "hostname": "DESKTOP-Q5DGAKN",
    "architecture": "x86_64",
    "os": {
      "name": "Ubuntu",
      "kernel": "5.4.72-microsoft-standard-WSL2",
      "codename": "focal",
      "type": "linux",
      "platform": "ubuntu",
      "version": "20.04.2 LTS (Focal Fossa)",
      "family": "debian"
    },
    "containerized": false,
    "name": "DESKTOP-Q5DGAKN"
  }
}

참고자료

copolio commented 3 years ago

In GitLab by @gm2202981 on Jun 16, 2021, 13:42

changed the description

copolio commented 3 years ago

In GitLab by @gm2202981 on Jun 16, 2021, 15:23

changed the description

copolio commented 3 years ago

In GitLab by @gm2202981 on Jun 16, 2021, 15:32

changed the description

copolio commented 3 years ago

In GitLab by @gm2202981 on Jun 16, 2021, 15:38

changed the description