otelcol

[zking2000]

# 1. 首先创建日志接收器
apiVersion: logging.cnrm.cloud.google.com/v1beta1
kind: LoggingLogSink
metadata:
  name: all-logs-sink
  namespace: monitoring
spec:
  projectRef:
    external: ${PROJECT_ID}
  destination: "pubsub.googleapis.com/projects/${PROJECT_ID}/topics/${TOPIC_NAME}"
  filter: "" # 空过滤器表示接收所有日志
  uniqueWriterIdentity: true

---
# otel-collector-config.yaml
receivers:
  googlecloudpubsub:
    project: ${PROJECT_ID}
    subscription: ${SUBSCRIPTION_ID}
    endpoint: pubsub.googleapis.com:443

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

  resourcedetection:
    detectors: [gcp]
    timeout: 2s

  attributes:
    actions:
      # 保留所有原始字段
      - key: resource_type
        action: upsert
        from_attribute: resource.type
      - key: project_id
        action: upsert
        from_attribute: resource.labels.project_id
      - key: severity
        action: upsert
        from_attribute: severity
      - key: log_id
        action: upsert
        from_attribute: logName

exporters:
  loki:
    endpoint: "http://loki:3100/loki/api/v1/push"
    tenant_id: "tenant-1"
    labels:
      resource:
        resource_type: "resource_type"
        project_id: "project_id"
      attributes:
        severity: "severity"
        log_id: "log_id"
    default_labels_enabled: true

service:
  telemetry:
    metrics:
      address: ":8888"
  pipelines:
    logs:
      receivers: [googlecloudpubsub]
      processors: [resourcedetection, attributes, batch]
      exporters: [loki]

---
# kubernetes-deployment.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-collector
  namespace: monitoring
  annotations:
    iam.gke.io/gcp-service-account: otel-collector@${PROJECT_ID}.iam.gserviceaccount.com

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-config
  namespace: monitoring
data:
  otel-collector-config.yaml: |
    # 将上面的配置粘贴到这里

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector
  namespace: monitoring
spec:
  replicas: 2
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      serviceAccountName: otel-collector
      containers:
      - name: otel-collector
        image: otel/opentelemetry-collector-contrib:latest
        args:
          - "--config=/conf/otel-collector-config.yaml"
        ports:
          - containerPort: 8888
            name: metrics
        env:
          - name: GCP_PROJECT_ID
            value: "${PROJECT_ID}"
          - name: PUBSUB_SUBSCRIPTION_ID
            value: "${SUBSCRIPTION_ID}"
        resources:
          requests:
            cpu: 1
            memory: 2Gi
          limits:
            cpu: 2
            memory: 4Gi
        volumeMounts:
          - name: otel-collector-config
            mountPath: /conf
      volumes:
        - name: otel-collector-config
          configMap:
            name: otel-collector-config

[zking2000]

processors:
  filter/parse:
    logs:
      # 使用 JQ 表达式解析 JSON 字符串并提取需要的字段
      jq: |
        .body as $raw |
        try (
          ($raw | fromjson) as $parsed |
          {
            "attributes": {
              "project_id": $parsed.resource.labels.project_id,
              "log_name": $parsed.logName,
              "severity": $parsed.severity
              # 添加其他你需要作为索引的字段
            },
            "body": $parsed.textPayload # 或者 $parsed.jsonPayload
          }
        ) catch $raw

[zking2000]

processors:
  filter:
    logs:
      strict: false
      expr: |
        # 尝试解析 JSON
        body = parse_json(body)
        # 设置属性作为索引
        attributes["project_id"] = body.resource.labels.project_id
        attributes["log_name"] = body.logName
        attributes["severity"] = body.severity
        # 设置日志内容
        body = body.textPayload # 或者 body.jsonPayload