search

grafana / loki

Like Prometheus, but for logs.
https://grafana.com/loki
GNU Affero General Public License v3.0
23.74k stars 3.43k forks source link

NotFound: Not Found\n\tstatus code: 404, request id: , host id: #8094

Open wkshare opened 1 year ago

wkshare commented 1 year ago

Describe the bug I can query the data normally, but I don't know why these error logs are appearing.

Version Info loki, version 2.6.1 (branch: HEAD, revision: 6bd05c9a4) build user: root@af90ed01061f build date: 2022-07-18T08:41:09Z go version: go1.17.6 platform: linux/amd64

Environment: LSB Version: :core-4.1-amd64:core-4.1-noarch Distributor ID: OracleServer Description: Oracle Linux Server release 7.9 Release: 7.9 Codename: n/a Single binary deploy

Screenshots, Promtail config, or terminal output my loki config

auth_enabled: true

common:
  replication_factor: 1
  ring:
    instance_addr: 0.0.0.0
    kvstore:
      store: memberlist
#    instance_interface_names:
#      - ens160

server:
  log_level: warn
  http_listen_port: 3100
  grpc_listen_port: 13100
  grpc_server_max_recv_msg_size: 114857600
  grpc_server_max_send_msg_size: 114857600
  grpc_server_max_concurrent_streams: 1024
  http_server_write_timeout: 60s
  http_server_read_timeout: 60s

  http_tls_config:
    cert_file: /mon/app/loki/certs/server.crt
    key_file: /mon/app/loki/certs/server.key

ingester:
  lifecycler:
    join_after: 60s
    observe_period: 5s
    ring:
      kvstore:
        store: memberlist
    final_sleep: 0s
  autoforget_unhealthy: true
  chunk_idle_period: 1h
  wal:
    enabled: true
    dir: /mon/data/loki/wal
  max_chunk_age: 1h
  chunk_retain_period: 30s
  chunk_encoding: snappy
  chunk_target_size: 0
  chunk_block_size: 262144

memberlist:
  abort_if_cluster_join_fails: false

  bind_port: 7946

  join_members:
    - 10.157.156.194
    - 10.157.156.198

  dead_node_reclaim_time: 30s
  gossip_to_dead_nodes_time: 15s
  left_ingesters_timeout: 30s

  max_join_backoff: 1m
  max_join_retries: 10
  min_join_backoff: 1s

storage_config:
  max_chunk_batch_size: 1024
  boltdb_shipper:
    active_index_directory: /mon/data/loki/boltdb-shipper-active
    cache_location: /mon/data/loki/boltdb-shipper-cache
    cache_ttl: 24h
    resync_interval: 5s
    shared_store: s3
  aws:
    bucketnames: loki
    endpoint: https://10.157.156.198:3443
    access_key_id: ${ACCESS_KEY_ID}
    secret_access_key: ${SECRET_ACCESS_KEY}
    insecure: true
    http_config:
      idle_conn_timeout: 90s
      response_header_timeout: 5s
      insecure_skip_verify: true
    s3forcepathstyle: true

schema_config:
  configs:
    - from: 2022-03-22
      store: boltdb-shipper
      object_store: s3
      schema: v11
      index:
        prefix: index_
        period: 24h

limits_config:
  max_query_series: 1000
  max_cache_freshness_per_query: '10m'
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  per_stream_rate_limit: 20MB
  ingestion_rate_mb: 300
  ingestion_burst_size_mb: 500
  split_queries_by_interval: 15m
  retention_period: 720h

table_manager:
  retention_deletes_enabled: true
  retention_period: 336h

query_range:
  align_queries_with_step: true
  max_retries: 5
  parallelise_shardable_queries: true
  cache_results: true
  results_cache:
    cache:
      enable_fifocache: true
      fifocache:
        size: 1024
        validity: 24h

compactor:
  working_directory: /mon/data/loki/compactor
  shared_store: s3
  compaction_interval: 5m
  retention_enabled: true
  retention_delete_delay: 1h
  retention_delete_worker_count: 150

ruler:
  enable_api: false
  enable_sharding: true
  rule_path: /mon/data/loki/scratch
  storage:
    type: local
    local:
      directory: /mon/data/loki/rules
  remote_write:
    enabled: true
    client:
      url: https://10.157.156.198:9443/api/v1/write
      tls_config:
        ca_file: /mon/app/loki/certs/ca.crt
        cert_file: /mon/app/loki/certs/client.crt
        key_file: /mon/app/loki/certs/client.key
        insecure_skip_verify: true
  wal:
    dir: /mon/data/loki/ruler-wal

querier:
  max_concurrent: 2048
  engine:
    timeout: 5m
  query_timeout: 5m
  query_ingesters_within: 2h

my promtail config in grafana agent

logs:
  configs:
  - name: loki
    clients:
      - url: https://10.157.156.198:3443/api/prom/push
        tenant_id: log_selfmonitoring
        tls_config:
          cert_file: /mon/app/grafana_agent/certs/client.crt
          key_file: /mon/app/grafana_agent/certs/client.key
          insecure_skip_verify: true
        batchwait: 1s
        batchsize: 1048576
    positions:
      filename: /mon/data/grafana_agent/positions.yaml
    scrape_configs:
      - job_name: syslog
        syslog:
          listen_address: 127.0.0.1:1515
          labels:
            type: syslog
      - job_name: journal
        journal:
          json: false
          max_age: 12h
          path: /run/log/journal
          labels:
            type: journal
        relabel_configs:
        - source_labels:
          - __journal__systemd_unit
          target_label: systemd_unit
        - source_labels:
          - __journal__hostname
          target_label: nodename
        - source_labels:
          - __journal_syslog_identifier
          target_label: syslog_identifier

      - job_name: varlogs
        static_configs:
          - targets: [localhost]
            labels:
              hostname: 10.157.156.194
              type: dmesg
              __path__: /var/log/dmesg
Jan 11 18:38:39 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:38:39.51712991Z caller=index_set.go:104 table-name=index_19368 msg="failed to initialize table index_19368, cleaning it up" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:38:39 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:38:39.51715826Z caller=table.go:294 table-name=index_19368 msg="failed to init user index set " err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:38:39 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:38:39.51719524Z caller=table.go:312 table-name=index_19368 org_id=log_selfmonitoring msg="index set  has some problem, cleaning it up" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:39:29 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:39:29.618071868Z caller=reporter.go:203 msg="failed to delete corrupted cluster seed file, deleting it" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:41:21 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:41:21.310140843Z caller=cached_client.go:65 msg="failed to build cache" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:41:21 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:41:21.310217385Z caller=compactor.go:432 msg="failed to run compaction" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:43:38 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:43:38.347397205Z caller=reporter.go:203 msg="failed to delete corrupted cluster seed file, deleting it" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:46:21 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:46:21.310044264Z caller=cached_client.go:65 msg="failed to build cache" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:46:21 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:46:21.310120092Z caller=compactor.go:432 msg="failed to run compaction" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:48:16 ech-10-157-156-194 start.sh[1006491]: level=error ts=2023-01-11T10:48:16.44624135Z caller=reporter.go:203 msg="failed to delete corrupted cluster seed file, deleting it" err="NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.986576173Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.986493098Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.989045969Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.989807534Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.990267425Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.991425668Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.993760776Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.99515964Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.997264712Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
Jan 11 18:55:24 ech-10-157-156-198 start.sh[703987]: level=error ts=2023-01-11T10:55:24.998843214Z caller=flush.go:146 org_id=log_selfmonitoring msg="failed to flush user" err="store put chunk: NotFound: Not Found\n\tstatus code: 404, request id: , host id: "
chaudum commented 1 year ago

Are you sure the bucket exists? If you use MinIO (which I assume based on your config), you need to create the data directory for the buckets.