search

influxdata / telegraf

Agent for collecting, processing, aggregating, and writing metrics, logs, and other arbitrary data.
https://influxdata.com/telegraf
MIT License
14.51k stars 5.55k forks source link

MQTT Output writes metrics regardless of MQTT Broker being switched off #14750

Closed dbleul closed 7 months ago

dbleul commented 7 months ago

Relevant telegraf.conf

[agent]
  interval = "1s"
  flush_interval = "20s"
  metric_batch_size = 10000
  metric_buffer_limit = 1000000
  debug = true

# Accept metrics over InfluxDB 1.x HTTP API
[[inputs.influxdb_listener]]
  ## Address and port to host HTTP listener on
  service_address = ":8186"

  ## maximum duration before timing out read of the request
  read_timeout = "10s"
  ## maximum duration before timing out write of the response
  write_timeout = "10s"

  ## Maximum allowed HTTP request body size in bytes.
  ## 0 means to use the default of 32MiB.
  max_body_size = 0

  ## Maximum line size allowed to be sent in bytes.
  ##   deprecated in 1.14; parser now handles lines of unlimited length and option is ignored
  # max_line_size = 0

  ## Set one or more allowed client CA certificate file names to
  ## enable mutually authenticated TLS connections
  #tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"]

  ## Add service certificate and key
  #tls_cert = "/etc/telegraf/cert.pem"
  #tls_key = "/etc/telegraf/key.pem"

  ## Optional tag name used to store the database name.
  ## If the write has a database in the query string then it will be kept in this tag name.
  ## This tag can be used in downstream outputs.
  ## The default value of nothing means it will be off and the database will not be recorded.
  ## If you have a tag that is the same as the one specified below, and supply a database,
  ## the tag will be overwritten with the database supplied.
  # database_tag = ""

  ## If set the retention policy specified in the write query will be added as
  ## the value of this tag name.
  # retention_policy_tag = ""

  ## Optional username and password to accept for HTTP basic authentication
  ## or authentication token.
  ## You probably want to make sure you have TLS configured above for this.
  ## Use these options for the authentication token in the form
  ##   Authentication: Token <basic_username>:<basic_password>
  # basic_username = "foobar"
  # basic_password = "barfoo"

  ## Optional JWT token authentication for HTTP requests
  ## Please see the documentation at
  ##   https://docs.influxdata.com/influxdb/v1.8/administration/authentication_and_authorization/#authenticate-using-jwt-tokens
  ## for further details.
  ## Please note: Token authentication and basic authentication cannot be used
  ##              at the same time.
  # token_shared_secret = ""
  # token_username = ""

  ## Influx line protocol parser
  ## 'internal' is the default. 'upstream' is a newer parser that is faster
  ## and more memory efficient.
  # parser_type = "internal"

[[outputs.mqtt]]
  ## MQTT Brokers
  ## The list of brokers should only include the hostname or IP address and the
  ## port to the broker. This should follow the format `[{scheme}://]{host}:{port}`. For
  ## example, `localhost:1883` or `mqtt://localhost:1883`.
  ## Scheme can be any of the following: tcp://, mqtt://, tls://, mqtts://
  ## non-TLS and TLS servers can not be mix-and-matched.
  servers = ["nats-master-1:1883"] # or ["mqtts://tls.example.com:1883"]

  ## Protocol can be `3.1.1` or `5`. Default is `3.1.1`
  protocol = "3.1.1"

  ## MQTT Topic for Producer Messages
  ## MQTT outputs send metrics to this topic format:
  ## {{ .TopicPrefix }}/{{ .Hostname }}/{{ .PluginName }}/{{ .Tag "tag_key" }}
  ## (e.g. prefix/web01.example.com/mem/some_tag_value)
  ## Each path segment accepts either a template placeholder, an environment variable, or a tag key
  ## of the form `{{.Tag "tag_key_name"}}`. Empty path elements as well as special MQTT characters
  ## (such as `+` or `#`) are invalid to form the topic name and will lead to an error.
  ## In case a tag is missing in the metric, that path segment omitted for the final topic.
  topic = "telegraf/{{ .PluginName }}"

  ## QoS policy for messages
  ## The mqtt QoS policy for sending messages.
  ## See https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.0.0/com.ibm.mq.dev.doc/q029090_.htm
  ##   0 = at most once
  ##   1 = at least once
  ##   2 = exactly once
  qos = 1

  ## Keep Alive
  ## Defines the maximum length of time that the broker and client may not
  ## communicate. Defaults to 0 which turns the feature off.
  ##
  ## For version v2.0.12 and later mosquitto there is a bug
  ## (see https://github.com/eclipse/mosquitto/issues/2117), which requires
  ## this to be non-zero. As a reference eclipse/paho.mqtt.golang defaults to 30.
  keep_alive = 0

  ## username and password to connect MQTT server.
  username = "cluster"
  password = "password"

  ## client ID
  ## The unique client id to connect MQTT server. If this parameter is not set
  ## then a random ID is generated.
  # client_id = ""

  ## Timeout for write operations. default: 5s
  timeout = "8s"

  ## Optional TLS Config
  # tls_ca = "/etc/telegraf/ca.pem"
  # tls_cert = "/etc/telegraf/cert.pem"
  # tls_key = "/etc/telegraf/key.pem"

  ## Use TLS but skip chain & host verification
  # insecure_skip_verify = false

  ## When true, metrics will be sent in one MQTT message per flush. Otherwise,
  ## metrics are written one metric per MQTT message.
  ## DEPRECATED: Use layout option instead
  # batch = true

  ## When true, metric will have RETAIN flag set, making broker cache entries until someone
  ## actually reads it
  # retain = false

  ## Layout of the topics published.
  ## The following choices are available:
  ##   non-batch -- send individual messages, one for each metric
  ##   batch     -- send all metric as a single message per MQTT topic
  ## NOTE: The following options will ignore the 'data_format' option and send single values
  ##   field     -- send individual messages for each field, appending its name to the metric topic
  ##   homie-v4  -- send metrics with fields and tags according to the 4.0.0 specs
  ##                see https://homieiot.github.io/specification/
  layout = "batch"

  ## HOMIE specific settings
  ## The following options provide templates for setting the device name
  ## and the node-ID for the topics. Both options are MANDATORY and can contain
  ## {{ .PluginName }} (metric name), {{ .Tag "key"}} (tag reference to 'key')
  ## or constant strings. The templays MAY NOT contain slashes!
  # homie_device_name = ""
  # homie_node_id = ""

  ## Each data format has its own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
  #data_format = "influx"

  ## NOTE: Due to the way TOML is parsed, tables must be at the END of the
  ## plugin definition, otherwise additional config options are read as part of
  ## the table

  ## Optional MQTT 5 publish properties
  ## These setting only apply if the "protocol" property is set to 5. This must
  ## be defined at the end of the plugin settings, otherwise TOML will assume
  ## anything else is part of this table. For more details on publish properties
  ## see the spec:
  ## https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901109
  # [outputs.mqtt.v5]
  #   content_type = ""
  #   response_topic = ""
  #   message_expiry = "0s"
  #   topic_alias = 0
  # [outputs.mqtt.v5.user_properties]
  #   "key1" = "value 1"
  #   "key2" = "value 2"

# Send telegraf measurements to NATS
#[[outputs.nats]]
  ## URLs of NATS servers
#  servers = ["nats://nats-master-1:4222"]

  ## Optional client name
  # name = ""

  ## Optional credentials
#  username = "cluster"
#  password = "password"

  ## Optional NATS 2.0 and NATS NGS compatible user credentials
  # credentials = "/etc/telegraf/nats.creds"

  ## NATS subject for producer messages
  ## For jetstream this is also the subject where messages will be published
#  subject = "telegraf.test"

  ## Use Transport Layer Security
  # secure = false

  ## Optional TLS Config
  # tls_ca = "/etc/telegraf/ca.pem"
  # tls_cert = "/etc/telegraf/cert.pem"
  # tls_key = "/etc/telegraf/key.pem"
  ## Use TLS but skip chain & host verification
  # insecure_skip_verify = false

  ## Data format to output.
  ## Each data format has its own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
#  data_format = "influx"

  ## Jetstream specific configuration. If not nil, it will assume Jetstream context.
  ## Since this is a table, it should be present at the end of the plugin section. Else you can use inline table format.
  # [outputs.nats.jetstream]
    ## Name of the stream, required when using jetstream. Telegraf will
    ## use the union of the above subject and below the subjects array.
    # name = "" 
    # subjects = []

    ## Full jetstream create stream config, refer: https://docs.nats.io/nats-concepts/jetstream/streams
    # retention = "limits"
    # max_consumers = -1
    # max_msgs_per_subject = -1
    # max_msgs = -1
    # max_bytes = -1
    # max_age = 0
    # max_msg_size = -1
    # storage = "file"
    # discard = "old"
    # num_replicas = 1
    # duplicate_window = 120000000000
    # sealed = false
    # deny_delete = false
    # deny_purge = false
    # allow_rollup_hdrs = false
    # allow_direct = true
    # mirror_direct = false

Logs from Telegraf

telegraf-telegraf-1  | 2024-02-11T19:22:28Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:22:36Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
**telegraf-telegraf-1  | 2024-02-11T19:22:36Z D! [outputs.mqtt] Wrote batch of 6834 metrics in 1m12.02919199s**
telegraf-telegraf-1  | 2024-02-11T19:22:36Z D! [outputs.mqtt] Buffer fullness: 24564 / 1000000 metrics
telegraf-telegraf-1  | 2024-02-11T19:22:44Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:22:52Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:00Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:08Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:16Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:24Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:32Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:40Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:23:48Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
**telegraf-telegraf-1  | 2024-02-11T19:23:48Z D! [outputs.mqtt] Wrote batch of 10000 metrics in 1m12.036357054s**
telegraf-telegraf-1  | 2024-02-11T19:23:48Z D! [outputs.mqtt] Buffer fullness: 39130 / 1000000 metrics
telegraf-telegraf-1  | 2024-02-11T19:23:56Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:04Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:12Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:20Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:28Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:36Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:44Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:24:52Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:00Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
**telegraf-telegraf-1  | 2024-02-11T19:25:00Z D! [outputs.mqtt] Wrote batch of 10000 metrics in 1m12.04881222s**
telegraf-telegraf-1  | 2024-02-11T19:25:00Z D! [outputs.mqtt] Buffer fullness: 53673 / 1000000 metrics
telegraf-telegraf-1  | 2024-02-11T19:25:08Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:16Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:24Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:32Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:40Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:48Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:25:56Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:26:04Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out
telegraf-telegraf-1  | 2024-02-11T19:26:12Z W! [outputs.mqtt] Could not publish message to MQTT server: command timed out

System info

Telegraf 1.29.4

Docker

No response

Steps to reproduce

  1. Start MQTT Broker
  2. Start Telegraf with MQTT Output
  3. Stop MQTT Broker ...

Expected behavior

Telegraf no longer writes metrics to the MQTT output and fills its buffer

Actual behavior

Despite the MQTT broker being switched off, Telegraf reports a successful write after one minute and deletes the metrics from the buffer

Additional info

No response

srebhan commented 7 months ago

@dbleul can you please test the binary in PR #14752 available once CI finished all tests?! Let me know if this fixes the issue for you!

dbleul commented 7 months ago

Hello @srebhan

the fix solves the problem! Thanks for the quick response time!

2024-02-12T18:36:55Z I! Loading config: C:\Data\git\nats\telegraf\telegraf_dc.conf 2024-02-12T18:36:55Z I! Starting Telegraf 1.30.0-3782eb70 brought to you by InfluxData the makers of InfluxDB 2024-02-12T18:36:55Z I! Available plugins: 241 inputs, 9 aggregators, 30 processors, 24 parsers, 61 outputs, 5 secret-stores 2024-02-12T18:36:55Z I! Loaded inputs: influxdb_listener 2024-02-12T18:36:55Z I! Loaded aggregators: 2024-02-12T18:36:55Z I! Loaded processors: 2024-02-12T18:36:55Z I! Loaded secretstores: 2024-02-12T18:36:55Z I! Loaded outputs: mqtt 2024-02-12T18:36:55Z I! Tags enabled: host=annika-laptop 2024-02-12T18:36:55Z I! [agent] Config: Interval:1s, Quiet:false, Hostname:"annika-laptop", Flush Interval:5s 2024-02-12T18:36:55Z D! [agent] Initializing plugins 2024-02-12T18:36:55Z D! [agent] Connecting outputs 2024-02-12T18:36:55Z D! [agent] Attempting connection to [outputs.mqtt] 2024-02-12T18:36:55Z D! [agent] Successfully connected to outputs.mqtt 2024-02-12T18:36:55Z D! [agent] Starting service inputs 2024-02-12T18:36:55Z I! [inputs.influxdb_listener] Started HTTP listener service on :8186 2024-02-12T18:37:00Z D! [outputs.mqtt] Wrote batch of 8846 metrics in 109.1811ms 2024-02-12T18:37:00Z D! [outputs.mqtt] Buffer fullness: 0 / 1000000 metrics 2024-02-12T18:37:05Z D! [outputs.mqtt] Wrote batch of 2068 metrics in 31.3318ms 2024-02-12T18:37:05Z D! [outputs.mqtt] Buffer fullness: 0 / 1000000 metrics 2024-02-12T18:37:10Z D! [outputs.mqtt] Wrote batch of 1360 metrics in 29.9697ms 2024-02-12T18:37:10Z D! [outputs.mqtt] Buffer fullness: 0 / 1000000 metrics 2024-02-12T18:37:15Z D! [outputs.mqtt] Wrote batch of 2041 metrics in 29.9164ms 2024-02-12T18:37:15Z D! [outputs.mqtt] Buffer fullness: 0 / 1000000 metrics 2024-02-12T18:37:20Z D! [outputs.mqtt] Wrote batch of 1364 metrics in 25.7452ms 2024-02-12T18:37:20Z D! [outputs.mqtt] Buffer fullness: 0 / 1000000 metrics 2024-02-12T18:37:25Z D! [outputs.mqtt] Wrote batch of 2040 metrics in 29.121ms 2024-02-12T18:37:25Z D! [outputs.mqtt] Buffer fullness: 0 / 1000000 metrics ---- STOPPED MQTT BROKER ------ 2024-02-12T18:37:35Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:37:35Z D! [outputs.mqtt] Buffer fullness: 3400 / 1000000 metrics 2024-02-12T18:37:38Z D! [outputs.mqtt] Buffer fullness: 4109 / 1000000 metrics 2024-02-12T18:37:38Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:37:45Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:37:45Z D! [outputs.mqtt] Buffer fullness: 6834 / 1000000 metrics 2024-02-12T18:37:48Z D! [outputs.mqtt] Buffer fullness: 7514 / 1000000 metrics 2024-02-12T18:37:48Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:37:55Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:37:55Z D! [outputs.mqtt] Buffer fullness: 10234 / 1000000 metrics 2024-02-12T18:37:58Z D! [outputs.mqtt] Buffer fullness: 10914 / 1000000 metrics 2024-02-12T18:37:58Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:38:05Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:38:05Z D! [outputs.mqtt] Buffer fullness: 13634 / 1000000 metrics 2024-02-12T18:38:08Z D! [outputs.mqtt] Buffer fullness: 14324 / 1000000 metrics 2024-02-12T18:38:08Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:38:15Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:38:15Z D! [outputs.mqtt] Buffer fullness: 17063 / 1000000 metrics 2024-02-12T18:38:18Z D! [outputs.mqtt] Buffer fullness: 17743 / 1000000 metrics 2024-02-12T18:38:18Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:38:25Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:38:25Z D! [outputs.mqtt] Buffer fullness: 20468 / 1000000 metrics 2024-02-12T18:38:28Z D! [outputs.mqtt] Buffer fullness: 21148 / 1000000 metrics 2024-02-12T18:38:28Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:38:35Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:38:35Z D! [outputs.mqtt] Buffer fullness: 23868 / 1000000 metrics 2024-02-12T18:38:38Z D! [outputs.mqtt] Buffer fullness: 24548 / 1000000 metrics 2024-02-12T18:38:38Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:38:45Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:38:45Z D! [outputs.mqtt] Buffer fullness: 27280 / 1000000 metrics 2024-02-12T18:38:48Z D! [outputs.mqtt] Buffer fullness: 27948 / 1000000 metrics 2024-02-12T18:38:48Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:38:55Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:38:55Z D! [outputs.mqtt] Buffer fullness: 30697 / 1000000 metrics 2024-02-12T18:38:58Z D! [outputs.mqtt] Buffer fullness: 31382 / 1000000 metrics 2024-02-12T18:38:58Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:39:05Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:39:05Z D! [outputs.mqtt] Buffer fullness: 34102 / 1000000 metrics 2024-02-12T18:39:08Z D! [outputs.mqtt] Buffer fullness: 34782 / 1000000 metrics 2024-02-12T18:39:08Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:39:15Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:39:15Z D! [outputs.mqtt] Buffer fullness: 37502 / 1000000 metrics 2024-02-12T18:39:18Z D! [outputs.mqtt] Buffer fullness: 38211 / 1000000 metrics 2024-02-12T18:39:18Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:39:25Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:39:25Z D! [outputs.mqtt] Buffer fullness: 40936 / 1000000 metrics 2024-02-12T18:39:28Z D! [outputs.mqtt] Buffer fullness: 41616 / 1000000 metrics 2024-02-12T18:39:28Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:39:35Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:39:35Z D! [outputs.mqtt] Buffer fullness: 44336 / 1000000 metrics 2024-02-12T18:39:38Z D! [outputs.mqtt] Buffer fullness: 45016 / 1000000 metrics 2024-02-12T18:39:38Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:39:45Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:39:45Z D! [outputs.mqtt] Buffer fullness: 47736 / 1000000 metrics 2024-02-12T18:39:48Z D! [outputs.mqtt] Buffer fullness: 48421 / 1000000 metrics 2024-02-12T18:39:48Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:39:55Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:39:55Z D! [outputs.mqtt] Buffer fullness: 51167 / 1000000 metrics 2024-02-12T18:39:58Z D! [outputs.mqtt] Buffer fullness: 51845 / 1000000 metrics 2024-02-12T18:39:58Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:40:05Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:40:05Z D! [outputs.mqtt] Buffer fullness: 54570 / 1000000 metrics 2024-02-12T18:40:08Z D! [outputs.mqtt] Buffer fullness: 55250 / 1000000 metrics 2024-02-12T18:40:08Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:40:15Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:40:15Z D! [outputs.mqtt] Buffer fullness: 57970 / 1000000 metrics 2024-02-12T18:40:18Z D! [outputs.mqtt] Buffer fullness: 58662 / 1000000 metrics 2024-02-12T18:40:18Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:40:25Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:40:25Z D! [outputs.mqtt] Buffer fullness: 61399 / 1000000 metrics 2024-02-12T18:40:28Z D! [outputs.mqtt] Buffer fullness: 62084 / 1000000 metrics 2024-02-12T18:40:28Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:40:35Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:40:35Z D! [outputs.mqtt] Buffer fullness: 64804 / 1000000 metrics 2024-02-12T18:40:38Z D! [outputs.mqtt] Buffer fullness: 65484 / 1000000 metrics 2024-02-12T18:40:38Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:40:45Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:40:45Z D! [outputs.mqtt] Buffer fullness: 68204 / 1000000 metrics 2024-02-12T18:40:48Z D! [outputs.mqtt] Buffer fullness: 68884 / 1000000 metrics 2024-02-12T18:40:48Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:40:55Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:40:55Z D! [outputs.mqtt] Buffer fullness: 71633 / 1000000 metrics 2024-02-12T18:40:58Z D! [outputs.mqtt] Buffer fullness: 72313 / 1000000 metrics 2024-02-12T18:40:58Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:41:05Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:41:05Z D! [outputs.mqtt] Buffer fullness: 75038 / 1000000 metrics 2024-02-12T18:41:08Z D! [outputs.mqtt] Buffer fullness: 75718 / 1000000 metrics 2024-02-12T18:41:08Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:41:15Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:41:15Z D! [outputs.mqtt] Buffer fullness: 78438 / 1000000 metrics 2024-02-12T18:41:18Z D! [outputs.mqtt] Buffer fullness: 79118 / 1000000 metrics 2024-02-12T18:41:18Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:41:25Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:41:25Z D! [outputs.mqtt] Buffer fullness: 81838 / 1000000 metrics 2024-02-12T18:41:28Z D! [outputs.mqtt] Buffer fullness: 82518 / 1000000 metrics 2024-02-12T18:41:28Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:41:35Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:41:35Z D! [outputs.mqtt] Buffer fullness: 85267 / 1000000 metrics 2024-02-12T18:41:38Z D! [outputs.mqtt] Buffer fullness: 85947 / 1000000 metrics 2024-02-12T18:41:38Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:41:45Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:41:45Z D! [outputs.mqtt] Buffer fullness: 88672 / 1000000 metrics 2024-02-12T18:41:48Z D! [outputs.mqtt] Buffer fullness: 89352 / 1000000 metrics 2024-02-12T18:41:48Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out 2024-02-12T18:41:55Z W! [agent] ["outputs.mqtt"] did not complete within its flush interval 2024-02-12T18:41:55Z D! [outputs.mqtt] Buffer fullness: 92072 / 1000000 metrics 2024-02-12T18:41:58Z D! [outputs.mqtt] Buffer fullness: 92752 / 1000000 metrics 2024-02-12T18:41:58Z E! [agent] Error writing to outputs.mqtt: could not publish message to MQTT server: command timed out