search

sensu / sensu-go-graphite-handler

MIT License
4 stars 5 forks source link

handler errors with failed to validate event #1

Closed rreilly-edr closed 5 years ago

rreilly-edr commented 5 years ago

Hi, Following your example in the readme i have the following handler and check:

{
    "name": "sensu-graphite-handler",
    "type": "pipe",
    "command": "sensu-go-graphite-handler --host tf-p-ubu-graphite-00.xx.com -p 2003",
    "timeout": 0,
    "handlers": null,
    "filters": [
      "has_metrics"
    ],
    "env_vars": null,
    "environment": "default",
    "organization": "default"
  }

and the following metric check

{
    "check_hooks": null,
    "command": "system-profile-linux",
    "env_vars": null,
    "environment": "default",
    "handlers": [],
    "high_flap_threshold": 0,
    "interval": 10,
    "low_flap_threshold": 0,
    "name": "os-metrics",
    "organization": "default",
    "output_metric_format": "influxdb_line",
    "output_metric_handlers": [
      "sensu-graphite-handler"
    ],
    "proxy_entity_id": "",
    "publish": true,
    "round_robin": false,
    "runtime_assets": [
      "system-profile-linux"
    ],
    "stdin": false,
    "subdue": null,
    "subscriptions": [
      "linux"
    ],
    "timeout": 0,
    "ttl": 0,
    "executed": 0,
    "history": null,
    "issued": 0,
    "last_ok": 0,
    "occurrences": 0,
    "occurrences_watermark": 0,
    "output": "",
    "status": 0,
    "total_state_change": 0
  }

I can see the handler being called in the sensu debug log but I get the following error:

{
  "component": "pipelined",
  "environment": "default",
  "handler": "sensu-graphite-handler",
  "level": "info",
  "msg": "event pipe handler executed",
  "organization": "default",
  "output": "Error: failed to validate event: entity is invalid: entity name must not be empty\nUsage:\n  sensu-graphite-handler [flags]\n\nFlags:\n  -h, --help            help for sensu-graphite-handler\n  -H, --host string     the hostname or address of the graphite server (default \"localhost\")\n  -p, --port int        the port number to which to connect on the graphite server (default 2003)\n  -P, --prefix string   the prefix to use in graphite for these metrics (default \"sensu\")\n\nerror: failed to validate event: entity is invalid: entity name must not be empty\n",
  "status": 1,
  "time": "2019-05-09T14:45:31Z"
}
nixwiz commented 5 years ago

Can you send me a sample event which coincides with this error? The simplest way would be to attach a handler similar to the following (it assumes you have jq installed):

{
  "type": "Handler",
  "api_version": "core/v2",
  "metadata": {
    "name": "event-debug",
    "namespace": "default"
  },
  "spec": {
    "command": "/usr/bin/jq . >> /tmp/event-debug.out",
    "env_vars": null,
    "filters": null,
    "handlers": null,
    "runtime_assets": null,
    "timeout": 10,
    "type": "pipe"
  }
}
rreilly-edr commented 5 years ago

Here is the event from the system-profile-linux check.

 "check": {
    "command": "system-profile-linux",
    "environment": "default",
    "handlers": [],
    "high_flap_threshold": 0,
    "interval": 10,
    "low_flap_threshold": 0,
    "name": "os-metrics",
    "organization": "default",
    "publish": true,
    "runtime_assets": [
      "system-profile-linux"
    ],
    "subscriptions": [
      "linux"
    ],
    "proxy_entity_id": "",
    "check_hooks": null,
    "stdin": false,
    "subdue": null,
    "ttl": 0,
    "timeout": 0,
    "round_robin": false,
    "duration": 0.002398484,
    "executed": 1557415546,
    "history": [
      {
        "status": 0,
        "executed": 1557415351
      },
      {
        "status": 0,
        "executed": 1557415361
      },
      {
        "status": 0,
        "executed": 1557415371
      },
      {
        "status": 0,
        "executed": 1557415381
      },
      {
        "status": 0,
        "executed": 1557415391
      },
      {
        "status": 0,
        "executed": 1557415401
      },
      {
        "status": 0,
        "executed": 1557415411
      },
      {
        "status": 0,
        "executed": 1557415421
      },
      {
        "status": 0,
        "executed": 1557415431
      },
      {
        "status": 0,
        "executed": 1557415441
      },
      {
        "status": 0,
        "executed": 1557415451
      },
      {
        "status": 0,
        "executed": 1557415461
      },
      {
        "status": 0,
        "executed": 1557415471
      },
      {
        "status": 0,
        "executed": 1557415476
      },
      {
        "status": 0,
        "executed": 1557415486
      },
      {
        "status": 0,
        "executed": 1557415496
      },
      {
        "status": 0,
        "executed": 1557415506
      },
      {
        "status": 0,
        "executed": 1557415516
      },
      {
        "status": 0,
        "executed": 1557415526
      },
      {
        "status": 0,
        "executed": 1557415536
      },
      {
        "status": 0,
        "executed": 1557415546
      }
    ],
    "issued": 1557415546,
    "output": "cpu.user 43851 1557415546\ncpu.nice 10956 1557415546\ncpu.system 19303 1557415546\ncpu.idle 18221701 1557415546\ncpu.iowait 6922 1557415546\ncpu.irq 0 1557415546\ncpu.softirq 155 1557415546\ncpu.steal 15396 1557415546\ncpu0.user 17057 1557415546\ncpu0.nice 1608 1557415546\ncpu0.system 9114 1557415546\ncpu0.idle 9120829 1557415546\ncpu0.iowait 3051 1557415546\ncpu0.irq 0 1557415546\ncpu0.softirq 34 1557415546\ncpu0.steal 6914 1557415546\ncpu1.user 26794 1557415546\ncpu1.nice 9347 1557415546\ncpu1.system 10189 1557415546\ncpu1.idle 9100872 1557415546\ncpu1.iowait 3870 1557415546\ncpu1.irq 0 1557415546\ncpu1.softirq 121 1557415546\ncpu1.steal 8482 1557415546\nintr 0 1557415546\nctxt 11990700 1557415546\nbtime 1557323994 1557415546\nprocesses 50896 1557415546\nprocs_running 1 1557415546\nprocs_blocked 0 1557415546\nsoftirq 879495 1557415546\nload_avg.1_min 0.37 1557415546\nload_avg.5_min 0.12 1557415546\nload_avg.15_min 0.04 1557415546\nmemory.memtotal 3985344 1557415546\nmemory.memfree 1915952 1557415546\nmemory.memavailable 3418320 1557415546\nmemory.buffers 223316 1557415546\nmemory.cached 1413088 1557415546\nswap.swapcached 0 1557415546\nmemory.active 1119736 1557415546\nswap.swaptotal 0 1557415546\nswap.swapfree 0 1557415546\nmemory.committed_as 653056 1557415546\nnet.face.rxbytes |bytes 1557415546\nnet.face.rxpackets packets 1557415546\nnet.face.rxerrors errs 1557415546\nnet.face.rxdrops drop 1557415546\nnet.face.rxfifo fifo 1557415546\nnet.face.rxframe frame 1557415546\nnet.face.rxcompressed compressed 1557415546\nnet.face.rxmulticast multicast|bytes 1557415546\nnet.face.txbytes packets 1557415546\nnet.face.txpackets errs 1557415546\nnet.face.txerrors drop 1557415546\nnet.face.txdrops fifo 1557415546\nnet.face.txfifo colls 1557415546\nnet.face.txcolls carrier 1557415546\nnet.face.txcarrier compressed 1557415546\nnet.lo.rxbytes 79572517 1557415546\nnet.lo.rxpackets 217832 1557415546\nnet.lo.rxerrors 0 1557415546\nnet.lo.rxdrops 0 1557415546\nnet.lo.rxfifo 0 1557415546\nnet.lo.rxframe 0 1557415546\nnet.lo.rxcompressed 0 1557415546\nnet.lo.rxmulticast 0 1557415546\nnet.lo.txbytes 79572517 1557415546\nnet.lo.txpackets 217832 1557415546\nnet.lo.txerrors 0 1557415546\nnet.lo.txdrops 0 1557415546\nnet.lo.txfifo 0 1557415546\nnet.lo.txcolls 0 1557415546\nnet.lo.txcarrier 0 1557415546\nnet.lo.txcompressed 0 1557415546\nnet.ens5.rxbytes 344145899 1557415546\nnet.ens5.rxpackets 263548 1557415546\nnet.ens5.rxerrors 0 1557415546\nnet.ens5.rxdrops 0 1557415546\nnet.ens5.rxfifo 0 1557415546\nnet.ens5.rxframe 0 1557415546\nnet.ens5.rxcompressed 0 1557415546\nnet.ens5.rxmulticast 0 1557415546\nnet.ens5.txbytes 22613957 1557415546\nnet.ens5.txpackets 76826 1557415546\nnet.ens5.txerrors 0 1557415546\nnet.ens5.txdrops 0 1557415546\nnet.ens5.txfifo 0 1557415546\nnet.ens5.txcolls 0 1557415546\nnet.ens5.txcarrier 0 1557415546\nnet.ens5.txcompressed 0 1557415546\n",
    "state": "passing",
    "status": 0,
    "total_state_change": 0,
    "last_ok": 1557415546,
    "occurrences": 1,
    "occurrences_watermark": 5,
    "output_metric_format": "influxdb_line",
    "output_metric_handlers": [
      "sensu-graphite-handler",
      "handler"
    ],
    "env_vars": null
  },
  "metrics": {
    "handlers": [
      "sensu-graphite-handler",
      "handler"
    ],
    "points": null
  }
}
nixwiz commented 5 years ago

I'm not seeing the metrics in that event. I see the output, but it's not being converted into metrics (the points attribute is null). You specified influxdb_line as the output metrics format, but it appears to actually be in graphite_plaintext format. Can you try changing the check's output_metric_format to graphite_plaintext?

Reference: Checks attributes

rreilly-edr commented 5 years ago

Hi, changed it to graphite_plaintext, same thing i had started with graphite_plaintext but swatiched when i could not figure out what failing.

 "check": {
    "command": "system-profile-linux",
    "environment": "default",
    "handlers": [],
    "high_flap_threshold": 0,
    "interval": 10,
    "low_flap_threshold": 0,
    "name": "os-metrics",
    "organization": "default",
    "publish": true,
    "runtime_assets": [
      "system-profile-linux"
    ],
    "subscriptions": [
      "linux"
    ],
    "proxy_entity_id": "",
    "check_hooks": null,
    "stdin": false,
    "subdue": null,
    "ttl": 0,
    "timeout": 0,
    "round_robin": false,
    "duration": 0.002290721,
    "executed": 1557416482,
    "history": [
      {
        "status": 0,
        "executed": 1557416286
      },
      {
        "status": 0,
        "executed": 1557416296
      },
      {
        "status": 0,
        "executed": 1557416306
      },
      {
        "status": 0,
        "executed": 1557416316
      },
      {
        "status": 0,
        "executed": 1557416326
      },
      {
        "status": 0,
        "executed": 1557416336
      },
      {
        "status": 0,
        "executed": 1557416346
      },
      {
        "status": 0,
        "executed": 1557416356
      },
      {
        "status": 0,
        "executed": 1557416366
      },
      {
        "status": 0,
        "executed": 1557416376
      },
      {
        "status": 0,
        "executed": 1557416386
      },
      {
        "status": 0,
        "executed": 1557416396
      },
      {
        "status": 0,
        "executed": 1557416406
      },
      {
        "status": 0,
        "executed": 1557416416
      },
      {
        "status": 0,
        "executed": 1557416426
      },
      {
        "status": 0,
        "executed": 1557416432
      },
      {
        "status": 0,
        "executed": 1557416442
      },
      {
        "status": 0,
        "executed": 1557416452
      },
      {
        "status": 0,
        "executed": 1557416462
      },
      {
        "status": 0,
        "executed": 1557416472
      },
      {
        "status": 0,
        "executed": 1557416482
      }
    ],
    "issued": 1557416482,
    "output": "cpu.user 46024 1557416482\ncpu.nice 10956 1557416482\ncpu.system 20357 1557416482\ncpu.idle 18406263 1557416482\ncpu.iowait 7154 1557416482\ncpu.irq 0 1557416482\ncpu.softirq 162 1557416482\ncpu.steal 15799 1557416482\ncpu0.user 18013 1557416482\ncpu0.nice 1608 1557416482\ncpu0.system 9626 1557416482\ncpu0.idle 9213279 1557416482\ncpu0.iowait 3172 1557416482\ncpu0.irq 0 1557416482\ncpu0.softirq 36 1557416482\ncpu0.steal 7090 1557416482\ncpu1.user 28010 1557416482\ncpu1.nice 9347 1557416482\ncpu1.system 10730 1557416482\ncpu1.idle 9192984 1557416482\ncpu1.iowait 3982 1557416482\ncpu1.irq 0 1557416482\ncpu1.softirq 126 1557416482\ncpu1.steal 8709 1557416482\nintr 0 1557416482\nctxt 12675811 1557416482\nbtime 1557323994 1557416482\nprocesses 52684 1557416482\nprocs_running 1 1557416482\nprocs_blocked 0 1557416482\nsoftirq 901803 1557416482\nload_avg.1_min 0.00 1557416482\nload_avg.5_min 0.01 1557416482\nload_avg.15_min 0.00 1557416482\nmemory.memtotal 3985344 1557416482\nmemory.memfree 1915560 1557416482\nmemory.memavailable 3421144 1557416482\nmemory.buffers 223980 1557416482\nmemory.cached 1415460 1557416482\nswap.swapcached 0 1557416482\nmemory.active 1119832 1557416482\nswap.swaptotal 0 1557416482\nswap.swapfree 0 1557416482\nmemory.committed_as 647808 1557416482\nnet.face.rxbytes |bytes 1557416482\nnet.face.rxpackets packets 1557416482\nnet.face.rxerrors errs 1557416482\nnet.face.rxdrops drop 1557416482\nnet.face.rxfifo fifo 1557416482\nnet.face.rxframe frame 1557416482\nnet.face.rxcompressed compressed 1557416482\nnet.face.rxmulticast multicast|bytes 1557416482\nnet.face.txbytes packets 1557416482\nnet.face.txpackets errs 1557416482\nnet.face.txerrors drop 1557416482\nnet.face.txdrops fifo 1557416482\nnet.face.txfifo colls 1557416482\nnet.face.txcolls carrier 1557416482\nnet.face.txcarrier compressed 1557416482\nnet.lo.rxbytes 91724676 1557416482\nnet.lo.rxpackets 248756 1557416482\nnet.lo.rxerrors 0 1557416482\nnet.lo.rxdrops 0 1557416482\nnet.lo.rxfifo 0 1557416482\nnet.lo.rxframe 0 1557416482\nnet.lo.rxcompressed 0 1557416482\nnet.lo.rxmulticast 0 1557416482\nnet.lo.txbytes 91724676 1557416482\nnet.lo.txpackets 248756 1557416482\nnet.lo.txerrors 0 1557416482\nnet.lo.txdrops 0 1557416482\nnet.lo.txfifo 0 1557416482\nnet.lo.txcolls 0 1557416482\nnet.lo.txcarrier 0 1557416482\nnet.lo.txcompressed 0 1557416482\nnet.ens5.rxbytes 345640190 1557416482\nnet.ens5.rxpackets 266078 1557416482\nnet.ens5.rxerrors 0 1557416482\nnet.ens5.rxdrops 0 1557416482\nnet.ens5.rxfifo 0 1557416482\nnet.ens5.rxframe 0 1557416482\nnet.ens5.rxcompressed 0 1557416482\nnet.ens5.rxmulticast 0 1557416482\nnet.ens5.txbytes 24325565 1557416482\nnet.ens5.txpackets 79595 1557416482\nnet.ens5.txerrors 0 1557416482\nnet.ens5.txdrops 0 1557416482\nnet.ens5.txfifo 0 1557416482\nnet.ens5.txcolls 0 1557416482\nnet.ens5.txcarrier 0 1557416482\nnet.ens5.txcompressed 0 1557416482\n",
    "state": "passing",
    "status": 0,
    "total_state_change": 0,
    "last_ok": 1557416482,
    "occurrences": 1,
    "occurrences_watermark": 5,
    "output_metric_format": "graphite_plaintext",
    "output_metric_handlers": [
      "sensu-graphite-handler",
      "handler"
    ],
    "env_vars": null
  },
  "metrics": {
    "handlers": [
      "sensu-graphite-handler",
      "handler"
    ],
    "points": null
  }
rreilly-edr commented 5 years ago

2.0.0~nightly+20181011-1 is the version of sensu

nixwiz commented 5 years ago

Ouch, I've not tested on anything that old, that's back during the beta cycles. I'm sure the event definitions have changed since then.

My only advice at this point is to upgrade to a 5.x released version.

rreilly-edr commented 5 years ago

Yea, that is old, i assumed chef was installing the latest.. ok i will close upgrade and restest. thanks for the help