search

CloudHealth / agent-install-helpers

6 stars 3 forks source link

ruby perfmon collector does not send metrics #9

Open root360-StefanHeitmueller opened 3 years ago

root360-StefanHeitmueller commented 3 years ago

Operating System

Ubuntu Bionic Beaver 18.04

Installation

sh install_cht_perfmon.sh 20 <key> aws disable-update

Result

perfmon collector

I've tapped the instance with mitmproxy to see why metrics are not being sent to https://chapi.cloudhealthtech.com/metrics/v1 (according to the docs)

There are no requests to the endpoint:

Screenshot from 2020-11-27 10-18-39

root360-StefanHeitmueller commented 3 years ago

Agent status in dashboard reports as late upload

Screenshot from 2020-11-27 11-05-20

root360-StefanHeitmueller commented 3 years ago

https://support.cloudhealthtech.com/hc/en-us/articles/360015713131-The-CloudHealth-Agent-has-a-status-of-Late-upload-What-does-this-mean- mentions to check /opt/cht_perfmon/cht_perfmon_collector.output which does not exist on the system.

root360-StefanHeitmueller commented 3 years ago

ls -l /var/lib/chtcollectd/* does show active metrics collected by collectd:

root@i-xxx:~# ls -l /var/lib/chtcollectd/*
total 80
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 cpu-average
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-dev
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-dev-shm
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-root
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-lock
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2002
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2003
drwxrwsr-x 2 root cht_agent 4096 Nov 27 07:56 df-run-user-2007
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-user-2013
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2203
drwxr-sr-x 2 root cht_agent 4096 Nov 27 07:56 df-run-user-2207
drwxr-sr-x 2 root cht_agent 4096 Nov 26 06:25 df-run-user-2213
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-user-3001
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-sys-fs-cgroup
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-var-log-remote
drwxr-sr-x 2 root cht_agent 4096 Nov 27 10:00 interface-ens5
drwxr-sr-x 2 root cht_agent 4096 Nov 27 10:00 interface-lo
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 memory
root360-StefanHeitmueller commented 3 years ago

error

root360-StefanHeitmueller commented 3 years ago

Ah, okay, got another instance which is sending data, but to another path at https://api.cloudhealthtech.com/v1/agent/upload with a JSON structure like

{
  "instance": "i-xxx",
  "namespace": "CloudHealth/Perfmon",
  "cloud_name": "aws",
  "start_time": "2020-11-27T10:00:00Z",
  "stop_time": "2020-11-27T11:00:00Z",
  "facter": null,
  "perf_data": [
    {
      "monitor_name": "df",
      "value": {
        "/etc/hosts": {
          "sample_specific_stats": {
            "fs_src": "/dev/nvme0n1p1",
            "fs_type": "ext4",
            "fs_options": "rw,relatime,discard"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716718.933333,
            "avg_perc": 21.07834829061642
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376090897.066675,
            "avg_perc": 78.92165170938362
          }
        },
        "/": {
          "sample_specific_stats": {
            "fs_src": "overlay",
            "fs_type": "overlay",
            "fs_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/BGPWVDQKFNCX6MCF52TUKXXI4C:/var/lib/docker/overlay2/l/37CWUGOOKJMVFCT4LZWOG3UX6R:/var/lib/docker/overlay2/l/4KAVPSAZWW3MUFLXSVWFT2O5AM:/var/lib/docker/overlay2/l/LAJUGTRHGVXWM4YNI7I465F3TK:/var/lib/docker/overlay2/l/V663TUYTSHF7E4FOZQIGIYU5MY:/var/lib/docker/overlay2/l/PAZYZAK4XNQJ3GY7525HSUZZOF:/var/lib/docker/overlay2/l/DNPA4DSYTZJAXO6ABVGSSXBVQA:/var/lib/docker/overlay2/l/AHXBYO6TZTCGU6J2OIZ23EYJA6,upperdir=/var/lib/docker/overlay2/ca900efa9b02b89c277afe0fa6f87b427b2dc1d8132c83c4be490ea789c15ce1/diff,workdir=/var/lib/docker/overlay2/ca900efa9b02b89c277afe0fa6f87b427b2dc1d8132c83c4be490ea789c15ce1/work,xino=off"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716377.599999,
            "avg_perc": 21.07834664562125
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376091238.40001,
            "avg_perc": 78.92165335437879
          }
        },
        "/etc/hostname": {
          "sample_specific_stats": {
            "fs_src": "/dev/nvme0n1p1",
            "fs_type": "ext4",
            "fs_options": "rw,relatime,discard"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716718.933333,
            "avg_perc": 21.07834829061642
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376090897.066675,
            "avg_perc": 78.92165170938362
          }
        },
        "/etc/resolv.conf": {
          "sample_specific_stats": {
            "fs_src": "/dev/nvme0n1p1",
            "fs_type": "ext4",
            "fs_options": "rw,relatime,discard"
          },
          "used": {
            "min": 4373340160,
            "max": 4374106112,
            "min_perc": 21.076533531943472,
            "max_perc": 21.080224901107826,
            "avg": 4373716718.933333,
            "avg_perc": 21.07834829061642
          },
          "free": {
            "min": 16375701504,
            "max": 16376467456,
            "min_perc": 78.91977509889217,
            "max_perc": 78.92346646805653,
            "avg": 16376090897.066675,
            "avg_perc": 78.92165170938362
          }
        }
      }
    },
    {
      "monitor_name": "cpu",
      "value": {
        "average": {
          "sample_specific_stats": {},
          "unused": {
            "min": 96.165178,
            "max": 98.01659099999999,
            "min_perc": 96.56903840984343,
            "max_perc": 98.37711159237382,
            "avg": 97.7576948392857,
            "avg_perc": 98.10920257351773
          },
          "used": {
            "min": 1.616666,
            "max": 3.416613,
            "min_perc": 1.6228884076261711,
            "max_perc": 3.4309615901565778,
            "avg": 1.8839269642857142,
            "avg_perc": 1.890797426482287
          }
        }
      }
    },
    {
      "monitor_name": "memory",
      "value": {
        "memory": {
          "sample_specific_stats": {},
          "used": {
            "min": 1085796352,
            "max": 1212633088,
            "min_perc": 53.105129674221814,
            "max_perc": 59.30857777047683,
            "avg": 1134323097.6,
            "avg_perc": 55.47852051589226
          },
          "free": {
            "min": 831983616,
            "max": 958820352,
            "min_perc": 40.69142222952317,
            "max_perc": 46.894870325778186,
            "avg": 910293606.4000001,
            "avg_perc": 44.52147948410776
          }
        }
      }
    },
    {
      "monitor_name": "interface",
      "value": {
        "eth0": {
          "sample_specific_stats": {},
          "packets_rx": {
            "min": 0,
            "max": 2.799998,
            "min_perc": 0,
            "max_perc": 1.4084501091055877,
            "avg": 0.0736119888888889,
            "avg_perc": 0.017528217636934594,
            "sum": 265.00316
          },
          "packets_tx": {
            "min": 0,
            "max": 3.199998,
            "min_perc": 0,
            "max_perc": 0.5591795690015908,
            "avg": 0.08222320277777777,
            "avg_perc": 0.015628157848361808,
            "sum": 296.00353
          },
          "octets_rx": {
            "min": 0,
            "max": 757.599152,
            "min_perc": 0,
            "max_perc": 98.59154989089441,
            "avg": 20.120893205555554,
            "avg_perc": 3.5463880319384504,
            "sum": 72435.21554
          },
          "octets_tx": {
            "min": 0,
            "max": 396.499556,
            "min_perc": 0,
            "max_perc": 38.676608014737276,
            "avg": 10.272531488888887,
            "avg_perc": 1.6982333703540307,
            "sum": 36981.11336
          }
        }
      }
    },
    {
      "monitor_name": "disk",
      "value": {}
    }
  ],
  "command_data": []
}