Open root360-StefanHeitmueller opened 3 years ago
Agent status in dashboard reports as late upload
https://support.cloudhealthtech.com/hc/en-us/articles/360015713131-The-CloudHealth-Agent-has-a-status-of-Late-upload-What-does-this-mean- mentions to check /opt/cht_perfmon/cht_perfmon_collector.output
which does not exist on the system.
ls -l /var/lib/chtcollectd/*
does show active metrics collected by collectd:
root@i-xxx:~# ls -l /var/lib/chtcollectd/*
total 80
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 cpu-average
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-dev
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-dev-shm
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-root
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-lock
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2002
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2003
drwxrwsr-x 2 root cht_agent 4096 Nov 27 07:56 df-run-user-2007
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-user-2013
drwxrwsr-x 2 root cht_agent 4096 Nov 25 06:25 df-run-user-2203
drwxr-sr-x 2 root cht_agent 4096 Nov 27 07:56 df-run-user-2207
drwxr-sr-x 2 root cht_agent 4096 Nov 26 06:25 df-run-user-2213
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-run-user-3001
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-sys-fs-cgroup
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 df-var-log-remote
drwxr-sr-x 2 root cht_agent 4096 Nov 27 10:00 interface-ens5
drwxr-sr-x 2 root cht_agent 4096 Nov 27 10:00 interface-lo
drwxrwsr-x 2 root cht_agent 4096 Nov 27 06:25 memory
Ah, okay, got another instance which is sending data, but to another path at https://api.cloudhealthtech.com/v1/agent/upload
with a JSON structure like
{
"instance": "i-xxx",
"namespace": "CloudHealth/Perfmon",
"cloud_name": "aws",
"start_time": "2020-11-27T10:00:00Z",
"stop_time": "2020-11-27T11:00:00Z",
"facter": null,
"perf_data": [
{
"monitor_name": "df",
"value": {
"/etc/hosts": {
"sample_specific_stats": {
"fs_src": "/dev/nvme0n1p1",
"fs_type": "ext4",
"fs_options": "rw,relatime,discard"
},
"used": {
"min": 4373340160,
"max": 4374106112,
"min_perc": 21.076533531943472,
"max_perc": 21.080224901107826,
"avg": 4373716718.933333,
"avg_perc": 21.07834829061642
},
"free": {
"min": 16375701504,
"max": 16376467456,
"min_perc": 78.91977509889217,
"max_perc": 78.92346646805653,
"avg": 16376090897.066675,
"avg_perc": 78.92165170938362
}
},
"/": {
"sample_specific_stats": {
"fs_src": "overlay",
"fs_type": "overlay",
"fs_options": "rw,relatime,lowerdir=/var/lib/docker/overlay2/l/BGPWVDQKFNCX6MCF52TUKXXI4C:/var/lib/docker/overlay2/l/37CWUGOOKJMVFCT4LZWOG3UX6R:/var/lib/docker/overlay2/l/4KAVPSAZWW3MUFLXSVWFT2O5AM:/var/lib/docker/overlay2/l/LAJUGTRHGVXWM4YNI7I465F3TK:/var/lib/docker/overlay2/l/V663TUYTSHF7E4FOZQIGIYU5MY:/var/lib/docker/overlay2/l/PAZYZAK4XNQJ3GY7525HSUZZOF:/var/lib/docker/overlay2/l/DNPA4DSYTZJAXO6ABVGSSXBVQA:/var/lib/docker/overlay2/l/AHXBYO6TZTCGU6J2OIZ23EYJA6,upperdir=/var/lib/docker/overlay2/ca900efa9b02b89c277afe0fa6f87b427b2dc1d8132c83c4be490ea789c15ce1/diff,workdir=/var/lib/docker/overlay2/ca900efa9b02b89c277afe0fa6f87b427b2dc1d8132c83c4be490ea789c15ce1/work,xino=off"
},
"used": {
"min": 4373340160,
"max": 4374106112,
"min_perc": 21.076533531943472,
"max_perc": 21.080224901107826,
"avg": 4373716377.599999,
"avg_perc": 21.07834664562125
},
"free": {
"min": 16375701504,
"max": 16376467456,
"min_perc": 78.91977509889217,
"max_perc": 78.92346646805653,
"avg": 16376091238.40001,
"avg_perc": 78.92165335437879
}
},
"/etc/hostname": {
"sample_specific_stats": {
"fs_src": "/dev/nvme0n1p1",
"fs_type": "ext4",
"fs_options": "rw,relatime,discard"
},
"used": {
"min": 4373340160,
"max": 4374106112,
"min_perc": 21.076533531943472,
"max_perc": 21.080224901107826,
"avg": 4373716718.933333,
"avg_perc": 21.07834829061642
},
"free": {
"min": 16375701504,
"max": 16376467456,
"min_perc": 78.91977509889217,
"max_perc": 78.92346646805653,
"avg": 16376090897.066675,
"avg_perc": 78.92165170938362
}
},
"/etc/resolv.conf": {
"sample_specific_stats": {
"fs_src": "/dev/nvme0n1p1",
"fs_type": "ext4",
"fs_options": "rw,relatime,discard"
},
"used": {
"min": 4373340160,
"max": 4374106112,
"min_perc": 21.076533531943472,
"max_perc": 21.080224901107826,
"avg": 4373716718.933333,
"avg_perc": 21.07834829061642
},
"free": {
"min": 16375701504,
"max": 16376467456,
"min_perc": 78.91977509889217,
"max_perc": 78.92346646805653,
"avg": 16376090897.066675,
"avg_perc": 78.92165170938362
}
}
}
},
{
"monitor_name": "cpu",
"value": {
"average": {
"sample_specific_stats": {},
"unused": {
"min": 96.165178,
"max": 98.01659099999999,
"min_perc": 96.56903840984343,
"max_perc": 98.37711159237382,
"avg": 97.7576948392857,
"avg_perc": 98.10920257351773
},
"used": {
"min": 1.616666,
"max": 3.416613,
"min_perc": 1.6228884076261711,
"max_perc": 3.4309615901565778,
"avg": 1.8839269642857142,
"avg_perc": 1.890797426482287
}
}
}
},
{
"monitor_name": "memory",
"value": {
"memory": {
"sample_specific_stats": {},
"used": {
"min": 1085796352,
"max": 1212633088,
"min_perc": 53.105129674221814,
"max_perc": 59.30857777047683,
"avg": 1134323097.6,
"avg_perc": 55.47852051589226
},
"free": {
"min": 831983616,
"max": 958820352,
"min_perc": 40.69142222952317,
"max_perc": 46.894870325778186,
"avg": 910293606.4000001,
"avg_perc": 44.52147948410776
}
}
}
},
{
"monitor_name": "interface",
"value": {
"eth0": {
"sample_specific_stats": {},
"packets_rx": {
"min": 0,
"max": 2.799998,
"min_perc": 0,
"max_perc": 1.4084501091055877,
"avg": 0.0736119888888889,
"avg_perc": 0.017528217636934594,
"sum": 265.00316
},
"packets_tx": {
"min": 0,
"max": 3.199998,
"min_perc": 0,
"max_perc": 0.5591795690015908,
"avg": 0.08222320277777777,
"avg_perc": 0.015628157848361808,
"sum": 296.00353
},
"octets_rx": {
"min": 0,
"max": 757.599152,
"min_perc": 0,
"max_perc": 98.59154989089441,
"avg": 20.120893205555554,
"avg_perc": 3.5463880319384504,
"sum": 72435.21554
},
"octets_tx": {
"min": 0,
"max": 396.499556,
"min_perc": 0,
"max_perc": 38.676608014737276,
"avg": 10.272531488888887,
"avg_perc": 1.6982333703540307,
"sum": 36981.11336
}
}
}
},
{
"monitor_name": "disk",
"value": {}
}
],
"command_data": []
}
Operating System
Ubuntu Bionic Beaver 18.04
Installation
sh install_cht_perfmon.sh 20 <key> aws disable-update
Result
perfmon collector
I've tapped the instance with mitmproxy to see why metrics are not being sent to
https://chapi.cloudhealthtech.com/metrics/v1
(according to the docs)There are no requests to the endpoint: