search

prometheus / node_exporter

Exporter for machine metrics
https://prometheus.io/
Apache License 2.0
10.97k stars 2.33k forks source link

NVMesh storage support for constituent disks #1981

Closed aned closed 3 years ago

aned commented 3 years ago

node exporter is covering the volumes already, but not the constituent disks.

There is this NVMesh storage plugin: https://github.com/Excelero/telegraf-plugin, nvmesh_telegraf.py produces:

$ ./nvmesh_telegraf.py
nvmesh,volume=test num_ops_read=75249951,num_ops_write=83687437,size_in_bytes_read=1225626472448,size_in_bytes_write=1002465906688,latency_read=25601078223.2,latency_write=63258875663.4
nvmesh,volume=test2 num_ops_read=346231706,num_ops_write=1395256,size_in_bytes_read=5310346731520,size_in_bytes_write=365757988864,latency_read=320947650638.4,latency_write=63905189848.8
nvmesh,volume=Nag_test num_ops_read=6366,num_ops_write=0,size_in_bytes_read=157016064,size_in_bytes_write=0,latency_read=428306.6,latency_write=0.0
nvmesh,disk=BTLE8166092N3P2EGN.1 disk_reads=49347192,disk_writes=43596683,disk_read_in_bytes=748975898624,disk_write_in_bytes=562281676800,disk_read_latency=153075838.8,disk_write_latency=333881038.8
nvmesh,disk=BTLE817409MY3P2EGN.1 disk_reads=29652252,disk_writes=464731,disk_read_in_bytes=442878586880,disk_write_in_bytes=60913221632,disk_read_latency=241275435.0,disk_write_latency=63152343.1
nvmesh,disk=BTLE817409Q03P2EGN.1 disk_reads=29642279,disk_writes=464731,disk_read_in_bytes=442334400512,disk_write_in_bytes=60913221632,disk_read_latency=164453305.2,disk_write_latency=65486267.9
nvmesh,disk=BTLE81740CA23P2EGN.1 disk_reads=29634431,disk_writes=465050,disk_read_in_bytes=442228936704,disk_write_in_bytes=60955033600,disk_read_latency=155779799.9,disk_write_latency=67561272.0
nvmesh,disk=BTLE81740CZW3P2EGN.1 disk_reads=29672130,disk_writes=464731,disk_read_in_bytes=442898657280,disk_write_in_bytes=60913221632,disk_read_latency=205215430.1,disk_write_latency=66256638.3
nvmesh,disk=BTLE818609KC3P2EGN.1 disk_reads=29641571,disk_writes=464731,disk_read_in_bytes=442261843968,disk_write_in_bytes=60913221632,disk_read_latency=141803706.0,disk_write_latency=67331486.1
nvmesh,disk=PHLE7264009F3P2EGN.1 disk_reads=49330849,disk_writes=43578136,disk_read_in_bytes=748842262528,disk_write_in_bytes=562205708288,disk_read_latency=213710160.3,disk_write_latency=264849440.4
nvmesh,disk=PHLE726400LT3P2EGN.1 disk_reads=29629269,disk_writes=465050,disk_read_in_bytes=442200788992,disk_write_in_bytes=60955033600,disk_read_latency=203209332.5,disk_write_latency=71202331.4
nvmesh,disk=PHLE726400NS3P2EGN.1 disk_reads=49355317,disk_writes=43596683,disk_read_in_bytes=748957933568,disk_write_in_bytes=562281676800,disk_read_latency=187940639.0,disk_write_latency=294326650.6
nvmesh,disk=PHLE726401R53P2EGN.1 disk_reads=29655380,disk_writes=465050,disk_read_in_bytes=442890465280,disk_write_in_bytes=60955033600,disk_read_latency=130508963.6,disk_write_latency=68992168.0
nvmesh,disk=PHLE8221026N3P2EGN.1 disk_reads=29659872,disk_writes=465050,disk_read_in_bytes=442823344128,disk_write_in_bytes=60955033600,disk_read_latency=174305262.6,disk_write_latency=69439292.8
nvmesh,disk=PHLE822102PK3P2EGN.1 disk_reads=49342549,disk_writes=43578136,disk_read_in_bytes=748885749760,disk_write_in_bytes=562205708288,disk_read_latency=255384659.0,disk_write_latency=268322146.2

"nvmesh telegraph plugin" - "telegraph" - "influxdb" - "influxdb prometheus.client" produces:

 curl localhost:9373/metrics

# HELP nvmesh_disk_read_in_bytes Telegraf collected metric
# TYPE nvmesh_disk_read_in_bytes untyped
nvmesh_disk_read_in_bytes{disk="BTLE8166092N3P2EGN.1",host="host.com"} 7.48975898624e+11
nvmesh_disk_read_in_bytes{disk="BTLE817409MY3P2EGN.1",host="host.com"} 4.4287858688e+11
nvmesh_disk_read_in_bytes{disk="BTLE817409Q03P2EGN.1",host="host.com"} 4.42334400512e+11
nvmesh_disk_read_in_bytes{disk="BTLE81740CA23P2EGN.1",host="host.com"} 4.42228936704e+11
nvmesh_disk_read_in_bytes{disk="BTLE81740CZW3P2EGN.1",host="host.com"} 4.4289865728e+11
nvmesh_disk_read_in_bytes{disk="BTLE818609KC3P2EGN.1",host="host.com"} 4.42261843968e+11
nvmesh_disk_read_in_bytes{disk="PHLE7264009F3P2EGN.1",host="host.com"} 7.48842262528e+11
nvmesh_disk_read_in_bytes{disk="PHLE726400LT3P2EGN.1",host="host.com"} 4.42200788992e+11
nvmesh_disk_read_in_bytes{disk="PHLE726400NS3P2EGN.1",host="host.com"} 7.48957933568e+11
nvmesh_disk_read_in_bytes{disk="PHLE726401R53P2EGN.1",host="host.com"} 4.4289046528e+11
nvmesh_disk_read_in_bytes{disk="PHLE8221026N3P2EGN.1",host="host.com"} 4.42823344128e+11
nvmesh_disk_read_in_bytes{disk="PHLE822102PK3P2EGN.1",host="host.com"} 7.4888574976e+11
# HELP nvmesh_disk_read_latency Telegraf collected metric
# TYPE nvmesh_disk_read_latency untyped
nvmesh_disk_read_latency{disk="BTLE8166092N3P2EGN.1",host="host.com"} 1.530758388e+08
nvmesh_disk_read_latency{disk="BTLE817409MY3P2EGN.1",host="host.com"} 2.41275435e+08
nvmesh_disk_read_latency{disk="BTLE817409Q03P2EGN.1",host="host.com"} 1.644533052e+08
nvmesh_disk_read_latency{disk="BTLE81740CA23P2EGN.1",host="host.com"} 1.557797999e+08
nvmesh_disk_read_latency{disk="BTLE81740CZW3P2EGN.1",host="host.com"} 2.052154301e+08
nvmesh_disk_read_latency{disk="BTLE818609KC3P2EGN.1",host="host.com"} 1.41803706e+08
nvmesh_disk_read_latency{disk="PHLE7264009F3P2EGN.1",host="host.com"} 2.137101603e+08
nvmesh_disk_read_latency{disk="PHLE726400LT3P2EGN.1",host="host.com"} 2.032093325e+08
nvmesh_disk_read_latency{disk="PHLE726400NS3P2EGN.1",host="host.com"} 1.87940639e+08
nvmesh_disk_read_latency{disk="PHLE726401R53P2EGN.1",host="host.com"} 1.305089636e+08
nvmesh_disk_read_latency{disk="PHLE8221026N3P2EGN.1",host="host.com"} 1.743052626e+08
nvmesh_disk_read_latency{disk="PHLE822102PK3P2EGN.1",host="host.com"} 2.55384659e+08
# HELP nvmesh_disk_reads Telegraf collected metric
# TYPE nvmesh_disk_reads untyped
nvmesh_disk_reads{disk="BTLE8166092N3P2EGN.1",host="host.com"} 4.9347192e+07
nvmesh_disk_reads{disk="BTLE817409MY3P2EGN.1",host="host.com"} 2.9652252e+07
nvmesh_disk_reads{disk="BTLE817409Q03P2EGN.1",host="host.com"} 2.9642279e+07
nvmesh_disk_reads{disk="BTLE81740CA23P2EGN.1",host="host.com"} 2.9634431e+07
nvmesh_disk_reads{disk="BTLE81740CZW3P2EGN.1",host="host.com"} 2.967213e+07
nvmesh_disk_reads{disk="BTLE818609KC3P2EGN.1",host="host.com"} 2.9641571e+07
nvmesh_disk_reads{disk="PHLE7264009F3P2EGN.1",host="host.com"} 4.9330849e+07
nvmesh_disk_reads{disk="PHLE726400LT3P2EGN.1",host="host.com"} 2.9629269e+07
nvmesh_disk_reads{disk="PHLE726400NS3P2EGN.1",host="host.com"} 4.9355317e+07
nvmesh_disk_reads{disk="PHLE726401R53P2EGN.1",host="host.com"} 2.965538e+07
nvmesh_disk_reads{disk="PHLE8221026N3P2EGN.1",host="host.com"} 2.9659872e+07
nvmesh_disk_reads{disk="PHLE822102PK3P2EGN.1",host="host.com"} 4.9342549e+07
# HELP nvmesh_disk_write_in_bytes Telegraf collected metric
# TYPE nvmesh_disk_write_in_bytes untyped
nvmesh_disk_write_in_bytes{disk="BTLE8166092N3P2EGN.1",host="host.com"} 5.622816768e+11
nvmesh_disk_write_in_bytes{disk="BTLE817409MY3P2EGN.1",host="host.com"} 6.0913221632e+10
nvmesh_disk_write_in_bytes{disk="BTLE817409Q03P2EGN.1",host="host.com"} 6.0913221632e+10
nvmesh_disk_write_in_bytes{disk="BTLE81740CA23P2EGN.1",host="host.com"} 6.09550336e+10
nvmesh_disk_write_in_bytes{disk="BTLE81740CZW3P2EGN.1",host="host.com"} 6.0913221632e+10
nvmesh_disk_write_in_bytes{disk="BTLE818609KC3P2EGN.1",host="host.com"} 6.0913221632e+10
nvmesh_disk_write_in_bytes{disk="PHLE7264009F3P2EGN.1",host="host.com"} 5.62205708288e+11
nvmesh_disk_write_in_bytes{disk="PHLE726400LT3P2EGN.1",host="host.com"} 6.09550336e+10
nvmesh_disk_write_in_bytes{disk="PHLE726400NS3P2EGN.1",host="host.com"} 5.622816768e+11
nvmesh_disk_write_in_bytes{disk="PHLE726401R53P2EGN.1",host="host.com"} 6.09550336e+10
nvmesh_disk_write_in_bytes{disk="PHLE8221026N3P2EGN.1",host="host.com"} 6.09550336e+10
nvmesh_disk_write_in_bytes{disk="PHLE822102PK3P2EGN.1",host="host.com"} 5.62205708288e+11
# HELP nvmesh_disk_write_latency Telegraf collected metric
# TYPE nvmesh_disk_write_latency untyped
nvmesh_disk_write_latency{disk="BTLE8166092N3P2EGN.1",host="host.com"} 3.338810388e+08
nvmesh_disk_write_latency{disk="BTLE817409MY3P2EGN.1",host="host.com"} 6.31523431e+07
nvmesh_disk_write_latency{disk="BTLE817409Q03P2EGN.1",host="host.com"} 6.54862679e+07
nvmesh_disk_write_latency{disk="BTLE81740CA23P2EGN.1",host="host.com"} 6.7561272e+07
nvmesh_disk_write_latency{disk="BTLE81740CZW3P2EGN.1",host="host.com"} 6.62566383e+07
nvmesh_disk_write_latency{disk="BTLE818609KC3P2EGN.1",host="host.com"} 6.73314861e+07
nvmesh_disk_write_latency{disk="PHLE7264009F3P2EGN.1",host="host.com"} 2.648494404e+08
nvmesh_disk_write_latency{disk="PHLE726400LT3P2EGN.1",host="host.com"} 7.12023314e+07
nvmesh_disk_write_latency{disk="PHLE726400NS3P2EGN.1",host="host.com"} 2.943266506e+08
nvmesh_disk_write_latency{disk="PHLE726401R53P2EGN.1",host="host.com"} 6.8992168e+07
nvmesh_disk_write_latency{disk="PHLE8221026N3P2EGN.1",host="host.com"} 6.94392928e+07
nvmesh_disk_write_latency{disk="PHLE822102PK3P2EGN.1",host="host.com"} 2.683221462e+08
# HELP nvmesh_disk_writes Telegraf collected metric
# TYPE nvmesh_disk_writes untyped
nvmesh_disk_writes{disk="BTLE8166092N3P2EGN.1",host="host.com"} 4.3596683e+07
nvmesh_disk_writes{disk="BTLE817409MY3P2EGN.1",host="host.com"} 464731
nvmesh_disk_writes{disk="BTLE817409Q03P2EGN.1",host="host.com"} 464731
nvmesh_disk_writes{disk="BTLE81740CA23P2EGN.1",host="host.com"} 465050
nvmesh_disk_writes{disk="BTLE81740CZW3P2EGN.1",host="host.com"} 464731
nvmesh_disk_writes{disk="BTLE818609KC3P2EGN.1",host="host.com"} 464731
nvmesh_disk_writes{disk="PHLE7264009F3P2EGN.1",host="host.com"} 4.3578136e+07
nvmesh_disk_writes{disk="PHLE726400LT3P2EGN.1",host="host.com"} 465050
nvmesh_disk_writes{disk="PHLE726400NS3P2EGN.1",host="host.com"} 4.3596683e+07
nvmesh_disk_writes{disk="PHLE726401R53P2EGN.1",host="host.com"} 465050
nvmesh_disk_writes{disk="PHLE8221026N3P2EGN.1",host="host.com"} 465050
nvmesh_disk_writes{disk="PHLE822102PK3P2EGN.1",host="host.com"} 4.3578136e+07
# HELP nvmesh_latency_read Telegraf collected metric
# TYPE nvmesh_latency_read untyped
nvmesh_latency_read{host="host.com",volume="Nag_test"} 428306.6
nvmesh_latency_read{host="host.com",volume="test"} 2.56010782232e+10
nvmesh_latency_read{host="host.com",volume="test2"} 3.209476506384e+11
# HELP nvmesh_latency_write Telegraf collected metric
# TYPE nvmesh_latency_write untyped
nvmesh_latency_write{host="host.com",volume="Nag_test"} 0
nvmesh_latency_write{host="host.com",volume="test"} 6.32588756634e+10
nvmesh_latency_write{host="host.com",volume="test2"} 6.39051898488e+10
# HELP nvmesh_num_ops_read Telegraf collected metric
# TYPE nvmesh_num_ops_read untyped
nvmesh_num_ops_read{host="host.com",volume="Nag_test"} 6366
nvmesh_num_ops_read{host="host.com",volume="test"} 7.5249951e+07
nvmesh_num_ops_read{host="host.com",volume="test2"} 3.46231706e+08
# HELP nvmesh_num_ops_write Telegraf collected metric
# TYPE nvmesh_num_ops_write untyped
nvmesh_num_ops_write{host="host.com",volume="Nag_test"} 0
nvmesh_num_ops_write{host="host.com",volume="test"} 8.3687437e+07
nvmesh_num_ops_write{host="host.com",volume="test2"} 1.395256e+06
# HELP nvmesh_size_in_bytes_read Telegraf collected metric
# TYPE nvmesh_size_in_bytes_read untyped
nvmesh_size_in_bytes_read{host="host.com",volume="Nag_test"} 1.57016064e+08
nvmesh_size_in_bytes_read{host="host.com",volume="test"} 1.225626472448e+12
nvmesh_size_in_bytes_read{host="host.com",volume="test2"} 5.31034673152e+12
# HELP nvmesh_size_in_bytes_write Telegraf collected metric
# TYPE nvmesh_size_in_bytes_write untyped
nvmesh_size_in_bytes_write{host="host.com",volume="Nag_test"} 0
nvmesh_size_in_bytes_write{host="host.com",volume="test"} 1.002465906688e+12
nvmesh_size_in_bytes_write{host="host.com",volume="test2"} 3.65757988864e+11

Current node_exporter output: 

curl localhost:9100/metrics | grep disk

# TYPE node_disk_io_now gauge
node_disk_io_now{device="md0"} 0
node_disk_io_now{device="md1"} 0
node_disk_io_now{device="md2"} 0
node_disk_io_now{device="md3"} 0
node_disk_io_now{device="nvme0n1"} 0
node_disk_io_now{device="nvmesh/Nag_test"} 0
node_disk_io_now{device="nvmesh/test"} 41
node_disk_io_now{device="nvmesh/test2"} 0
node_disk_io_now{device="sda"} 0
node_disk_io_now{device="sdb"} 0
# HELP node_disk_io_time_seconds_total Total seconds spent doing I/Os.
# TYPE node_disk_io_time_seconds_total counter
node_disk_io_time_seconds_total{device="md0"} 0
node_disk_io_time_seconds_total{device="md1"} 0
node_disk_io_time_seconds_total{device="md2"} 0
node_disk_io_time_seconds_total{device="md3"} 0
node_disk_io_time_seconds_total{device="nvme0n1"} 6.78
node_disk_io_time_seconds_total{device="nvmesh/Nag_test"} 0
node_disk_io_time_seconds_total{device="nvmesh/test"} 2022.5130000000001
node_disk_io_time_seconds_total{device="nvmesh/test2"} 3749.607
node_disk_io_time_seconds_total{device="sda"} 260566.845
node_disk_io_time_seconds_total{device="sdb"} 262089.193
# HELP node_disk_io_time_weighted_seconds_total The weighted # of seconds spent doing I/Os.
# TYPE node_disk_io_time_weighted_seconds_total counter
node_disk_io_time_weighted_seconds_total{device="md0"} 0
node_disk_io_time_weighted_seconds_total{device="md1"} 0
node_disk_io_time_weighted_seconds_total{device="md2"} 0
node_disk_io_time_weighted_seconds_total{device="md3"} 0
node_disk_io_time_weighted_seconds_total{device="nvme0n1"} 8.709
node_disk_io_time_weighted_seconds_total{device="nvmesh/Nag_test"} 0
node_disk_io_time_weighted_seconds_total{device="nvmesh/test"} 98566.431
node_disk_io_time_weighted_seconds_total{device="nvmesh/test2"} 391921.234
node_disk_io_time_weighted_seconds_total{device="sda"} 406397
node_disk_io_time_weighted_seconds_total{device="sdb"} 411935.564
# HELP node_disk_read_bytes_total The total number of bytes read successfully.
# TYPE node_disk_read_bytes_total counter
node_disk_read_bytes_total{device="md0"} 7.54025472e+08
node_disk_read_bytes_total{device="md1"} 8.3513856e+08
node_disk_read_bytes_total{device="md2"} 3.575808e+06
node_disk_read_bytes_total{device="md3"} 5.18864896e+09
node_disk_read_bytes_total{device="nvme0n1"} 2.984247296e+09
node_disk_read_bytes_total{device="nvmesh/Nag_test"} 1.6125952e+08
node_disk_read_bytes_total{device="nvmesh/test"} 1.634600251392e+12
node_disk_read_bytes_total{device="nvmesh/test2"} 5.310350974976e+12
node_disk_read_bytes_total{device="sda"} 5.889173504e+09
node_disk_read_bytes_total{device="sdb"} 1.223876608e+09
# HELP node_disk_read_time_seconds_total The total number of seconds spent by all reads.
# TYPE node_disk_read_time_seconds_total counter
node_disk_read_time_seconds_total{device="md0"} 0
node_disk_read_time_seconds_total{device="md1"} 0
node_disk_read_time_seconds_total{device="md2"} 0
node_disk_read_time_seconds_total{device="md3"} 0
node_disk_read_time_seconds_total{device="nvme0n1"} 8.715
node_disk_read_time_seconds_total{device="nvmesh/Nag_test"} 0.448
node_disk_read_time_seconds_total{device="nvmesh/test"} 32228.032
node_disk_read_time_seconds_total{device="nvmesh/test2"} 327546.67600000004
node_disk_read_time_seconds_total{device="sda"} 733.139
node_disk_read_time_seconds_total{device="sdb"} 435.116
# HELP node_disk_reads_completed_total The total number of reads completed successfully.
# TYPE node_disk_reads_completed_total counter
node_disk_reads_completed_total{device="md0"} 47878
node_disk_reads_completed_total{device="md1"} 21652
node_disk_reads_completed_total{device="md2"} 132
node_disk_reads_completed_total{device="md3"} 96578
node_disk_reads_completed_total{device="nvme0n1"} 116344
node_disk_reads_completed_total{device="nvmesh/Nag_test"} 6538
node_disk_reads_completed_total{device="nvmesh/test"} 7.6810144e+07
node_disk_reads_completed_total{device="nvmesh/test2"} 3.46231878e+08
node_disk_reads_completed_total{device="sda"} 171476
node_disk_reads_completed_total{device="sdb"} 59987
# HELP node_disk_reads_merged_total The total number of reads merged.
# TYPE node_disk_reads_merged_total counter
node_disk_reads_merged_total{device="md0"} 0
node_disk_reads_merged_total{device="md1"} 0
node_disk_reads_merged_total{device="md2"} 0
node_disk_reads_merged_total{device="md3"} 0
node_disk_reads_merged_total{device="nvme0n1"} 0
node_disk_reads_merged_total{device="nvmesh/Nag_test"} 0
node_disk_reads_merged_total{device="nvmesh/test"} 0
node_disk_reads_merged_total{device="nvmesh/test2"} 0
node_disk_reads_merged_total{device="sda"} 5597
node_disk_reads_merged_total{device="sdb"} 3243
# HELP node_disk_write_time_seconds_total This is the total number of seconds spent by all writes.
# TYPE node_disk_write_time_seconds_total counter
node_disk_write_time_seconds_total{device="md0"} 0
node_disk_write_time_seconds_total{device="md1"} 0
node_disk_write_time_seconds_total{device="md2"} 0
node_disk_write_time_seconds_total{device="md3"} 0
node_disk_write_time_seconds_total{device="nvme0n1"} 0
node_disk_write_time_seconds_total{device="nvmesh/Nag_test"} 0
node_disk_write_time_seconds_total{device="nvmesh/test"} 66101.198
node_disk_write_time_seconds_total{device="nvmesh/test2"} 64027.61
node_disk_write_time_seconds_total{device="sda"} 405674.599
node_disk_write_time_seconds_total{device="sdb"} 411488.072
# HELP node_disk_writes_completed_total The total number of writes completed successfully.
# TYPE node_disk_writes_completed_total counter
node_disk_writes_completed_total{device="md0"} 1.8688624e+07
node_disk_writes_completed_total{device="md1"} 1.1957472e+07
node_disk_writes_completed_total{device="md2"} 0
node_disk_writes_completed_total{device="md3"} 5.600318e+06
node_disk_writes_completed_total{device="nvme0n1"} 0
node_disk_writes_completed_total{device="nvmesh/Nag_test"} 0
node_disk_writes_completed_total{device="nvmesh/test"} 8.4352393e+07
node_disk_writes_completed_total{device="nvmesh/test2"} 1.395256e+06
node_disk_writes_completed_total{device="sda"} 3.1376308e+07
node_disk_writes_completed_total{device="sdb"} 3.1377355e+07
# HELP node_disk_writes_merged_total The number of writes merged.
# TYPE node_disk_writes_merged_total counter
node_disk_writes_merged_total{device="md0"} 0
node_disk_writes_merged_total{device="md1"} 0
node_disk_writes_merged_total{device="md2"} 0
node_disk_writes_merged_total{device="md3"} 0
node_disk_writes_merged_total{device="nvme0n1"} 0
node_disk_writes_merged_total{device="nvmesh/Nag_test"} 0
node_disk_writes_merged_total{device="nvmesh/test"} 0
node_disk_writes_merged_total{device="nvmesh/test2"} 0
node_disk_writes_merged_total{device="sda"} 1.8871877e+07
node_disk_writes_merged_total{device="sdb"} 1.8871074e+07
# HELP node_disk_written_bytes_total The total number of bytes written successfully.
# TYPE node_disk_written_bytes_total counter
node_disk_written_bytes_total{device="md0"} 9.5478095872e+10
node_disk_written_bytes_total{device="md1"} 1.66071640064e+11
node_disk_written_bytes_total{device="md2"} 0
node_disk_written_bytes_total{device="md3"} 3.8666489856e+10
node_disk_written_bytes_total{device="nvme0n1"} 0
node_disk_written_bytes_total{device="nvmesh/Nag_test"} 0
node_disk_written_bytes_total{device="nvmesh/test"} 1.176780132352e+12
node_disk_written_bytes_total{device="nvmesh/test2"} 3.65757988864e+11
node_disk_written_bytes_total{device="sda"} 3.08371201536e+11
node_disk_written_bytes_total{device="sdb"} 3.08371201536e+11
# HELP node_md_disks Number of active/failed/spare disks of device.
# TYPE node_md_disks gauge
node_md_disks{device="md0",state="active"} 2
10node_md_disks{device="md0",state="failed"} 0
node_md_disks{device="md0",state="spare"} 0
0node_md_disks{device="md1",state="active"} 2
node_md_disks{device="md1",state="failed"} 0
 node_md_disks{device="md1",state="spare"} 0
node_md_disks{device="md2",state="active"} 2
 node_md_disks{device="md2",state="failed"} 0
node_md_disks{device="md2",state="spare"} 0
1node_md_disks{device="md3",state="active"} 2
node_md_disks{device="md3",state="failed"} 0
2node_md_disks{device="md3",state="spare"} 0
# HELP node_md_disks_required Total number of disks of device.
9# TYPE node_md_disks_required gauge
knode_md_disks_required{device="md0"} 2
node_md_disks_required{device="md1"} 2
 node_md_disks_required{device="md2"} 2
node_md_disks_required{device="md3"} 2
SuperQ commented 3 years ago

It looks like the plugin reads from /proc/nvmeibc. In order to support this, there are a few steps.

Once those things are done, a collector can be added here.

aned commented 3 years ago

Thanks @SuperQ for looking into this!

Confirming that file permissions on the /proc are readable by non-root users (I also ran the above plugin as non-root):

[proc]$
ls -al | grep nvme
dr-xr-xr-x    3 root           root                         0 Jan 28 18:00 nvmeib
dr-xr-xr-x    2 root           root                         0 Mar  1 20:28 nvmeiba
dr-xr-xr-x    9 root           root                         0 Feb  1 22:20 nvmeibc
nvmeibc]$ ls -al
total 0
dr-xr-xr-x   9 root root 0 Feb  1 22:20 .
drwxr-xr-x 710 root root 0 Jan 28 17:50 ..
-r--r--r--   1 root root 0 Mar  1 20:29 cflags
dr-xr-xr-x   2 root root 0 Mar  1 20:29 cli
-r--r--r--   1 root root 0 Mar  1 20:29 .debug
-r--r--r--   1 root root 0 Mar  1 20:29 dict_sign
dr-xr-xr-x  14 root root 0 Mar  1 20:29 disks
dr-xr-xr-x   2 root root 0 Mar  1 20:29 dvlp
--w-------   1 root root 0 Mar  1 20:29 echo
-rw-------   1 root root 0 Mar  1 20:29 instctls
-r--r--r--   1 root root 0 Mar  1 20:29 inst_list.json
dr-xr-xr-x   2 root root 0 Mar  1 20:29 jam
dr-xr-xr-x   2 root root 0 Feb  1 22:20 mcs
dr-xr-xr-x   6 root root 0 Mar  1 20:29 net
-r--r--r--   1 root root 0 Mar  1 20:29 rsrc_info.json
-r--r--r--   1 root root 0 Mar  1 20:29 status
-r--r--r--   1 root root 0 Mar  1 20:29 version
dr-xr-xr-x   5 root root 0 Mar  1 20:29 volumes
/proc/nvmeibc/volumes/test
dr-xr-xr-x 2 root root 0 Mar  1 20:31 .
dr-xr-xr-x 5 root root 0 Mar  1 20:31 ..
-r--r--r-- 1 root root 0 Mar  1 20:32 client_processes
-r--r--r-- 1 root root 0 Mar  1 20:32 flow_cntr.json
-r--r--r-- 1 root root 0 Mar  1 20:32 iostats
-r--r--r-- 1 root root 0 Mar  1 20:32 iostats.json
-r--r--r-- 1 root root 0 Mar  1 20:32 io_throttle
-r--r--r-- 1 root root 0 Mar  1 20:32 profiling
-r--r--r-- 1 root root 0 Mar  1 20:32 profiling.csv
-r--r--r-- 1 root root 0 Mar  1 20:32 recov_stats
-r--r--r-- 1 root root 0 Mar  1 20:32 status
-r--r--r-- 1 root root 0 Mar  1 20:32 status.json
aned commented 3 years ago

This is completed via #1832