Missing metrics on recent elasticsearch versions

[sigilioso]

Missing metrics on recent elasticsearch versions

Description

The following metrics are not being reported using the integration with elasticsearch7.17.1 or elascticsearch8.1.0:

• fs.bytesReadsInBytes
• fs.iOOperations
• fs.reads
• fs.writesInBytes
• fs.writeOperations
• get.requestsDcoumentExists
• get.requestsDcoumentMissing
• jvm.gc.majorCollectionsYoungGenerationObjects
• jvm.gc.majorCollectionsYoungGenerationObjectsInMilliseconds
• threadpool.activefetchShardStarted
• threadpool.bulkActive
• threadpool.bulkQueue
• threadpool.bulkRejected
• threadpool.bulkThreads
• threadpool.indexActive
• threadpool.indexQueue
• threadpool.indexRejected

Expected Behavior

Those metrics being reported in the same way are being reported for previous elasticsearch versions.

NR Diag results

Steps to Reproduce

• Run an elasticsearch cluster (7.17.1+) and install the nri-elasticsearch integration.
• This can be easily addressed using coreint-canaries.

Your Environment

• minikube kubernetes cluster for testing purposes.

Additional context

Some elasticsearch stats have changed in recent versions. Example:

Fragment from _nodes/stats request in elasticsearch6.0.0:

{
// ...
            "thread_pool": {
                "bulk": {
                    "active": 0,
                    "completed": 20,
                    "largest": 6,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 6
                },
                "fetch_shard_started": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "fetch_shard_store": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "flush": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "force_merge": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "generic": {
                    "active": 0,
                    "completed": 157,
                    "largest": 8,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 8
                },
                "get": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "index": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "listener": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "management": {
                    "active": 1,
                    "completed": 73,
                    "largest": 3,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 3
                },
                "ml_autodetect": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "ml_datafeed": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "ml_utility": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "refresh": {
                    "active": 0,
                    "completed": 92,
                    "largest": 1,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 1
                },
                "search": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "security-token-key": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "snapshot": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "warmer": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                },
                "watcher": {
                    "active": 0,
                    "completed": 0,
                    "largest": 0,
                    "queue": 0,
                    "rejected": 0,
                    "threads": 0
                }
            }
            "fs": {
                "data": [
                    {
                        "available_in_bytes": 48402522112,
                        "free_in_bytes": 51619237888,
                        "mount": "/ (overlay)",
                        "path": "/usr/share/elasticsearch/data/nodes/0",
                        "total_in_bytes": 62725623808,
                        "type": "overlay"
                    }
                ],
                "io_stats": {},
                "least_usage_estimate": {
                    "available_in_bytes": 48402067456,
                    "path": "/usr/share/elasticsearch/data/nodes/0",
                    "total_in_bytes": 62725623808,
                    "used_disk_percent": 22.835255326983585
                },
                "most_usage_estimate": {
                    "available_in_bytes": 48402067456,
                    "path": "/usr/share/elasticsearch/data/nodes/0",
                    "total_in_bytes": 62725623808,
                    "used_disk_percent": 22.835255326983585
                },
                "timestamp": 1647511922101,
                "total": {
                    "available_in_bytes": 48402522112,
                    "free_in_bytes": 51619237888,
                    "total_in_bytes": 62725623808
                }
            },
//...
}

Fragment from the same request for elasticsearch8.1.0:

{
// ...
            "thread_pool": {
                "analyze": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "auto_complete": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "azure_event_loop": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "ccr": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "fetch_shard_started": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "fetch_shard_store": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "flush": {
                    "threads": 1,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 1,
                    "completed": 3
                },
                "force_merge": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "generic": {
                    "threads": 11,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 11,
                    "completed": 577
                },
                "get": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "management": {
                    "threads": 2,
                    "queue": 0,
                    "active": 1,
                    "rejected": 0,
                    "largest": 2,
                    "completed": 85
                },
                "ml_datafeed": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "ml_job_comms": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "ml_utility": {
                    "threads": 2,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 2,
                    "completed": 376
                },
                "refresh": {
                    "threads": 1,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 1,
                    "completed": 307
                },
                "repository_azure": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "rollup_indexing": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "search": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "search_coordination": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "search_throttled": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "searchable_snapshots_cache_fetch_async": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "searchable_snapshots_cache_prewarming": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "security-crypto": {
                    "threads": 2,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 2,
                    "completed": 2
                },
                "security-token-key": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "snapshot": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "snapshot_meta": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "system_critical_read": {
                    "threads": 1,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 1,
                    "completed": 1
                },
                "system_critical_write": {
                    "threads": 3,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 3,
                    "completed": 3
                },
                "system_read": {
                    "threads": 3,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 3,
                    "completed": 94
                },
                "system_write": {
                    "threads": 3,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 3,
                    "completed": 45
                },
                "vector_tile_generation": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "warmer": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "watcher": {
                    "threads": 0,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 0,
                    "completed": 0
                },
                "write": {
                    "threads": 6,
                    "queue": 0,
                    "active": 0,
                    "rejected": 0,
                    "largest": 6,
                    "completed": 47
                }
            },
            "fs": {
                "timestamp": 1647506922692,
                "total": {
                    "total_in_bytes": 62725623808,
                    "free_in_bytes": 53532381184,
                    "available_in_bytes": 50315665408
                },
                "data": [
                    {
                        "path": "/usr/share/elasticsearch/data",
                        "mount": "/ (overlay)",
                        "type": "overlay",
                        "total_in_bytes": 62725623808,
                        "free_in_bytes": 53532381184,
                        "available_in_bytes": 50315665408
                    }
                ],
                "io_stats": {}
            }
// ...
}

[workato-integration[bot]]

https://issues.newrelic.com/browse/NEWRELIC-3999