Add shard count to the Elasticsearch index stats scrapper

[nathan-maves]

Component(s)

receiver/elasticsearch

Is your feature request related to a problem? Please describe.

I was trying to be able to compare 3 dimensions of Elasticsearch index stats document count, shards size and shard count but this receiver only scapes 2 of the 3 from the /_stats endpoint.

Describe the solution you'd like

Add the shard count to the collected metrics for each index. The /users/_stats endpoint contains the total_count for shards.

{
    "_shards": {
        "total": 6,
        "successful": 6,
        "failed": 0
    },
    "_all": {
        "primaries": {
            "docs": {
                "count": 599007114,
                "deleted": 6255158,
                "total_size_in_bytes": 106202644463
            },
            "shard_stats": {
                "total_count": 3
            },
            "store": {
                "size_in_bytes": 106202656601,
                "total_data_set_size_in_bytes": 106202656601,
                "reserved_in_bytes": 0
            },
            "indexing": {
                "index_total": 15653444,
                "index_time_in_millis": 5137004,
                "index_current": 0,
                "index_failed": 0,
                "delete_total": 0,
                "delete_time_in_millis": 0,
                "delete_current": 0,
                "noop_update_total": 75007,
                "is_throttled": false,
                "throttle_time_in_millis": 0,
                "write_load": 0.002295616505724265
            },
            "get": {
                "total": 1077709,
                "time_in_millis": 217716,
                "exists_total": 1077709,
                "exists_time_in_millis": 217716,
                "missing_total": 0,
                "missing_time_in_millis": 0,
                "current": 0
            },
            "search": {
                "open_contexts": 0,
                "query_total": 1,
                "query_time_in_millis": 0,
                "query_current": 0,
                "fetch_total": 0,
                "fetch_time_in_millis": 0,
                "fetch_current": 0,
                "scroll_total": 1,
                "scroll_time_in_millis": 1294,
                "scroll_current": 0,
                "suggest_total": 0,
                "suggest_time_in_millis": 0,
                "suggest_current": 0
            },
            "merges": {
                "current": 0,
                "current_docs": 0,
                "current_size_in_bytes": 0,
                "total": 1052,
                "total_time_in_millis": 17872746,
                "total_docs": 568163196,
                "total_size_in_bytes": 105534994910,
                "total_stopped_time_in_millis": 0,
                "total_throttled_time_in_millis": 14177420,
                "total_auto_throttle_in_bytes": 45279418
            },
            "refresh": {
                "total": 3070,
                "total_time_in_millis": 100790,
                "external_total": 1064,
                "external_total_time_in_millis": 834,
                "listeners": 0
            },
            "flush": {
                "total": 904,
                "periodic": 902,
                "total_time_in_millis": 434681,
                "total_time_excluding_waiting_on_lock_in_millis": 466358
            },
            "warmer": {
                "current": 0,
                "total": 1061,
                "total_time_in_millis": 313
            },
            "query_cache": {
                "memory_size_in_bytes": 0,
                "total_count": 99,
                "hit_count": 0,
                "miss_count": 99,
                "cache_size": 0,
                "cache_count": 0,
                "evictions": 0
            },
            "fielddata": {
                "memory_size_in_bytes": 0,
                "evictions": 0,
                "global_ordinals": {
                    "build_time_in_millis": 0
                }
            },
            "completion": {
                "size_in_bytes": 0
            },
            "segments": {
                "count": 99,
                "memory_in_bytes": 0,
                "terms_memory_in_bytes": 0,
                "stored_fields_memory_in_bytes": 0,
                "term_vectors_memory_in_bytes": 0,
                "norms_memory_in_bytes": 0,
                "points_memory_in_bytes": 0,
                "doc_values_memory_in_bytes": 0,
                "index_writer_memory_in_bytes": 0,
                "version_map_memory_in_bytes": 0,
                "fixed_bit_set_memory_in_bytes": 75662136,
                "max_unsafe_auto_id_timestamp": -1,
                "file_sizes": {}
            },
            "translog": {
                "operations": 0,
                "size_in_bytes": 165,
                "uncommitted_operations": 0,
                "uncommitted_size_in_bytes": 165,
                "earliest_last_modified_age": 408423717
            },
            "request_cache": {
                "memory_size_in_bytes": 0,
                "evictions": 0,
                "hit_count": 0,
                "miss_count": 0
            },
            "recovery": {
                "current_as_source": 0,
                "current_as_target": 0,
                "throttle_time_in_millis": 309022
            },
            "bulk": {
                "total_operations": 183844,
                "total_time_in_millis": 5419278,
                "total_size_in_bytes": 26490845523,
                "avg_time_in_millis": 16,
                "avg_size_in_bytes": 63504
            },
            "dense_vector": {
                "value_count": 0
            },
            "sparse_vector": {
                "value_count": 0
            }
        },
        "total": {
            "docs": {
                "count": 1198014228,
                "deleted": 12506844,
                "total_size_in_bytes": 212356147240
            },
            "shard_stats": {
                "total_count": 6
            },
            "store": {
                "size_in_bytes": 212356170745,
                "total_data_set_size_in_bytes": 212356170745,
                "reserved_in_bytes": 0
            },
            "indexing": {
                "index_total": 15653444,
                "index_time_in_millis": 5137004,
                "index_current": 0,
                "index_failed": 0,
                "delete_total": 0,
                "delete_time_in_millis": 0,
                "delete_current": 0,
                "noop_update_total": 75007,
                "is_throttled": false,
                "throttle_time_in_millis": 0,
                "write_load": 0.0012662688070682509
            },
            "get": {
                "total": 1077709,
                "time_in_millis": 217716,
                "exists_total": 1077709,
                "exists_time_in_millis": 217716,
                "missing_total": 0,
                "missing_time_in_millis": 0,
                "current": 0
            },
            "search": {
                "open_contexts": 0,
                "query_total": 1,
                "query_time_in_millis": 0,
                "query_current": 0,
                "fetch_total": 0,
                "fetch_time_in_millis": 0,
                "fetch_current": 0,
                "scroll_total": 1,
                "scroll_time_in_millis": 1294,
                "scroll_current": 0,
                "suggest_total": 0,
                "suggest_time_in_millis": 0,
                "suggest_current": 0
            },
            "merges": {
                "current": 0,
                "current_docs": 0,
                "current_size_in_bytes": 0,
                "total": 1053,
                "total_time_in_millis": 17873029,
                "total_docs": 568207232,
                "total_size_in_bytes": 105545476271,
                "total_stopped_time_in_millis": 0,
                "total_throttled_time_in_millis": 14177420,
                "total_auto_throttle_in_bytes": 108193978
            },
            "refresh": {
                "total": 3091,
                "total_time_in_millis": 101623,
                "external_total": 1075,
                "external_total_time_in_millis": 987,
                "listeners": 0
            },
            "flush": {
                "total": 910,
                "periodic": 907,
                "total_time_in_millis": 434756,
                "total_time_excluding_waiting_on_lock_in_millis": 466433
            },
            "warmer": {
                "current": 0,
                "total": 1069,
                "total_time_in_millis": 451
            },
            "query_cache": {
                "memory_size_in_bytes": 0,
                "total_count": 99,
                "hit_count": 0,
                "miss_count": 99,
                "cache_size": 0,
                "cache_count": 0,
                "evictions": 0
            },
            "fielddata": {
                "memory_size_in_bytes": 0,
                "evictions": 0,
                "global_ordinals": {
                    "build_time_in_millis": 0
                }
            },
            "completion": {
                "size_in_bytes": 0
            },
            "segments": {
                "count": 191,
                "memory_in_bytes": 0,
                "terms_memory_in_bytes": 0,
                "stored_fields_memory_in_bytes": 0,
                "term_vectors_memory_in_bytes": 0,
                "norms_memory_in_bytes": 0,
                "points_memory_in_bytes": 0,
                "doc_values_memory_in_bytes": 0,
                "index_writer_memory_in_bytes": 0,
                "version_map_memory_in_bytes": 0,
                "fixed_bit_set_memory_in_bytes": 151323528,
                "max_unsafe_auto_id_timestamp": -1,
                "file_sizes": {}
            },
            "translog": {
                "operations": 0,
                "size_in_bytes": 330,
                "uncommitted_operations": 0,
                "uncommitted_size_in_bytes": 330,
                "earliest_last_modified_age": 408423717
            },
            "request_cache": {
                "memory_size_in_bytes": 0,
                "evictions": 0,
                "hit_count": 0,
                "miss_count": 0
            },
            "recovery": {
                "current_as_source": 0,
                "current_as_target": 0,
                "throttle_time_in_millis": 1678130
            },
            "bulk": {
                "total_operations": 183844,
                "total_time_in_millis": 5419278,
                "total_size_in_bytes": 26490845523,
                "avg_time_in_millis": 16,
                "avg_size_in_bytes": 63504
            },
            "dense_vector": {
                "value_count": 0
            },
            "sparse_vector": {
                "value_count": 0
            }
        }
    },
    "indices": {
        "users": {
            "uuid": "6UqMYhVwS4OwyngTqKG8ow",
            "health": "green",
            "status": "open",
            "primaries": {
                "docs": {
                    "count": 599007114,
                    "deleted": 6255158,
                    "total_size_in_bytes": 106202644463
                },
                "shard_stats": {
                    "total_count": 3
                },
                "store": {
                    "size_in_bytes": 106202656601,
                    "total_data_set_size_in_bytes": 106202656601,
                    "reserved_in_bytes": 0
                },
                "indexing": {
                    "index_total": 15653444,
                    "index_time_in_millis": 5137004,
                    "index_current": 0,
                    "index_failed": 0,
                    "delete_total": 0,
                    "delete_time_in_millis": 0,
                    "delete_current": 0,
                    "noop_update_total": 75007,
                    "is_throttled": false,
                    "throttle_time_in_millis": 0,
                    "write_load": 0.002295616505724265
                },
                "get": {
                    "total": 1077709,
                    "time_in_millis": 217716,
                    "exists_total": 1077709,
                    "exists_time_in_millis": 217716,
                    "missing_total": 0,
                    "missing_time_in_millis": 0,
                    "current": 0
                },
                "search": {
                    "open_contexts": 0,
                    "query_total": 1,
                    "query_time_in_millis": 0,
                    "query_current": 0,
                    "fetch_total": 0,
                    "fetch_time_in_millis": 0,
                    "fetch_current": 0,
                    "scroll_total": 1,
                    "scroll_time_in_millis": 1294,
                    "scroll_current": 0,
                    "suggest_total": 0,
                    "suggest_time_in_millis": 0,
                    "suggest_current": 0
                },
                "merges": {
                    "current": 0,
                    "current_docs": 0,
                    "current_size_in_bytes": 0,
                    "total": 1052,
                    "total_time_in_millis": 17872746,
                    "total_docs": 568163196,
                    "total_size_in_bytes": 105534994910,
                    "total_stopped_time_in_millis": 0,
                    "total_throttled_time_in_millis": 14177420,
                    "total_auto_throttle_in_bytes": 45279418
                },
                "refresh": {
                    "total": 3070,
                    "total_time_in_millis": 100790,
                    "external_total": 1064,
                    "external_total_time_in_millis": 834,
                    "listeners": 0
                },
                "flush": {
                    "total": 904,
                    "periodic": 902,
                    "total_time_in_millis": 434681,
                    "total_time_excluding_waiting_on_lock_in_millis": 466358
                },
                "warmer": {
                    "current": 0,
                    "total": 1061,
                    "total_time_in_millis": 313
                },
                "query_cache": {
                    "memory_size_in_bytes": 0,
                    "total_count": 99,
                    "hit_count": 0,
                    "miss_count": 99,
                    "cache_size": 0,
                    "cache_count": 0,
                    "evictions": 0
                },
                "fielddata": {
                    "memory_size_in_bytes": 0,
                    "evictions": 0,
                    "global_ordinals": {
                        "build_time_in_millis": 0
                    }
                },
                "completion": {
                    "size_in_bytes": 0
                },
                "segments": {
                    "count": 99,
                    "memory_in_bytes": 0,
                    "terms_memory_in_bytes": 0,
                    "stored_fields_memory_in_bytes": 0,
                    "term_vectors_memory_in_bytes": 0,
                    "norms_memory_in_bytes": 0,
                    "points_memory_in_bytes": 0,
                    "doc_values_memory_in_bytes": 0,
                    "index_writer_memory_in_bytes": 0,
                    "version_map_memory_in_bytes": 0,
                    "fixed_bit_set_memory_in_bytes": 75662136,
                    "max_unsafe_auto_id_timestamp": -1,
                    "file_sizes": {}
                },
                "translog": {
                    "operations": 0,
                    "size_in_bytes": 165,
                    "uncommitted_operations": 0,
                    "uncommitted_size_in_bytes": 165,
                    "earliest_last_modified_age": 408423717
                },
                "request_cache": {
                    "memory_size_in_bytes": 0,
                    "evictions": 0,
                    "hit_count": 0,
                    "miss_count": 0
                },
                "recovery": {
                    "current_as_source": 0,
                    "current_as_target": 0,
                    "throttle_time_in_millis": 309022
                },
                "bulk": {
                    "total_operations": 183844,
                    "total_time_in_millis": 5419278,
                    "total_size_in_bytes": 26490845523,
                    "avg_time_in_millis": 16,
                    "avg_size_in_bytes": 63504
                },
                "dense_vector": {
                    "value_count": 0
                },
                "sparse_vector": {
                    "value_count": 0
                }
            },
            "total": {
                "docs": {
                    "count": 1198014228,
                    "deleted": 12506844,
                    "total_size_in_bytes": 212356147240
                },
                "shard_stats": {
                    "total_count": 6
                },
                "store": {
                    "size_in_bytes": 212356170745,
                    "total_data_set_size_in_bytes": 212356170745,
                    "reserved_in_bytes": 0
                },
                "indexing": {
                    "index_total": 15653444,
                    "index_time_in_millis": 5137004,
                    "index_current": 0,
                    "index_failed": 0,
                    "delete_total": 0,
                    "delete_time_in_millis": 0,
                    "delete_current": 0,
                    "noop_update_total": 75007,
                    "is_throttled": false,
                    "throttle_time_in_millis": 0,
                    "write_load": 0.0012662688070682509
                },
                "get": {
                    "total": 1077709,
                    "time_in_millis": 217716,
                    "exists_total": 1077709,
                    "exists_time_in_millis": 217716,
                    "missing_total": 0,
                    "missing_time_in_millis": 0,
                    "current": 0
                },
                "search": {
                    "open_contexts": 0,
                    "query_total": 1,
                    "query_time_in_millis": 0,
                    "query_current": 0,
                    "fetch_total": 0,
                    "fetch_time_in_millis": 0,
                    "fetch_current": 0,
                    "scroll_total": 1,
                    "scroll_time_in_millis": 1294,
                    "scroll_current": 0,
                    "suggest_total": 0,
                    "suggest_time_in_millis": 0,
                    "suggest_current": 0
                },
                "merges": {
                    "current": 0,
                    "current_docs": 0,
                    "current_size_in_bytes": 0,
                    "total": 1053,
                    "total_time_in_millis": 17873029,
                    "total_docs": 568207232,
                    "total_size_in_bytes": 105545476271,
                    "total_stopped_time_in_millis": 0,
                    "total_throttled_time_in_millis": 14177420,
                    "total_auto_throttle_in_bytes": 108193978
                },
                "refresh": {
                    "total": 3091,
                    "total_time_in_millis": 101623,
                    "external_total": 1075,
                    "external_total_time_in_millis": 987,
                    "listeners": 0
                },
                "flush": {
                    "total": 910,
                    "periodic": 907,
                    "total_time_in_millis": 434756,
                    "total_time_excluding_waiting_on_lock_in_millis": 466433
                },
                "warmer": {
                    "current": 0,
                    "total": 1069,
                    "total_time_in_millis": 451
                },
                "query_cache": {
                    "memory_size_in_bytes": 0,
                    "total_count": 99,
                    "hit_count": 0,
                    "miss_count": 99,
                    "cache_size": 0,
                    "cache_count": 0,
                    "evictions": 0
                },
                "fielddata": {
                    "memory_size_in_bytes": 0,
                    "evictions": 0,
                    "global_ordinals": {
                        "build_time_in_millis": 0
                    }
                },
                "completion": {
                    "size_in_bytes": 0
                },
                "segments": {
                    "count": 191,
                    "memory_in_bytes": 0,
                    "terms_memory_in_bytes": 0,
                    "stored_fields_memory_in_bytes": 0,
                    "term_vectors_memory_in_bytes": 0,
                    "norms_memory_in_bytes": 0,
                    "points_memory_in_bytes": 0,
                    "doc_values_memory_in_bytes": 0,
                    "index_writer_memory_in_bytes": 0,
                    "version_map_memory_in_bytes": 0,
                    "fixed_bit_set_memory_in_bytes": 151323528,
                    "max_unsafe_auto_id_timestamp": -1,
                    "file_sizes": {}
                },
                "translog": {
                    "operations": 0,
                    "size_in_bytes": 330,
                    "uncommitted_operations": 0,
                    "uncommitted_size_in_bytes": 330,
                    "earliest_last_modified_age": 408423717
                },
                "request_cache": {
                    "memory_size_in_bytes": 0,
                    "evictions": 0,
                    "hit_count": 0,
                    "miss_count": 0
                },
                "recovery": {
                    "current_as_source": 0,
                    "current_as_target": 0,
                    "throttle_time_in_millis": 1678130
                },
                "bulk": {
                    "total_operations": 183844,
                    "total_time_in_millis": 5419278,
                    "total_size_in_bytes": 26490845523,
                    "avg_time_in_millis": 16,
                    "avg_size_in_bytes": 63504
                },
                "dense_vector": {
                    "value_count": 0
                },
                "sparse_vector": {
                    "value_count": 0
                }
            }
        }
    }
}

Describe alternatives you've considered

No response

Additional context

No response

[github-actions[bot]]

Pinging code owners:

• receiver/elasticsearch: @djaglowski @BinaryFissionGames

  See Adding Labels via Comments if you do not have permissions to add labels yourself.