Closed eresgie closed 8 years ago
@gseven can you send the output of your server from
http://<url>:<port>/_cluster/health?level=indices
Hi,
Got the same issue on ES 2.1 with either Telegraf 0.2.3 and 0.2.4. Here is the indices output:
curl 0:9200/_cluster/health?level=indices
{
"cluster_name" : "MyCluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 29,
"active_shards" : 58,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0,
"indices" : {
".marvel-es-2015.12.10" : {
"status" : "green",
"number_of_shards" : 1,
"number_of_replicas" : 1,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"shinken-logs-2015.12.09" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"logstash-2015.12.09" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"log-2015.12.11" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"log-2015.12.10" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
".kibana" : {
"status" : "green",
"number_of_shards" : 1,
"number_of_replicas" : 1,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
".marvel-es-2015.12.09" : {
"status" : "green",
"number_of_shards" : 1,
"number_of_replicas" : 1,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"logstash-2015.12.10" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"logstash-2015.12.11" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
".marvel-es-data" : {
"status" : "green",
"number_of_shards" : 1,
"number_of_replicas" : 1,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"shinken-logs-2015.12.11" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
".marvel-es-2015.12.11" : {
"status" : "green",
"number_of_shards" : 1,
"number_of_replicas" : 1,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
},
"shinken-logs-2015.12.10" : {
"status" : "green",
"number_of_shards" : 3,
"number_of_replicas" : 1,
"active_primary_shards" : 3,
"active_shards" : 6,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0
}
}
}
Thx
thanks @descrepes, could you also provide the output from http://<url>:<port>//_nodes/_local/stats
or http://<url>:<port>//_nodes/stats
if local = false
Here is the output if local=false
{
"cluster_name" : "MyCluster",
"nodes" : {
"wAO2SUZKQfW-S0vosneONw" : {
"timestamp" : 1449865265541,
"name" : "es2",
"transport_address" : "10.24.1.112:9300",
"host" : "10.24.1.112",
"ip" : [ "10.24.1.112:9300", "NONE" ],
"attributes" : {
"master" : "true"
},
"indices" : {
"docs" : {
"count" : 41369912,
"deleted" : 4316
},
"store" : {
"size_in_bytes" : 20564569225,
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 41706342,
"index_time_in_millis" : 15019196,
"index_current" : 156,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 208,
"time_in_millis" : 31,
"exists_total" : 208,
"exists_time_in_millis" : 31,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 62139,
"query_time_in_millis" : 134536,
"query_current" : 0,
"fetch_total" : 39224,
"fetch_time_in_millis" : 241913,
"fetch_current" : 0,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 2181,
"total_time_in_millis" : 7015107,
"total_docs" : 104161600,
"total_size_in_bytes" : 55654519452,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 2240076,
"total_auto_throttle_in_bytes" : 449050142
},
"refresh" : {
"total" : 19509,
"total_time_in_millis" : 2407306
},
"flush" : {
"total" : 240,
"total_time_in_millis" : 63207
},
"warmer" : {
"current" : 0,
"total" : 39498,
"total_time_in_millis" : 5965
},
"query_cache" : {
"memory_size_in_bytes" : 56992,
"total_count" : 1417951,
"hit_count" : 620,
"miss_count" : 1417331,
"cache_size" : 99,
"cache_count" : 23390,
"evictions" : 23291
},
"fielddata" : {
"memory_size_in_bytes" : 23884,
"evictions" : 0
},
"percolate" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0,
"memory_size_in_bytes" : -1,
"memory_size" : "-1b",
"queries" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 209,
"memory_in_bytes" : 75050605,
"terms_memory_in_bytes" : 65407293,
"stored_fields_memory_in_bytes" : 9203664,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 259200,
"doc_values_memory_in_bytes" : 180448,
"index_writer_memory_in_bytes" : 17933608,
"index_writer_max_memory_in_bytes" : 1718693885,
"version_map_memory_in_bytes" : 2747370,
"fixed_bit_set_memory_in_bytes" : 0
},
"translog" : {
"operations" : 690244,
"size_in_bytes" : 493629570
},
"suggest" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 153744
}
},
"os" : {
"timestamp" : 1449865265550,
"load_average" : 0.45,
"mem" : {
"total_in_bytes" : 33795493888,
"free_in_bytes" : 4583841792,
"used_in_bytes" : 29211652096,
"free_percent" : 14,
"used_percent" : 86
},
"swap" : {
"total_in_bytes" : 3998216192,
"free_in_bytes" : 3998216192,
"used_in_bytes" : 0
}
},
"process" : {
"timestamp" : 1449865265550,
"open_file_descriptors" : 727,
"max_file_descriptors" : 65535,
"cpu" : {
"percent" : 0,
"total_in_millis" : 30681500
},
"mem" : {
"total_virtual_in_bytes" : 23391014912
}
},
"jvm" : {
"timestamp" : 1449865265551,
"uptime_in_millis" : 187616941,
"mem" : {
"heap_used_in_bytes" : 2986966512,
"heap_used_percent" : 17,
"heap_committed_in_bytes" : 17110138880,
"heap_max_in_bytes" : 17110138880,
"non_heap_used_in_bytes" : 119796264,
"non_heap_committed_in_bytes" : 121733120,
"pools" : {
"young" : {
"used_in_bytes" : 399428904,
"max_in_bytes" : 558432256,
"peak_used_in_bytes" : 558432256,
"peak_max_in_bytes" : 558432256
},
"survivor" : {
"used_in_bytes" : 17172856,
"max_in_bytes" : 69730304,
"peak_used_in_bytes" : 69730304,
"peak_max_in_bytes" : 69730304
},
"old" : {
"used_in_bytes" : 2570364752,
"max_in_bytes" : 16481976320,
"peak_used_in_bytes" : 2570364752,
"peak_max_in_bytes" : 16481976320
}
}
},
"threads" : {
"count" : 116,
"peak_count" : 127
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 4615,
"collection_time_in_millis" : 369165
},
"old" : {
"collection_count" : 2,
"collection_time_in_millis" : 414
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 1693,
"used_in_bytes" : 53059343,
"total_capacity_in_bytes" : 53059343
},
"mapped" : {
"count" : 0,
"used_in_bytes" : 0,
"total_capacity_in_bytes" : 0
}
}
},
"thread_pool" : {
"bulk" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 360524
},
"fetch_shard_started" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"fetch_shard_store" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 5,
"completed" : 10
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 2,
"completed" : 3377
},
"force_merge" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"generic" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 10,
"completed" : 81773
},
"get" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 208
},
"index" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 33
},
"listener" : {
"threads" : 4,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 4,
"completed" : 77827
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 5,
"completed" : 200616
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 3,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 3,
"completed" : 19578
},
"search" : {
"threads" : 13,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 13,
"completed" : 91823
},
"snapshot" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"suggest" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"warmer" : {
"threads" : 2,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 2,
"completed" : 21299
}
},
"fs" : {
"timestamp" : 1449865265551,
"total" : {
"total_in_bytes" : 487548043264,
"free_in_bytes" : 463857954816,
"available_in_bytes" : 439068307456
},
"data" : [ {
"path" : "/path/nodes/0",
"mount" : "/ (rootfs)",
"type" : "rootfs",
"total_in_bytes" : 487548043264,
"free_in_bytes" : 463857954816,
"available_in_bytes" : 439068307456
} ]
},
"transport" : {
"server_open" : 26,
"rx_count" : 1622208,
"rx_size_in_bytes" : 7490931037,
"tx_count" : 1622433,
"tx_size_in_bytes" : 6635847546
},
"http" : {
"current_open" : 2,
"total_opened" : 110111
},
"breakers" : {
"request" : {
"limit_size_in_bytes" : 6844055552,
"limit_size" : "6.3gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
},
"fielddata" : {
"limit_size_in_bytes" : 10266083328,
"limit_size" : "9.5gb",
"estimated_size_in_bytes" : 23884,
"estimated_size" : "23.3kb",
"overhead" : 1.03,
"tripped" : 0
},
"parent" : {
"limit_size_in_bytes" : 11977097216,
"limit_size" : "11.1gb",
"estimated_size_in_bytes" : 23884,
"estimated_size" : "23.3kb",
"overhead" : 1.0,
"tripped" : 0
}
},
"script" : {
"compilations" : 0,
"cache_evictions" : 0
}
},
"X0l8U62nQVOyzto9bKZV8A" : {
"timestamp" : 1449865265541,
"name" : "es3",
"transport_address" : "10.24.1.113:9300",
"host" : "10.24.1.113",
"ip" : [ "10.24.1.113:9300", "NONE" ],
"attributes" : {
"master" : "true"
},
"indices" : {
"docs" : {
"count" : 41214090,
"deleted" : 9197
},
"store" : {
"size_in_bytes" : 20495853124,
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 29113429,
"index_time_in_millis" : 10572224,
"index_current" : 18379,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 14,
"time_in_millis" : 3,
"exists_total" : 12,
"exists_time_in_millis" : 3,
"missing_total" : 2,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 19276,
"query_time_in_millis" : 129239,
"query_current" : 0,
"fetch_total" : 2097,
"fetch_time_in_millis" : 273689,
"fetch_current" : 0,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 1595,
"total_time_in_millis" : 4918646,
"total_docs" : 77648656,
"total_size_in_bytes" : 41948854557,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 1637559,
"total_auto_throttle_in_bytes" : 424617500
},
"refresh" : {
"total" : 16601,
"total_time_in_millis" : 1687807
},
"flush" : {
"total" : 98,
"total_time_in_millis" : 31332
},
"warmer" : {
"current" : 0,
"total" : 34347,
"total_time_in_millis" : 4788
},
"query_cache" : {
"memory_size_in_bytes" : 63944,
"total_count" : 1082989,
"hit_count" : 432,
"miss_count" : 1082557,
"cache_size" : 106,
"cache_count" : 23187,
"evictions" : 23081
},
"fielddata" : {
"memory_size_in_bytes" : 24872,
"evictions" : 0
},
"percolate" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0,
"memory_size_in_bytes" : -1,
"memory_size" : "-1b",
"queries" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 196,
"memory_in_bytes" : 74131386,
"terms_memory_in_bytes" : 64548230,
"stored_fields_memory_in_bytes" : 9195496,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 264704,
"doc_values_memory_in_bytes" : 122956,
"index_writer_memory_in_bytes" : 12200308,
"index_writer_max_memory_in_bytes" : 1717669886,
"version_map_memory_in_bytes" : 1692345,
"fixed_bit_set_memory_in_bytes" : 0
},
"translog" : {
"operations" : 755002,
"size_in_bytes" : 540410446
},
"suggest" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 217309
}
},
"os" : {
"timestamp" : 1449865265548,
"load_average" : 0.28,
"mem" : {
"total_in_bytes" : 33795559424,
"free_in_bytes" : 6458634240,
"used_in_bytes" : 27336925184,
"free_percent" : 19,
"used_percent" : 81
},
"swap" : {
"total_in_bytes" : 3998216192,
"free_in_bytes" : 3998216192,
"used_in_bytes" : 0
}
},
"process" : {
"timestamp" : 1449865265548,
"open_file_descriptors" : 704,
"max_file_descriptors" : 65535,
"cpu" : {
"percent" : 0,
"total_in_millis" : 21220780
},
"mem" : {
"total_virtual_in_bytes" : 23372566528
}
},
"jvm" : {
"timestamp" : 1449865265549,
"uptime_in_millis" : 132127510,
"mem" : {
"heap_used_in_bytes" : 2601432912,
"heap_used_percent" : 15,
"heap_committed_in_bytes" : 17110138880,
"heap_max_in_bytes" : 17110138880,
"non_heap_used_in_bytes" : 112651432,
"non_heap_committed_in_bytes" : 114323456,
"pools" : {
"young" : {
"used_in_bytes" : 85231472,
"max_in_bytes" : 558432256,
"peak_used_in_bytes" : 558432256,
"peak_max_in_bytes" : 558432256
},
"survivor" : {
"used_in_bytes" : 35449056,
"max_in_bytes" : 69730304,
"peak_used_in_bytes" : 63691320,
"peak_max_in_bytes" : 69730304
},
"old" : {
"used_in_bytes" : 2480752384,
"max_in_bytes" : 16481976320,
"peak_used_in_bytes" : 2480752384,
"peak_max_in_bytes" : 16481976320
}
}
},
"threads" : {
"count" : 106,
"peak_count" : 111
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 3395,
"collection_time_in_millis" : 287239
},
"old" : {
"collection_count" : 1,
"collection_time_in_millis" : 113
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 1650,
"used_in_bytes" : 52226175,
"total_capacity_in_bytes" : 52226175
},
"mapped" : {
"count" : 0,
"used_in_bytes" : 0,
"total_capacity_in_bytes" : 0
}
}
},
"thread_pool" : {
"bulk" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 250820
},
"fetch_shard_started" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"fetch_shard_store" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 8
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 3,
"completed" : 2214
},
"force_merge" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"generic" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 11,
"completed" : 157932
},
"get" : {
"threads" : 6,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 6,
"completed" : 6
},
"index" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"listener" : {
"threads" : 4,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 4,
"completed" : 18882
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 5,
"completed" : 135319
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 3,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 3,
"completed" : 16601
},
"search" : {
"threads" : 13,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 13,
"completed" : 41722
},
"snapshot" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"suggest" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"warmer" : {
"threads" : 2,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 2,
"completed" : 17713
}
},
"fs" : {
"timestamp" : 1449865265549,
"total" : {
"total_in_bytes" : 487548043264,
"free_in_bytes" : 424729858048,
"available_in_bytes" : 399940210688
},
"data" : [ {
"path" : "/path/nodes/0",
"mount" : "/ (rootfs)",
"type" : "rootfs",
"total_in_bytes" : 487548043264,
"free_in_bytes" : 424729858048,
"available_in_bytes" : 399940210688
} ]
},
"transport" : {
"server_open" : 26,
"rx_count" : 844604,
"rx_size_in_bytes" : 7263246566,
"tx_count" : 844493,
"tx_size_in_bytes" : 1818015485
},
"http" : {
"current_open" : 2,
"total_opened" : 77143
},
"breakers" : {
"request" : {
"limit_size_in_bytes" : 6844055552,
"limit_size" : "6.3gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
},
"fielddata" : {
"limit_size_in_bytes" : 10266083328,
"limit_size" : "9.5gb",
"estimated_size_in_bytes" : 24872,
"estimated_size" : "24.2kb",
"overhead" : 1.03,
"tripped" : 0
},
"parent" : {
"limit_size_in_bytes" : 11977097216,
"limit_size" : "11.1gb",
"estimated_size_in_bytes" : 24872,
"estimated_size" : "24.2kb",
"overhead" : 1.0,
"tripped" : 0
}
},
"script" : {
"compilations" : 0,
"cache_evictions" : 0
}
},
"y_VtOpbWQ1-4snwf9nnROA" : {
"timestamp" : 1449865265541,
"name" : "es1",
"transport_address" : "10.24.1.111:9300",
"host" : "10.24.1.111",
"ip" : [ "10.24.1.111:9300", "NONE" ],
"attributes" : {
"master" : "true"
},
"indices" : {
"docs" : {
"count" : 41160716,
"deleted" : 3567
},
"store" : {
"size_in_bytes" : 20294778425,
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 41628942,
"index_time_in_millis" : 15379059,
"index_current" : 500,
"index_failed" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 264,
"time_in_millis" : 39,
"exists_total" : 259,
"exists_time_in_millis" : 39,
"missing_total" : 5,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 62859,
"query_time_in_millis" : 136515,
"query_current" : 0,
"fetch_total" : 40442,
"fetch_time_in_millis" : 282869,
"fetch_current" : 0,
"scroll_total" : 0,
"scroll_time_in_millis" : 0,
"scroll_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 2114,
"total_time_in_millis" : 7010276,
"total_docs" : 106746423,
"total_size_in_bytes" : 57705903517,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 2545609,
"total_auto_throttle_in_bytes" : 490385712
},
"refresh" : {
"total" : 21177,
"total_time_in_millis" : 2511144
},
"flush" : {
"total" : 250,
"total_time_in_millis" : 69249
},
"warmer" : {
"current" : 0,
"total" : 42374,
"total_time_in_millis" : 6350
},
"query_cache" : {
"memory_size_in_bytes" : 57400,
"total_count" : 1372803,
"hit_count" : 595,
"miss_count" : 1372208,
"cache_size" : 80,
"cache_count" : 23768,
"evictions" : 23688
},
"fielddata" : {
"memory_size_in_bytes" : 29588,
"evictions" : 0
},
"percolate" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0,
"memory_size_in_bytes" : -1,
"memory_size" : "-1b",
"queries" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 205,
"memory_in_bytes" : 70871298,
"terms_memory_in_bytes" : 61327386,
"stored_fields_memory_in_bytes" : 9194744,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 254464,
"doc_values_memory_in_bytes" : 94704,
"index_writer_memory_in_bytes" : 10092308,
"index_writer_max_memory_in_bytes" : 1718181885,
"version_map_memory_in_bytes" : 1713741,
"fixed_bit_set_memory_in_bytes" : 0
},
"translog" : {
"operations" : 205015,
"size_in_bytes" : 211612631
},
"suggest" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 146326
}
},
"os" : {
"timestamp" : 1449865265548,
"load_average" : 0.98,
"mem" : {
"total_in_bytes" : 33795493888,
"free_in_bytes" : 3113426944,
"used_in_bytes" : 30682066944,
"free_percent" : 9,
"used_percent" : 91
},
"swap" : {
"total_in_bytes" : 3998216192,
"free_in_bytes" : 3998216192,
"used_in_bytes" : 0
}
},
"process" : {
"timestamp" : 1449865265548,
"open_file_descriptors" : 667,
"max_file_descriptors" : 65535,
"cpu" : {
"percent" : 0,
"total_in_millis" : 33120720
},
"mem" : {
"total_virtual_in_bytes" : 23393722368
}
},
"jvm" : {
"timestamp" : 1449865265549,
"uptime_in_millis" : 191170739,
"mem" : {
"heap_used_in_bytes" : 4759426224,
"heap_used_percent" : 27,
"heap_committed_in_bytes" : 17110138880,
"heap_max_in_bytes" : 17110138880,
"non_heap_used_in_bytes" : 114393192,
"non_heap_committed_in_bytes" : 115834880,
"pools" : {
"young" : {
"used_in_bytes" : 555727080,
"max_in_bytes" : 558432256,
"peak_used_in_bytes" : 558432256,
"peak_max_in_bytes" : 558432256
},
"survivor" : {
"used_in_bytes" : 27311800,
"max_in_bytes" : 69730304,
"peak_used_in_bytes" : 69730304,
"peak_max_in_bytes" : 69730304
},
"old" : {
"used_in_bytes" : 4176387344,
"max_in_bytes" : 16481976320,
"peak_used_in_bytes" : 4176387344,
"peak_max_in_bytes" : 16481976320
}
}
},
"threads" : {
"count" : 116,
"peak_count" : 131
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 4893,
"collection_time_in_millis" : 395996
},
"old" : {
"collection_count" : 1,
"collection_time_in_millis" : 124
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 1807,
"used_in_bytes" : 55124089,
"total_capacity_in_bytes" : 55124089
},
"mapped" : {
"count" : 0,
"used_in_bytes" : 0,
"total_capacity_in_bytes" : 0
}
},
"classes" : {
"current_loaded_count" : 8814,
"total_loaded_count" : 8814,
"total_unloaded_count" : 0
}
},
"thread_pool" : {
"bulk" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 353712
},
"fetch_shard_started" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"fetch_shard_store" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 6,
"completed" : 16
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 3,
"completed" : 3422
},
"force_merge" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"generic" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 9,
"completed" : 217304
},
"get" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 248
},
"index" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 37
},
"listener" : {
"threads" : 4,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 4,
"completed" : 27141
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 5,
"completed" : 200063
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 3,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 3,
"completed" : 21237
},
"search" : {
"threads" : 13,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 13,
"completed" : 93223
},
"snapshot" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"suggest" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"warmer" : {
"threads" : 2,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 2,
"completed" : 22875
}
},
"fs" : {
"timestamp" : 1449865265549,
"total" : {
"total_in_bytes" : 487548043264,
"free_in_bytes" : 464252317696,
"available_in_bytes" : 439462670336
},
"data" : [ {
"path" : "/path/nodes/0",
"mount" : "/ (rootfs)",
"type" : "rootfs",
"total_in_bytes" : 487548043264,
"free_in_bytes" : 464252317696,
"available_in_bytes" : 439462670336
} ]
},
"transport" : {
"server_open" : 26,
"rx_count" : 1335517,
"rx_size_in_bytes" : 4925826207,
"tx_count" : 1335420,
"tx_size_in_bytes" : 9983588302
},
"http" : {
"current_open" : 3,
"total_opened" : 110989
},
"breakers" : {
"request" : {
"limit_size_in_bytes" : 6844055552,
"limit_size" : "6.3gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
},
"fielddata" : {
"limit_size_in_bytes" : 10266083328,
"limit_size" : "9.5gb",
"estimated_size_in_bytes" : 29588,
"estimated_size" : "28.8kb",
"overhead" : 1.03,
"tripped" : 0
},
"parent" : {
"limit_size_in_bytes" : 11977097216,
"limit_size" : "11.1gb",
"estimated_size_in_bytes" : 29588,
"estimated_size" : "28.8kb",
"overhead" : 1.0,
"tripped" : 0
}
},
"script" : {
"compilations" : 0,
"cache_evictions" : 0
}
}
}
}
Looks like your output is missing the network
key as shown here: https://github.com/influxdb/telegraf/blob/master/plugins/elasticsearch/testdata_test.go#L442
Do you know if the network stats are something that needs to be enabled on your elasticsearch cluster?
I can reproduce with a docker container elasticsearch. I'm not sure why there is no network field, maybe it got removed in a recent version. @jipperinbham do you have any thoughts?
Nevermind, I confirmed that it was removed in elasticsearch 2.x, I guess we just need to ignore any errors if this field is not there
yea, several items were removed in the 2.0 release.
If I have time this weekend, I'll send up a PR.
@gseven @descrepes @sparrc my plan for this is to not try and process any of the metrics removed from the 2.0 release rather than try to account for version differences.
If you feel those metrics are critical to keep for versions prior to 2.0, let me know.
There's no network section in Elasticsearch 2.* node statistics.
Trying to get ES statistics causes:
2015/12/10 14:28:52 elasticsearch: got unexpected type with value (network)