Icingabeat template and kibana dashboard not compatible with ELK 7.0

I installed a complete new ELK Stack in our test enviroment and tried to use the icingabeat from our productive icinga2 system. Unfortenetely the elastic search template and the kibana dashboards are not compatible with the new version. When I start icingabeat in the debugging mode

2019-04-18T17:34:41.781+0200    INFO    elasticsearch/client.go:713     Connected to Elasticsearch version 7.0.0
2019-04-18T17:34:41.781+0200    DEBUG   [elasticsearch] elasticsearch/client.go:731     HEAD http://IP:9200/_template/icingabeat-6.5.4  <nil>
2019-04-18T17:34:41.783+0200    INFO    template/load.go:82     Loading template for Elasticsearch version: 7.0.0
2019-04-18T17:34:41.783+0200    DEBUG   [template]      template/load.go:116    Load default fields.yml
2019-04-18T17:34:41.817+0200    DEBUG   [template]      template/load.go:139    Try loading template with name: icingabeat-6.5.4
2019-04-18T17:34:41.820+0200    DEBUG   [elasticsearch] elasticsearch/client.go:731     PUT http://IP:9200/_template/icingabeat-6.5.4  map[index_patterns:[icingabeat-6.5.4-*] mappings:{"doc":{"_meta":{"version":"6.5.4"},"date_detection":false,"dynamic_templates":[{"fields":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"fields.*"}},{"docker.container.labels":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"docker.container.labels.*"}},{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"properties":{"@timestamp":{"type":"date"},"beat":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"}}},"docker":{"properties":{"container":{"properties":{"id":{"ignore_above":1024,"type":"keyword"},"image":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"},"name":{"ignore_above":1024,"type":"keyword"}}}}},"error":{"properties":{"code":{"type":"long"},"message":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"}}},"fields":{"type":"object"},"host":{"properties":{"architecture":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"mac":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"family":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"}}}}},"icinga":{"properties":{"acknowledgement_type":{"type":"long"},"author":{"ignore_above":1024,"type":"keyword"},"check_result":{"properties":{"active":{"type":"boolean"},"check_source":{"ignore_above":1024,"type":"keyword"},"command":{"norms":false,"type":"text"},"execution_end":{"type":"date"},"execution_start":{"type":"date"},"exit_status":{"type":"long"},"output":{"norms":false,"type":"text"},"performance_data":{"norms":false,"type":"text"},"schedule_end":{"type":"date"},"schedule_start":{"type":"date"},"state":{"type":"long"},"ttl":{"type":"long"},"type":{"ignore_above":1024,"type":"keyword"},"vars_after":{"properties":{"attempt":{"type":"long"},"reachable":{"type":"boolean"},"state":{"type":"long"},"state_type":{"type":"long"}}},"vars_before":{"properties":{"attempt":{"type":"long"},"reachable":{"type":"boolean"},"state":{"type":"long"},"state_type":{"type":"long"}}}}},"comment":{"properties":{"__name":{"norms":false,"type":"text"},"author":{"ignore_above":1024,"type":"keyword"},"entry_time":{"type":"date"},"entry_type":{"type":"long"},"expire_time":{"type":"date"},"host_name":{"ignore_above":1024,"type":"keyword"},"legacy_id":{"type":"long"},"name":{"ignore_above":1024,"type":"keyword"},"package":{"ignore_above":1024,"type":"keyword"},"service_name":{"ignore_above":1024,"type":"keyword"},"templates":{"norms":false,"type":"text"},"text":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"zone":{"ignore_above":1024,"type":"keyword"}}},"downtime":{"properties":{"__name":{"norms":false,"type":"text"},"author":{"ignore_above":1024,"type":"keyword"},"comment":{"norms":false,"type":"text"},"config_owner":{"norms":false,"type":"text"},"duration":{"type":"long"},"end_time":{"type":"date"},"entry_time":{"type":"date"},"fixed":{"type":"boolean"},"host_name":{"ignore_above":1024,"type":"keyword"},"legacy_id":{"type":"long"},"name":{"ignore_above":1024,"type":"keyword"},"package":{"ignore_above":1024,"type":"keyword"},"scheduled_by":{"norms":false,"type":"text"},"service_name":{"ignore_above":1024,"type":"keyword"},"start_time":{"type":"date"},"templates":{"norms":false,"type":"text"},"trigger_time":{"type":"date"},"triggered_by":{"norms":false,"type":"text"},"triggers":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"was_cancelled":{"type":"boolean"},"zone":{"ignore_above":1024,"type":"keyword"}}},"expiry":{"type":"date"},"host":{"ignore_above":1024,"type":"keyword"},"notification_type":{"ignore_above":1024,"type":"keyword"},"notify":{"ignore_above":1024,"type":"keyword"},"service":{"ignore_above":1024,"type":"keyword"},"state":{"type":"long"},"state_type":{"type":"long"},"status":{"properties":{"active_host_checks":{"type":"long"},"active_host_checks_15min":{"type":"long"},"active_host_checks_1min":{"type":"long"},"active_host_checks_5min":{"type":"long"},"active_service_checks":{"type":"long"},"active_service_checks_15min":{"type":"long"},"active_service_checks_1min":{"type":"long"},"active_service_checks_5min":{"type":"long"},"api":{"properties":{"identity":{"ignore_above":1024,"type":"keyword"},"num_conn_endpoints":{"type":"long"},"num_endpoints":{"type":"long"},"num_not_conn_endpoints":{"type":"long"}}},"avg_execution_time":{"type":"long"},"avg_latency":{"type":"long"},"checkercomponent":{"properties":{"checker":{"properties":{"idle":{"type":"long"},"pending":{"type":"long"}}}}},"filelogger":{"properties":{"main-log":{"type":"long"}}},"icingaapplication":{"properties":{"app":{"properties":{"enable_event_handlers":{"type":"boolean"},"enable_flapping":{"type":"boolean"},"enable_host_checks":{"type":"boolean"},"enable_notifications":{"type":"boolean"},"enable_perfdata":{"type":"boolean"},"enable_service_checks":{"type":"boolean"},"node_name":{"ignore_above":1024,"type":"keyword"},"pid":{"type":"long"},"program_start":{"type":"long"},"version":{"ignore_above":1024,"type":"keyword"}}}}},"idomysqlconnection":{"properties":{"ido-mysql":{"properties":{"connected":{"type":"boolean"},"instance_name":{"ignore_above":1024,"type":"keyword"},"query_queue_items":{"type":"long"},"version":{"ignore_above":1024,"type":"keyword"}}}}},"max_execution_time":{"type":"long"},"max_latency":{"type":"long"},"min_execution_time":{"type":"long"},"min_latency":{"type":"long"},"notificationcomponent":{"properties":{"notification":{"type":"long"}}},"num_hosts_acknowledged":{"type":"long"},"num_hosts_down":{"type":"long"},"num_hosts_flapping":{"type":"long"},"num_hosts_in_downtime":{"type":"long"},"num_hosts_pending":{"type":"long"},"num_hosts_unreachable":{"type":"long"},"num_hosts_up":{"type":"long"},"num_services_acknowledged":{"type":"long"},"num_services_critical":{"type":"long"},"num_services_flapping":{"type":"long"},"num_services_in_downtime":{"type":"long"},"num_services_ok":{"type":"long"},"num_services_pending":{"type":"long"},"num_services_unknown":{"type":"long"},"num_services_unreachable":{"type":"long"},"num_services_warning":{"type":"long"},"passive_host_checks":{"type":"long"},"passive_host_checks_15min":{"type":"long"},"passive_host_checks_1min":{"type":"long"},"passive_host_checks_5min":{"type":"long"},"passive_service_checks":{"type":"long"},"passive_service_checks_15min":{"type":"long"},"passive_service_checks_1min":{"type":"long"},"passive_service_checks_5min":{"type":"long"},"uptime":{"type":"long"}}},"text":{"norms":false,"type":"text"},"timestamp":{"type":"date"},"type":{"ignore_above":1024,"type":"keyword"},"users":{"ignore_above":1024,"type":"keyword"}}},"kubernetes":{"properties":{"annotations":{"type":"object"},"container":{"properties":{"image":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"}}},"labels":{"type":"object"},"namespace":{"ignore_above":1024,"type":"keyword"},"node":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"pod":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"uid":{"ignore_above":1024,"type":"keyword"}}}}},"meta":{"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"instance_id":{"ignore_above":1024,"type":"keyword"},"instance_name":{"ignore_above":1024,"type":"keyword"},"machine_type":{"ignore_above":1024,"type":"keyword"},"project_id":{"ignore_above":1024,"type":"keyword"},"provider":{"ignore_above":1024,"type":"keyword"},"region":{"ignore_above":1024,"type":"keyword"}}}}},"tags":{"ignore_above":1024,"type":"keyword"},"timestamp":{"type":"date"},"type":{"ignore_above":1024,"type":"keyword"}}}} order:1 settings:{"index":{"mapping":{"total_fields":{"limit":10000}},"number_of_routing_shards":30,"query":{"default_field":["type","icinga.type","icinga.host","icinga.service","icinga.author","icinga.notification_type","icinga.text","icinga.users","icinga.notify","icinga.check_result.check_source","icinga.check_result.command","icinga.check_result.output","icinga.check_result.performance_data","icinga.check_result.type","icinga.comment.__name","icinga.comment.author","icinga.comment.host_name","icinga.comment.name","icinga.comment.package","icinga.comment.service_name","icinga.comment.templates","icinga.comment.text","icinga.comment.type","icinga.comment.version","icinga.comment.zone","icinga.downtime.__name","icinga.downtime.author","icinga.downtime.comment","icinga.downtime.config_owner","icinga.downtime.host_name","icinga.downtime.name","icinga.downtime.package","icinga.downtime.scheduled_by","icinga.downtime.service_name","icinga.downtime.templates","icinga.downtime.triggered_by","icinga.downtime.triggers","icinga.downtime.type","icinga.downtime.version","icinga.downtime.zone","icinga.status.api.identity","icinga.status.icingaapplication.app.node_name","icinga.status.icingaapplication.app.version","icinga.status.idomysqlconnection.ido-mysql.instance_name","icinga.status.idomysqlconnection.ido-mysql.version","beat.name","beat.hostname","beat.timezone","beat.version","tags","error.message","error.type","meta.cloud.provider","meta.cloud.instance_id","meta.cloud.instance_name","meta.cloud.machine_type","meta.cloud.availability_zone","meta.cloud.project_id","meta.cloud.region","docker.container.id","docker.container.image","docker.container.name","host.name","host.id","host.architecture","host.os.platform","host.os.version","host.os.family","host.mac","kubernetes.pod.name","kubernetes.pod.uid","kubernetes.namespace","kubernetes.node.name","kubernetes.container.name","kubernetes.container.image","fields.*"]},"refresh_interval":"5s"}}]

So I think the elasticsearch changed somethink to icingabeat needs a new template. The kibana dashboards are not working too. I can post the debugging log if someone need it at the moment the ELK is down because of easter holiday.

Im available for Tests next week if needed

Enviroment icingabeat version 6.5.4 (amd64), libbeat 6.5.4 icinga2 (version: r2.10.4-1) elastic search { "name" : "HOST", "cluster_name" : "Cluster", "cluster_uuid" : "UUID", "version" : { "number" : "7.0.0", "build_flavor" : "default", "build_type" : "deb", "build_hash" : "b7e28a7", "build_date" : "2019-04-05T22:55:32.697037Z", "build_snapshot" : false, "lucene_version" : "8.0.0", "minimum_wire_compatibility_version" : "6.7.0", "minimum_index_compatibility_version" : "6.0.0-beta1" }, "tagline" : "You Know, for Search" } Kibana 7.0 OS Ubuntu 16.04 LTS on Icinga2 Ubuntu 18.04 LTS for ELK Stack

Here is the output of the manuell try:

curl -XPUT -H 'Content-Type: application/json' 'http://IP:9200/_template/icingabeat-6.5.4-*' -d@/etc/icingabeat/elastic.json
{"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [doc : {_meta={version=6.5.4}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], date_detection=false, properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={uid={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, host={properties={os={properties={family={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, platform={ignore_above=1024, type=keyword}}}, ip={type=ip}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, mac={ignore_above=1024, type=keyword}, architecture={ignore_above=1024, type=keyword}}}, icinga={properties={notification_type={ignore_above=1024, type=keyword}, acknowledgement_type={type=long}, author={ignore_above=1024, type=keyword}, state_type={type=long}, type={ignore_above=1024, type=keyword}, notify={ignore_above=1024, type=keyword}, users={ignore_above=1024, type=keyword}, downtime={properties={__name={norms=false, type=text}, entry_time={type=date}, package={ignore_above=1024, type=keyword}, scheduled_by={norms=false, type=text}, author={ignore_above=1024, type=keyword}, service_name={ignore_above=1024, type=keyword}, was_cancelled={type=boolean}, templates={norms=false, type=text}, end_time={type=date}, config_owner={norms=false, type=text}, triggered_by={norms=false, type=text}, triggers={norms=false, type=text}, type={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, duration={type=long}, start_time={type=date}, trigger_time={type=date}, zone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, legacy_id={type=long}, comment={norms=false, type=text}, fixed={type=boolean}, host_name={ignore_above=1024, type=keyword}}}, check_result={properties={performance_data={norms=false, type=text}, active={type=boolean}, exit_status={type=long}, type={ignore_above=1024, type=keyword}, ttl={type=long}, command={norms=false, type=text}, execution_end={type=date}, output={norms=false, type=text}, vars_after={properties={state_type={type=long}, state={type=long}, attempt={type=long}, reachable={type=boolean}}}, check_source={ignore_above=1024, type=keyword}, schedule_end={type=date}, execution_start={type=date}, state={type=long}, vars_before={properties={state_type={type=long}, state={type=long}, attempt={type=long}, reachable={type=boolean}}}, schedule_start={type=date}}}, service={ignore_above=1024, type=keyword}, host={ignore_above=1024, type=keyword}, comment={properties={__name={norms=false, type=text}, entry_time={type=date}, package={ignore_above=1024, type=keyword}, author={ignore_above=1024, type=keyword}, service_name={ignore_above=1024, type=keyword}, templates={norms=false, type=text}, expire_time={type=date}, type={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, zone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, legacy_id={type=long}, text={norms=false, type=text}, entry_type={type=long}, host_name={ignore_above=1024, type=keyword}}}, expiry={type=date}, state={type=long}, text={norms=false, type=text}, status={properties={num_hosts_up={type=long}, avg_latency={type=long}, passive_service_checks_15min={type=long}, num_hosts_in_downtime={type=long}, num_services_acknowledged={type=long}, passive_host_checks={type=long}, active_service_checks_1min={type=long}, filelogger={properties={main-log={type=long}}}, passive_host_checks_15min={type=long}, num_services_in_downtime={type=long}, passive_service_checks={type=long}, num_hosts_unreachable={type=long}, num_services_flapping={type=long}, active_service_checks_15min={type=long}, api={properties={num_conn_endpoints={type=long}, identity={ignore_above=1024, type=keyword}, num_not_conn_endpoints={type=long}, num_endpoints={type=long}}}, active_host_checks_1min={type=long}, icingaapplication={properties={app={properties={enable_notifications={type=boolean}, enable_event_handlers={type=boolean}, enable_perfdata={type=boolean}, node_name={ignore_above=1024, type=keyword}, program_start={type=long}, pid={type=long}, enable_host_checks={type=boolean}, enable_service_checks={type=boolean}, version={ignore_above=1024, type=keyword}, enable_flapping={type=boolean}}}}}, num_services_ok={type=long}, active_host_checks={type=long}, checkercomponent={properties={checker={properties={idle={type=long}, pending={type=long}}}}}, active_host_checks_5min={type=long}, min_execution_time={type=long}, min_latency={type=long}, active_host_checks_15min={type=long}, num_hosts_flapping={type=long}, active_service_checks={type=long}, num_services_unknown={type=long}, notificationcomponent={properties={notification={type=long}}}, passive_host_checks_1min={type=long}, passive_service_checks_1min={type=long}, avg_execution_time={type=long}, num_services_unreachable={type=long}, passive_host_checks_5min={type=long}, max_execution_time={type=long}, active_service_checks_5min={type=long}, num_hosts_pending={type=long}, uptime={type=long}, max_latency={type=long}, num_services_critical={type=long}, passive_service_checks_5min={type=long}, num_hosts_acknowledged={type=long}, idomysqlconnection={properties={ido-mysql={properties={connected={type=boolean}, instance_name={ignore_above=1024, type=keyword}, query_queue_items={type=long}, version={ignore_above=1024, type=keyword}}}}}, num_services_pending={type=long}, num_hosts_down={type=long}, num_services_warning={type=long}}}, timestamp={type=date}}}, error={properties={code={type=long}, message={norms=false, type=text}, type={ignore_above=1024, type=keyword}}}, fields={type=object}, type={ignore_above=1024, type=keyword}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}, timestamp={type=date}}}]"}],"type":"mapper_parsing_exception","reason":"Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [doc : {_meta={version=6.5.4}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], date_detection=false, properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={uid={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, host={properties={os={properties={family={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, platform={ignore_above=1024, type=keyword}}}, ip={type=ip}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, mac={ignore_above=1024, type=keyword}, architecture={ignore_above=1024, type=keyword}}}, icinga={properties={notification_type={ignore_above=1024, type=keyword}, acknowledgement_type={type=long}, author={ignore_above=1024, type=keyword}, state_type={type=long}, type={ignore_above=1024, type=keyword}, notify={ignore_above=1024, type=keyword}, users={ignore_above=1024, type=keyword}, downtime={properties={__name={norms=false, type=text}, entry_time={type=date}, package={ignore_above=1024, type=keyword}, scheduled_by={norms=false, type=text}, author={ignore_above=1024, type=keyword}, service_name={ignore_above=1024, type=keyword}, was_cancelled={type=boolean}, templates={norms=false, type=text}, end_time={type=date}, config_owner={norms=false, type=text}, triggered_by={norms=false, type=text}, triggers={norms=false, type=text}, type={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, duration={type=long}, start_time={type=date}, trigger_time={type=date}, zone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, legacy_id={type=long}, comment={norms=false, type=text}, fixed={type=boolean}, host_name={ignore_above=1024, type=keyword}}}, check_result={properties={performance_data={norms=false, type=text}, active={type=boolean}, exit_status={type=long}, type={ignore_above=1024, type=keyword}, ttl={type=long}, command={norms=false, type=text}, execution_end={type=date}, output={norms=false, type=text}, vars_after={properties={state_type={type=long}, state={type=long}, attempt={type=long}, reachable={type=boolean}}}, check_source={ignore_above=1024, type=keyword}, schedule_end={type=date}, execution_start={type=date}, state={type=long}, vars_before={properties={state_type={type=long}, state={type=long}, attempt={type=long}, reachable={type=boolean}}}, schedule_start={type=date}}}, service={ignore_above=1024, type=keyword}, host={ignore_above=1024, type=keyword}, comment={properties={__name={norms=false, type=text}, entry_time={type=date}, package={ignore_above=1024, type=keyword}, author={ignore_above=1024, type=keyword}, service_name={ignore_above=1024, type=keyword}, templates={norms=false, type=text}, expire_time={type=date}, type={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, zone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, legacy_id={type=long}, text={norms=false, type=text}, entry_type={type=long}, host_name={ignore_above=1024, type=keyword}}}, expiry={type=date}, state={type=long}, text={norms=false, type=text}, status={properties={num_hosts_up={type=long}, avg_latency={type=long}, passive_service_checks_15min={type=long}, num_hosts_in_downtime={type=long}, num_services_acknowledged={type=long}, passive_host_checks={type=long}, active_service_checks_1min={type=long}, filelogger={properties={main-log={type=long}}}, passive_host_checks_15min={type=long}, num_services_in_downtime={type=long}, passive_service_checks={type=long}, num_hosts_unreachable={type=long}, num_services_flapping={type=long}, active_service_checks_15min={type=long}, api={properties={num_conn_endpoints={type=long}, identity={ignore_above=1024, type=keyword}, num_not_conn_endpoints={type=long}, num_endpoints={type=long}}}, active_host_checks_1min={type=long}, icingaapplication={properties={app={properties={enable_notifications={type=boolean}, enable_event_handlers={type=boolean}, enable_perfdata={type=boolean}, node_name={ignore_above=1024, type=keyword}, program_start={type=long}, pid={type=long}, enable_host_checks={type=boolean}, enable_service_checks={type=boolean}, version={ignore_above=1024, type=keyword}, enable_flapping={type=boolean}}}}}, num_services_ok={type=long}, active_host_checks={type=long}, checkercomponent={properties={checker={properties={idle={type=long}, pending={type=long}}}}}, active_host_checks_5min={type=long}, min_execution_time={type=long}, min_latency={type=long}, active_host_checks_15min={type=long}, num_hosts_flapping={type=long}, active_service_checks={type=long}, num_services_unknown={type=long}, notificationcomponent={properties={notification={type=long}}}, passive_host_checks_1min={type=long}, passive_service_checks_1min={type=long}, avg_execution_time={type=long}, num_services_unreachable={type=long}, passive_host_checks_5min={type=long}, max_execution_time={type=long}, active_service_checks_5min={type=long}, num_hosts_pending={type=long}, uptime={type=long}, max_latency={type=long}, num_services_critical={type=long}, passive_service_checks_5min={type=long}, num_hosts_acknowledged={type=long}, idomysqlconnection={properties={ido-mysql={properties={connected={type=boolean}, instance_name={ignore_above=1024, type=keyword}, query_queue_items={type=long}, version={ignore_above=1024, type=keyword}}}}}, num_services_pending={type=long}, num_hosts_down={type=long}, num_services_warning={type=long}}}, timestamp={type=date}}}, error={properties={code={type=long}, message={norms=false, type=text}, type={ignore_above=1024, type=keyword}}}, fields={type=object}, type={ignore_above=1024, type=keyword}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}, timestamp={type=date}}}]","caused_by":{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters:  [doc : {_meta={version=6.5.4}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], date_detection=false, properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={uid={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, host={properties={os={properties={family={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, platform={ignore_above=1024, type=keyword}}}, ip={type=ip}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, mac={ignore_above=1024, type=keyword}, architecture={ignore_above=1024, type=keyword}}}, icinga={properties={notification_type={ignore_above=1024, type=keyword}, acknowledgement_type={type=long}, author={ignore_above=1024, type=keyword}, state_type={type=long}, type={ignore_above=1024, type=keyword}, notify={ignore_above=1024, type=keyword}, users={ignore_above=1024, type=keyword}, downtime={properties={__name={norms=false, type=text}, entry_time={type=date}, package={ignore_above=1024, type=keyword}, scheduled_by={norms=false, type=text}, author={ignore_above=1024, type=keyword}, service_name={ignore_above=1024, type=keyword}, was_cancelled={type=boolean}, templates={norms=false, type=text}, end_time={type=date}, config_owner={norms=false, type=text}, triggered_by={norms=false, type=text}, triggers={norms=false, type=text}, type={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, duration={type=long}, start_time={type=date}, trigger_time={type=date}, zone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, legacy_id={type=long}, comment={norms=false, type=text}, fixed={type=boolean}, host_name={ignore_above=1024, type=keyword}}}, check_result={properties={performance_data={norms=false, type=text}, active={type=boolean}, exit_status={type=long}, type={ignore_above=1024, type=keyword}, ttl={type=long}, command={norms=false, type=text}, execution_end={type=date}, output={norms=false, type=text}, vars_after={properties={state_type={type=long}, state={type=long}, attempt={type=long}, reachable={type=boolean}}}, check_source={ignore_above=1024, type=keyword}, schedule_end={type=date}, execution_start={type=date}, state={type=long}, vars_before={properties={state_type={type=long}, state={type=long}, attempt={type=long}, reachable={type=boolean}}}, schedule_start={type=date}}}, service={ignore_above=1024, type=keyword}, host={ignore_above=1024, type=keyword}, comment={properties={__name={norms=false, type=text}, entry_time={type=date}, package={ignore_above=1024, type=keyword}, author={ignore_above=1024, type=keyword}, service_name={ignore_above=1024, type=keyword}, templates={norms=false, type=text}, expire_time={type=date}, type={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}, zone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, legacy_id={type=long}, text={norms=false, type=text}, entry_type={type=long}, host_name={ignore_above=1024, type=keyword}}}, expiry={type=date}, state={type=long}, text={norms=false, type=text}, status={properties={num_hosts_up={type=long}, avg_latency={type=long}, passive_service_checks_15min={type=long}, num_hosts_in_downtime={type=long}, num_services_acknowledged={type=long}, passive_host_checks={type=long}, active_service_checks_1min={type=long}, filelogger={properties={main-log={type=long}}}, passive_host_checks_15min={type=long}, num_services_in_downtime={type=long}, passive_service_checks={type=long}, num_hosts_unreachable={type=long}, num_services_flapping={type=long}, active_service_checks_15min={type=long}, api={properties={num_conn_endpoints={type=long}, identity={ignore_above=1024, type=keyword}, num_not_conn_endpoints={type=long}, num_endpoints={type=long}}}, active_host_checks_1min={type=long}, icingaapplication={properties={app={properties={enable_notifications={type=boolean}, enable_event_handlers={type=boolean}, enable_perfdata={type=boolean}, node_name={ignore_above=1024, type=keyword}, program_start={type=long}, pid={type=long}, enable_host_checks={type=boolean}, enable_service_checks={type=boolean}, version={ignore_above=1024, type=keyword}, enable_flapping={type=boolean}}}}}, num_services_ok={type=long}, active_host_checks={type=long}, checkercomponent={properties={checker={properties={idle={type=long}, pending={type=long}}}}}, active_host_checks_5min={type=long}, min_execution_time={type=long}, min_latency={type=long}, active_host_checks_15min={type=long}, num_hosts_flapping={type=long}, active_service_checks={type=long}, num_services_unknown={type=long}, notificationcomponent={properties={notification={type=long}}}, passive_host_checks_1min={type=long}, passive_service_checks_1min={type=long}, avg_execution_time={type=long}, num_services_unreachable={type=long}, passive_host_checks_5min={type=long}, max_execution_time={type=long}, active_service_checks_5min={type=long}, num_hosts_pending={type=long}, uptime={type=long}, max_latency={type=long}, num_services_critical={type=long}, passive_service_checks_5min={type=long}, num_hosts_acknowledged={type=long}, idomysqlconnection={properties={ido-mysql={properties={connected={type=boolean}, instance_name={ignore_above=1024, type=keyword}, query_queue_items={type=long}, version={ignore_above=1024, type=keyword}}}}}, num_services_pending={type=long}, num_hosts_down={type=long}, num_services_warning={type=long}}}, timestamp={type=date}}}, error={properties={code={type=long}, message={norms=false, type=text}, type={ignore_above=1024, type=keyword}}}, fields={type=object}, type={ignore_above=1024, type=keyword}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}, timestamp={type=date}}}]"}},"status":400}

My icingabeat isn't running too since elastic 7

Works well with ES 6.6.0 but has a template problem with ES v 7.x

ERROR pipeline/output.go:100 Failed to connect to backoff(elasticsearch(http://72.34.137.3:9200)): Connection marked as failed because the onConnect callback failed: Error loading Elasticsearch template: could not load template. Elasticsearch returned: couldn't load template: couldn't load json. Error: 400 Bad Request: {"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters: [doc : {_meta={version=6.5.4}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, ...

Created the template manually via ES Dev Tools with: ` POST _template/icingabeat-6.5.4?include_type_name=true { "index_patterns": [ "icingabeat-6.5.4-" ], "mappings": { "doc": { "_meta": { "version": "6.5.4" }, "date_detection": false, "dynamic_templates": [ { "fields": { "mapping": { "type": "keyword" }, "match_mapping_type": "string", "path_match": "fields." } }, { "docker.container.labels": { "mapping": { "type": "keyword" }, "match_mapping_type": "string", "path_match": "docker.container.labels.*" } }, { "strings_as_keyword": { "mapping": { "ignore_above": 1024, "type": "keyword" }, "match_mapping_type": "string" } } ], "properties": { "@timestamp": { "type": "date" }, "beat": { "properties": { "hostname": { "ignore_above": 1024, "type": "keyword" }, "name": { "ignore_above": 1024, "type": "keyword" }, "timezone": { "ignore_above": 1024, "type": "keyword" }, "version": { "ignore_above": 1024, "type": "keyword" } } }, "docker": { "properties": { "container": { "properties": { "id": { "ignore_above": 1024, "type": "keyword" }, "image": { "ignore_above": 1024, "type": "keyword" }, "labels": { "type": "object" }, "name": { "ignore_above": 1024, "type": "keyword" } } } } }, "error": { "properties": { "code": { "type": "long" }, "message": { "norms": false, "type": "text" }, "type": { "ignore_above": 1024, "type": "keyword" } } }, "fields": { "type": "object" }, "host": { "properties": { "architecture": { "ignore_above": 1024, "type": "keyword" }, "id": { "ignore_above": 1024, "type": "keyword" }, "ip": { "type": "ip" }, "mac": { "ignore_above": 1024, "type": "keyword" }, "name": { "ignore_above": 1024, "type": "keyword" }, "os": { "properties": { "family": { "ignore_above": 1024, "type": "keyword" }, "platform": { "ignore_above": 1024, "type": "keyword" }, "version": { "ignore_above": 1024, "type": "keyword" } } } } }, "icinga": { "properties": { "acknowledgement_type": { "type": "long" }, "author": { "ignore_above": 1024, "type": "keyword" }, "check_result": { "properties": { "active": { "type": "boolean" }, "check_source": { "ignore_above": 1024, "type": "keyword" }, "command": { "norms": false, "type": "text" }, "execution_end": { "type": "date" }, "execution_start": { "type": "date" }, "exit_status": { "type": "long" }, "output": { "norms": false, "type": "text" }, "performance_data": { "norms": false, "type": "text" }, "schedule_end": { "type": "date" }, "schedule_start": { "type": "date" }, "state": { "type": "long" }, "ttl": { "type": "long" }, "type": { "ignore_above": 1024, "type": "keyword" }, "vars_after": { "properties": { "attempt": { "type": "long" }, "reachable": { "type": "boolean" }, "state": { "type": "long" }, "state_type": { "type": "long" } } }, "vars_before": { "properties": { "attempt": { "type": "long" }, "reachable": { "type": "boolean" }, "state": { "type": "long" }, "state_type": { "type": "long" } } } } }, "comment": { "properties": { "name": { "norms": false, "type": "text" }, "author": { "ignore_above": 1024, "type": "keyword" }, "entry_time": { "type": "date" }, "entry_type": { "type": "long" }, "expire_time": { "type": "date" }, "host_name": { "ignore_above": 1024, "type": "keyword" }, "legacy_id": { "type": "long" }, "name": { "ignore_above": 1024, "type": "keyword" }, "package": { "ignore_above": 1024, "type": "keyword" }, "service_name": { "ignore_above": 1024, "type": "keyword" }, "templates": { "norms": false, "type": "text" }, "text": { "norms": false, "type": "text" }, "type": { "ignore_above": 1024, "type": "keyword" }, "version": { "ignore_above": 1024, "type": "keyword" }, "zone": { "ignore_above": 1024, "type": "keyword" } } }, "downtime": { "properties": { "name": { "norms": false, "type": "text" }, "author": { "ignore_above": 1024, "type": "keyword" }, "comment": { "norms": false, "type": "text" }, "config_owner": { "norms": false, "type": "text" }, "duration": { "type": "long" }, "end_time": { "type": "date" }, "entry_time": { "type": "date" }, "fixed": { "type": "boolean" }, "host_name": { "ignore_above": 1024, "type": "keyword" }, "legacy_id": { "type": "long" }, "name": { "ignore_above": 1024, "type": "keyword" }, "package": { "ignore_above": 1024, "type": "keyword" }, "scheduled_by": { "norms": false, "type": "text" }, "service_name": { "ignore_above": 1024, "type": "keyword" }, "start_time": { "type": "date" }, "templates": { "norms": false, "type": "text" }, "trigger_time": { "type": "date" }, "triggered_by": { "norms": false, "type": "text" }, "triggers": { "norms": false, "type": "text" }, "type": { "ignore_above": 1024, "type": "keyword" }, "version": { "ignore_above": 1024, "type": "keyword" }, "was_cancelled": { "type": "boolean" }, "zone": { "ignore_above": 1024, "type": "keyword" } } }, "expiry": { "type": "date" }, "host": { "ignore_above": 1024, "type": "keyword" }, "notification_type": { "ignore_above": 1024, "type": "keyword" }, "notify": { "ignore_above": 1024, "type": "keyword" }, "service": { "ignore_above": 1024, "type": "keyword" }, "state": { "type": "long" }, "state_type": { "type": "long" }, "status": { "properties": { "active_host_checks": { "type": "long" }, "active_host_checks_15min": { "type": "long" }, "active_host_checks_1min": { "type": "long" }, "active_host_checks_5min": { "type": "long" }, "active_service_checks": { "type": "long" }, "active_service_checks_15min": { "type": "long" }, "active_service_checks_1min": { "type": "long" }, "active_service_checks_5min": { "type": "long" }, "api": { "properties": { "identity": { "ignore_above": 1024, "type": "keyword" }, "num_conn_endpoints": { "type": "long" }, "num_endpoints": { "type": "long" }, "num_not_conn_endpoints": { "type": "long" } } }, "avg_execution_time": { "type": "long"

            },
            "avg_latency": {
              "type": "long"
            },
            "checkercomponent": {
              "properties": {
                "checker": {
                  "properties": {
                    "idle": {
                      "type": "long"
                    },
                    "pending": {
                      "type": "long"
                    }
                  }
                }
              }
            },
            "filelogger": {
              "properties": {
                "main-log": {
                  "type": "long"
                }
              }
            },
            "icingaapplication": {
              "properties": {
                "app": {
                  "properties": {
                    "enable_event_handlers": {
                      "type": "boolean"
                    },
                    "enable_flapping": {
                      "type": "boolean"
                    },
                    "enable_host_checks": {
                      "type": "boolean"
                    },
                    "enable_notifications": {
                      "type": "boolean"
                    },
                    "enable_perfdata": {
                      "type": "boolean"
                    },
                    "enable_service_checks": {
                      "type": "boolean"
                    },
                    "node_name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "pid": {
                      "type": "long"
                    },
                    "program_start": {
                      "type": "long"
                    },
                    "version": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                }
              }
            },
            "idomysqlconnection": {
              "properties": {
                "ido-mysql": {
                  "properties": {
                    "connected": {
                      "type": "boolean"
                    },
                    "instance_name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "query_queue_items": {
                      "type": "long"
                    },
                    "version": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                }
              }
            },
            "max_execution_time": {
              "type": "long"
            },

         "max_latency": {
              "type": "long"
            },
            "min_execution_time": {
              "type": "long"
            },
            "min_latency": {
              "type": "long"
            },
            "notificationcomponent": {
              "properties": {
                "notification": {
                  "type": "long"
                }
              }
            },
            "num_hosts_acknowledged": {
              "type": "long"
            },
            "num_hosts_down": {
              "type": "long"
            },
            "num_hosts_flapping": {
              "type": "long"
            },
            "num_hosts_in_downtime": {
              "type": "long"
            },
            "num_hosts_pending": {
              "type": "long"
            },
            "num_hosts_unreachable": {
              "type": "long"
            },
            "num_hosts_up": {
              "type": "long"
            },
            "num_services_acknowledged": {
              "type": "long"
            },
            "num_services_critical": {
              "type": "long"
            },
            "num_services_flapping": {
              "type": "long"
            },
            "num_services_in_downtime": {
              "type": "long"
            },
            "num_services_ok": {
              "type": "long"
            },
            "num_services_pending": {
              "type": "long"
            },
            "num_services_unknown": {
              "type": "long"
            },
            "num_services_unreachable": {
              "type": "long"
            },
            "num_services_warning": {
              "type": "long"
            },
            "passive_host_checks": {
              "type": "long"
            },
            "passive_host_checks_15min": {
              "type": "long"
            },
            "passive_host_checks_1min": {
              "type": "long"
            },
            "passive_host_checks_5min": {
              "type": "long"
            },
            "passive_service_checks": {
              "type": "long"
            },
            "passive_service_checks_15min": {
              "type": "long"
            },
            "passive_service_checks_1min": {
              "type": "long"
            },
            "passive_service_checks_5min": {
              "type": "long"
            },
            "uptime": {
              "type": "long"

}
          }
        },
        "text": {
          "norms": false,
          "type": "text"
        },
        "timestamp": {
          "type": "date"
        },
        "type": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "users": {
          "ignore_above": 1024,
          "type": "keyword"
        }
      }
    },
    "kubernetes": {
      "properties": {
        "annotations": {
          "type": "object"
        },
        "container": {
          "properties": {
            "image": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        },
        "labels": {
          "type": "object"
        },
        "namespace": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "node": {
          "properties": {
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        },
        "pod": {
          "properties": {
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "uid": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        }
      }
    },
    "meta": {
      "properties": {
        "cloud": {
          "properties": {
            "availability_zone": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "instance_id": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "instance_name": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "machine_type": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "project_id": {
              "ignore_above": 1024,
              "type": "keyword"
            },

         "provider": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "region": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        }
      }
    },
    "tags": {
      "ignore_above": 1024,
      "type": "keyword"
    },
    "timestamp": {
      "type": "date"
    },
    "type": {
      "ignore_above": 1024,
      "type": "keyword"
    }
  }
}

}, "order": 1, "settings": { "index": { "mapping": { "total_fields": { "limit": 10000 } }, "number_of_routing_shards": 30, "refresh_interval": "5s" } } } `

Ran into a second problem:

` (status=400): {"type":"mapper_parsing_exception","reason":"failed to parse field [icinga.timestamp] of type [date] in document with id '-8gbPW4BvKcUFBPr_vJk'. Preview of field's value: '1.572983275080961E9'","caused_by":{"type":"illegal_argument_exception","reason":"failed to parse date field [1.572983275080961e+09] with format [strict_date_optional_time||epoch_millis||epoch_second]","caused_by":{"type":"date_time_parse_exception","reason":"date_time_parse_exception: Failed to parse with all enclosed parsers"}}}

And as per https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html Date formats behavioural changeedit The epoch_millis and epoch_second formatters no longer support scientific notation. `

Should work now with the latest release, packages on packages.icinga.com will follow soon, but they're already available on GitHub: https://github.com/Icinga/icingabeat/releases/tag/v7.4.2

Feedback is welcome!

Icinga / icingabeat

Icingabeat template and kibana dashboard not compatible with ELK 7.0 #32