elasticmachine
commented
3 days ago Pinging @elastic/obs-ux-infra_services-team (Team:obs-ux-infra_services)
Open Erikg346 opened 1 week ago
elasticmachine
commented
3 days ago Pinging @elastic/obs-ux-infra_services-team (Team:obs-ux-infra_services)
cauemarcondes
commented
3 days ago Hi @Erikg346, When have you enabled it for the first time? Before or after the migration?
Can you share the output of this query, please? You can run it on Kibana devtools.
kbn:/internal/entities/definition?includeState=true
Erikg346
commented
3 days ago Hey @cauemarcondes
We enabled this after we upgraded to 8.16.0. I clicked on the enable button and it triggered the creation of the new indicies. Here is the output:
{
"definitions": [
{
"id": "builtin_containers_from_ecs_data",
"version": "0.1.0",
"name": "Containers from ECS data",
"description": "This definition extracts container entities from common data streams by looking for the ECS field container.id",
"type": "container",
"indexPatterns": [
"filebeat-*",
"logs-*",
"metrics-*",
"metricbeat-*"
],
"identityFields": [
{
"field": "container.id",
"optional": false
}
],
"displayNameTemplate": "{{container.id}}",
"metadata": [
{
"source": "_index",
"destination": "source_index",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "data_stream.type",
"destination": "source_data_stream.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "data_stream.dataset",
"destination": "source_data_stream.dataset",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "container.name",
"destination": "container.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "container.image.name",
"destination": "container.image.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "container.image.tag",
"destination": "container.image.tag",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "container.runtime",
"destination": "container.runtime",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.name",
"destination": "host.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.ip",
"destination": "host.ip",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.mac",
"destination": "host.mac",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.architecture",
"destination": "host.architecture",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.family",
"destination": "host.os.family",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.kernel",
"destination": "host.os.kernel",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.name",
"destination": "host.os.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.platform",
"destination": "host.os.platform",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.type",
"destination": "host.os.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.version",
"destination": "host.os.version",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.provider",
"destination": "cloud.provider",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.region",
"destination": "cloud.region",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.availability_zone",
"destination": "cloud.availability_zone",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.instance.id",
"destination": "cloud.instance.id",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.instance.name",
"destination": "cloud.instance.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.machine.type",
"destination": "cloud.machine.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.service.name",
"destination": "cloud.service.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.name",
"destination": "agent.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.type",
"destination": "agent.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.ephemeral_id",
"destination": "agent.ephemeral_id",
"aggregation": {
"type": "terms",
"limit": 10
}
}
],
"managed": true,
"latest": {
"timestampField": "@timestamp",
"lookbackPeriod": "10m",
"settings": {
"frequency": "5m"
}
},
"installStatus": "installed",
"installStartedAt": "2024-11-13T15:31:06.704Z",
"installedComponents": [
{
"type": "template",
"id": "entities_v1_latest_builtin_containers_from_ecs_data_index_template"
},
{
"type": "ingest_pipeline",
"id": "entities-v1-latest-builtin_containers_from_ecs_data"
},
{
"type": "transform",
"id": "entities-v1-latest-builtin_containers_from_ecs_data"
}
],
"state": {
"installed": true,
"running": true,
"components": {
"transforms": [
{
"id": "entities-v1-latest-builtin_containers_from_ecs_data",
"installed": true,
"running": true,
"stats": {
"id": "entities-v1-latest-builtin_containers_from_ecs_data",
"state": "started",
"node": {
"id": "YH61oXleS7a2knmzkn9r6g",
"name": "ElasticDev1",
"ephemeral_id": "YyyrHrnPRea2a5gm39HjLw",
"transport_address": "<redacted>:9300",
"attributes": {}
},
"stats": {
"pages_processed": 4874,
"documents_processed": 4098908,
"documents_indexed": 2442,
"documents_deleted": 0,
"trigger_count": 1301,
"index_time_in_ms": 6825,
"index_total": 1219,
"index_failures": 0,
"search_time_in_ms": 1083902,
"search_total": 4874,
"search_failures": 0,
"processing_time_in_ms": 0,
"processing_total": 4874,
"delete_time_in_ms": 0,
"exponential_avg_checkpoint_duration_ms": 482.9380113245745,
"exponential_avg_documents_indexed": 2.0000000000000004,
"exponential_avg_documents_processed": 1705.9914684785526
},
"checkpointing": {
"last": {
"checkpoint": 1219,
"timestamp_millis": 1732031657270,
"time_upper_bound_millis": 1732031597270
},
"changes_last_detected_at": 1732031655446,
"last_search_time": 1732031655446
},
"health": {
"status": "green"
}
}
}
],
"ingestPipelines": [
{
"id": "entities-v1-latest-builtin_containers_from_ecs_data",
"installed": true,
"stats": {
"count": 2442,
"failed": 0
}
}
],
"indexTemplates": [
{
"id": "entities_v1_latest_builtin_containers_from_ecs_data_index_template",
"installed": true,
"stats": {
"index_patterns": [
".entities.v1.latest.builtin_containers_from_ecs_data"
],
"template": {
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": "2000"
}
}
}
},
"mappings": {
"_meta": {
"version": "1.6.0"
},
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"text": {
"type": "text"
}
}
},
"match_mapping_type": "string"
}
},
{
"entity_metrics": {
"path_match": "entity.metrics.*",
"mapping": {
"type": "{dynamic_type}"
},
"match_mapping_type": [
"long",
"double"
]
}
}
],
"date_detection": false
},
"aliases": {
"entities-container-latest": {}
}
},
"composed_of": [
"entities_v1_latest_base",
"entities_v1_entity",
"entities_v1_event"
],
"priority": 200,
"_meta": {
"managed_by": "elastic_entity_model",
"managed": true,
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",
"ecs_version": "8.0.0"
},
"ignore_missing_component_templates": []
}
}
]
}
}
},
{
"id": "builtin_hosts_from_ecs_data",
"version": "0.1.0",
"name": "Hosts from ECS data",
"description": "This definition extracts host entities from common data streams by looking for the ECS field host.name",
"type": "host",
"indexPatterns": [
"filebeat-*",
"logs-*",
"metrics-*",
"metricbeat-*"
],
"identityFields": [
{
"field": "host.name",
"optional": false
}
],
"displayNameTemplate": "{{host.name}}",
"metadata": [
{
"source": "_index",
"destination": "source_index",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "data_stream.type",
"destination": "source_data_stream.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "data_stream.dataset",
"destination": "source_data_stream.dataset",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.hostname",
"destination": "host.hostname",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.ip",
"destination": "host.ip",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.mac",
"destination": "host.mac",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.architecture",
"destination": "host.architecture",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.containerized",
"destination": "host.containerized",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.platform",
"destination": "host.os.platform",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.name",
"destination": "host.os.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.type",
"destination": "host.os.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.codename",
"destination": "host.os.codename",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.family",
"destination": "host.os.family",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.kernel",
"destination": "host.os.kernel",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "host.os.version",
"destination": "host.os.version",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.provider",
"destination": "cloud.provider",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.region",
"destination": "cloud.region",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.availability_zone",
"destination": "cloud.availability_zone",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.instance.id",
"destination": "cloud.instance.id",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.instance.name",
"destination": "cloud.instance.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.service.name",
"destination": "cloud.service.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.machine.type",
"destination": "cloud.machine.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.account.id",
"destination": "cloud.account.id",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.project.id",
"destination": "cloud.project.id",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.id",
"destination": "agent.id",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.name",
"destination": "agent.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.type",
"destination": "agent.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.version",
"destination": "agent.version",
"aggregation": {
"type": "terms",
"limit": 10
}
}
],
"managed": true,
"latest": {
"timestampField": "@timestamp",
"lookbackPeriod": "10m",
"settings": {
"frequency": "5m"
}
},
"installStatus": "installed",
"installStartedAt": "2024-11-13T15:31:06.701Z",
"installedComponents": [
{
"type": "template",
"id": "entities_v1_latest_builtin_hosts_from_ecs_data_index_template"
},
{
"type": "ingest_pipeline",
"id": "entities-v1-latest-builtin_hosts_from_ecs_data"
},
{
"type": "transform",
"id": "entities-v1-latest-builtin_hosts_from_ecs_data"
}
],
"state": {
"installed": true,
"running": true,
"components": {
"transforms": [
{
"id": "entities-v1-latest-builtin_hosts_from_ecs_data",
"installed": true,
"running": true,
"stats": {
"id": "entities-v1-latest-builtin_hosts_from_ecs_data",
"state": "started",
"node": {
"id": "YH61oXleS7a2knmzkn9r6g",
"name": "ElasticDev1",
"ephemeral_id": "YyyrHrnPRea2a5gm39HjLw",
"transport_address": "<redacted>:9300",
"attributes": {}
},
"stats": {
"pages_processed": 9108,
"documents_processed": 296159879,
"documents_indexed": 946465,
"documents_deleted": 0,
"trigger_count": 1301,
"index_time_in_ms": 225623,
"index_total": 2631,
"index_failures": 0,
"search_time_in_ms": 2114398,
"search_total": 9108,
"search_failures": 0,
"processing_time_in_ms": 41856,
"processing_total": 9108,
"delete_time_in_ms": 0,
"exponential_avg_checkpoint_duration_ms": 2102.4709317405604,
"exponential_avg_documents_indexed": 1285.1823384100255,
"exponential_avg_documents_processed": 282728.1173330538
},
"checkpointing": {
"last": {
"checkpoint": 1219,
"timestamp_millis": 1732031657270,
"time_upper_bound_millis": 1732031597270
},
"changes_last_detected_at": 1732031655446,
"last_search_time": 1732031655446
},
"health": {
"status": "green"
}
}
}
],
"ingestPipelines": [
{
"id": "entities-v1-latest-builtin_hosts_from_ecs_data",
"installed": true,
"stats": {
"count": 946465,
"failed": 0
}
}
],
"indexTemplates": [
{
"id": "entities_v1_latest_builtin_hosts_from_ecs_data_index_template",
"installed": true,
"stats": {
"index_patterns": [
".entities.v1.latest.builtin_hosts_from_ecs_data"
],
"template": {
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": "2000"
}
}
}
},
"mappings": {
"_meta": {
"version": "1.6.0"
},
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"text": {
"type": "text"
}
}
},
"match_mapping_type": "string"
}
},
{
"entity_metrics": {
"path_match": "entity.metrics.*",
"mapping": {
"type": "{dynamic_type}"
},
"match_mapping_type": [
"long",
"double"
]
}
}
],
"date_detection": false
},
"aliases": {
"entities-host-latest": {}
}
},
"composed_of": [
"entities_v1_latest_base",
"entities_v1_entity",
"entities_v1_event"
],
"priority": 200,
"_meta": {
"managed_by": "elastic_entity_model",
"managed": true,
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",
"ecs_version": "8.0.0"
},
"ignore_missing_component_templates": []
}
}
]
}
}
},
{
"id": "builtin_services_from_ecs_data",
"version": "0.4.0",
"name": "Services from ECS data",
"description": "This definition extracts service entities from common data streams by looking for the ECS field service.name",
"type": "service",
"indexPatterns": [
"logs-*",
"filebeat*",
"traces-apm*"
],
"identityFields": [
{
"field": "service.name",
"optional": false
}
],
"displayNameTemplate": "{{service.name}}",
"metadata": [
{
"source": "_index",
"destination": "source_index",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "data_stream.type",
"destination": "source_data_stream.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "data_stream.dataset",
"destination": "source_data_stream.dataset",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "agent.name",
"destination": "agent.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.environment",
"destination": "service.environment",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.name",
"destination": "service.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.namespace",
"destination": "service.namespace",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.version",
"destination": "service.version",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.runtime.name",
"destination": "service.runtime.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.runtime.version",
"destination": "service.runtime.version",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "service.language.name",
"destination": "service.language.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.provider",
"destination": "cloud.provider",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.availability_zone",
"destination": "cloud.availability_zone",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "cloud.machine.type",
"destination": "cloud.machine.type",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "kubernetes.namespace",
"destination": "kubernetes.namespace",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "orchestrator.cluster.name",
"destination": "orchestrator.cluster.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "k8s.namespace.name",
"destination": "k8s.namespace.name",
"aggregation": {
"type": "terms",
"limit": 10
}
},
{
"source": "k8s.cluster.name",
"destination": "k8s.cluster.name",
"aggregation": {
"type": "terms",
"limit": 10
}
}
],
"managed": true,
"latest": {
"timestampField": "@timestamp",
"lookbackPeriod": "10m",
"settings": {
"syncDelay": "2m",
"frequency": "2m"
}
},
"installStatus": "installed",
"installStartedAt": "2024-11-13T15:31:06.703Z",
"installedComponents": [
{
"type": "template",
"id": "entities_v1_latest_builtin_services_from_ecs_data_index_template"
},
{
"type": "ingest_pipeline",
"id": "entities-v1-latest-builtin_services_from_ecs_data"
},
{
"type": "transform",
"id": "entities-v1-latest-builtin_services_from_ecs_data"
}
],
"state": {
"installed": true,
"running": true,
"components": {
"transforms": [
{
"id": "entities-v1-latest-builtin_services_from_ecs_data",
"installed": true,
"running": true,
"stats": {
"id": "entities-v1-latest-builtin_services_from_ecs_data",
"state": "indexing",
"node": {
"id": "YH61oXleS7a2knmzkn9r6g",
"name": "ElasticDev1",
"ephemeral_id": "YyyrHrnPRea2a5gm39HjLw",
"transport_address": "10.199.10.41:9300",
"attributes": {}
},
"stats": {
"pages_processed": 11194,
"documents_processed": 100325087,
"documents_indexed": 8639,
"documents_deleted": 0,
"trigger_count": 2902,
"index_time_in_ms": 15026,
"index_total": 2799,
"index_failures": 0,
"search_time_in_ms": 266485,
"search_total": 11194,
"search_failures": 0,
"processing_time_in_ms": 0,
"processing_total": 11194,
"delete_time_in_ms": 0,
"exponential_avg_checkpoint_duration_ms": 127.98349800389431,
"exponential_avg_documents_indexed": 3.450829562432207,
"exponential_avg_documents_processed": 34522.07737546007
},
"checkpointing": {
"last": {
"checkpoint": 2799,
"timestamp_millis": 1732031824792,
"time_upper_bound_millis": 1732031704792
},
"changes_last_detected_at": 1732031944507,
"last_search_time": 1732031944507
},
"health": {
"status": "green"
}
}
}
],
"ingestPipelines": [
{
"id": "entities-v1-latest-builtin_services_from_ecs_data",
"installed": true,
"stats": {
"count": 8639,
"failed": 0
}
}
],
"indexTemplates": [
{
"id": "entities_v1_latest_builtin_services_from_ecs_data_index_template",
"installed": true,
"stats": {
"index_patterns": [
".entities.v1.latest.builtin_services_from_ecs_data"
],
"template": {
"settings": {
"index": {
"codec": "best_compression",
"mapping": {
"total_fields": {
"limit": "2000"
}
}
}
},
"mappings": {
"_meta": {
"version": "1.6.0"
},
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword",
"fields": {
"text": {
"type": "text"
}
}
},
"match_mapping_type": "string"
}
},
{
"entity_metrics": {
"path_match": "entity.metrics.*",
"mapping": {
"type": "{dynamic_type}"
},
"match_mapping_type": [
"long",
"double"
]
}
}
],
"date_detection": false
},
"aliases": {
"entities-service-latest": {}
}
},
"composed_of": [
"entities_v1_latest_base",
"entities_v1_entity",
"entities_v1_event"
],
"priority": 200,
"_meta": {
"managed_by": "elastic_entity_model",
"managed": true,
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",
"ecs_version": "8.0.0"
},
"ignore_missing_component_templates": []
}
}
]
}
}
}
]
}
klacabane
commented
3 days ago Hi @Erikg346, could you also share the output of this query GET kbn:/internal/entities/managed/enablement ?
Erikg346
commented
3 days ago Hey @klacabane
{
"enabled": false,
"reason": "api_key_not_found"
}Thanks - did you notice any error when initially clicking Enable ?
Let's try again with these manual steps to run in the dev tools. Please report the output of each command:
DELETE kbn:/internal/entities/managed/enablement?deleteData=true to ensure we have a clean statePOST kbn:/internal/entities/managed/enablement this will Enable it againGET kbn:/internal/entities/managed/enablementErikg346
commented
11 hours ago Hey @klacabane , Here's the output for each command: 1.
{
"success": true
}
2.
{
"statusCode": 404,
"error": "Not Found",
"message": "Not Found"
}3.
{
"enabled": false,
"reason": "api_key_not_found"
}
Kibana version: 8.16.0 Elasticsearch version: 8.16.0 Browser version: Version 130.0.2849.68 (Official build) (64-bit) Browser OS version: Microsoft Edge
Describe the bug: Inventory page does not but it seems to load Steps to reproduce:
Expected behavior: The page should load.
Screenshots (if relevant): It created the indices, but nothing appears in Inventory UI
Errors in browser console (if relevant):
Provide logs and/or server output (if relevant):
Any additional context: