gpop63
commented
9 months ago I created a table so we can see which data streams have the required fields for cloud packages.
| Data stream | cloud.account.id | cloud.region | cloud.availability_zone | agent.id |
|---|---|---|---|---|
| compute | ✓ | ✓ | ✓ | ✓ |
| gke | ✓ | ✗ | ✗ | ✓ |
| loadbalancing_metrics | ✓ | ✗ | ✗ | ✓ |
| pubsub | ✓ | ✗ | ✗ | ✓ |
| redis | ✓ | ✗ | ✗ | ✓ |
| storage | ✓ | ✗ | ✗ | ✓ |
| cloudrun_metrics | ✓ | ✗ | ✗ | ✓ |
| cloudsql_mysql | ✓ | ✗ | ✗ | ✓ |
| cloudsql_postgresql | ✓ | ✗ | ✗ | ✓ |
| cloudsql_sqlserver | ✓ | ✗ | ✗ | ✓ |
| firestore | ✓ | ✗ | ✗ | ✓ |
| dataproc | ✓ | ✗ | ✗ | ✓ |
I will add actual documents for each data stream so we can check them at a later time if needed.
compute
```json { "cloud": { "availability_zone": "us-central1-c", "instance": { "name": "gke-miguel-kubecon-default-pool-8d586473-uj1d", "id": "7442613444431818544" }, "provider": "gcp", "machine": { "type": "e2-medium" }, "region": "us-central1", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:13:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "compute": { "instance": { "network": { "ingress": { "bytes": 9414, "packets": { "count": 134 } }, "egress": { "bytes": 11485, "packets": { "count": 89 } } } } }, "labels": { "metrics": { "loadbalanced": "true" }, "user": { "division": "engineering", "goog-k8s-cluster-name": "miguel-kubecon", "org": "obs", "goog-k8s-node-pool-name": "default-pool", "goog-k8s-cluster-location": "us-central1-c", "project": "miguelluna", "team": "cloud-native-monitoring", "goog-gke-node": "" } } }, "service": { "type": "gcp" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.compute" }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 3126038318, "agent_id_status": "verified", "ingested": "2023-11-28T11:17:09Z", "module": "gcp", "dataset": "gcp.compute" } } ```
gke
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:14:05.736Z", "ecs": { "version": "8.0.0" }, "gcp": { "gke": { "container": { "memory": { "request": { "bytes": 10485760 }, "limit": { "bytes": 52428800 } }, "restart": { "count": 0 }, "cpu": { "request_cores": { "value": 0.005 }, "limit_cores": { "value": 0 } }, "ephemeral_storage": { "request": { "bytes": 0 }, "limit": { "bytes": 0 } } } }, "labels": { "resource": { "cluster_name": "tetiana-prometheus", "container_name": "csi-driver-registrar", "location": "europe-west1", "pod_name": "pdcsi-node-7b6hq", "namespace_name": "kube-system" } } }, "service": { "type": "gcp" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.gke" }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 1244946929, "agent_id_status": "verified", "ingested": "2023-11-28T11:18:19Z", "module": "gcp", "dataset": "gcp.gke" } } ```
loadbalancing_metrics
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:11:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "loadbalancing_metrics": { "https": { "backend_request": { "bytes": 417, "count": 1 }, "backend_response": { "bytes": 488 } } }, "labels": { "resource": { "backend_scope_type": "INVALID_BACKEND", "matched_url_path_rule": "UNMATCHED", "backend_target_name": "tas-demo-elastic-http-lb", "backend_type": "INVALID_BACKEND", "backend_scope": "INVALID_BACKEND", "target_proxy_name": "tas-demo-elastic-http-lb", "forwarding_rule_name": "tas-demo-elastic-http-lb", "backend_name": "INVALID_BACKEND", "url_map_name": "tas-demo-elastic-https-lb", "backend_target_type": "BACKEND_SERVICE", "region": "global" }, "metrics": { "response_code": "502", "proxy_continent": "America", "cache_result": "DISABLED", "response_code_class": "500" } } }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.loadbalancing_metrics" }, "service": { "type": "gcp" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "family": "debian", "type": "linux", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 476084876, "agent_id_status": "verified", "ingested": "2023-11-28T11:15:18Z", "module": "gcp", "dataset": "gcp.loadbalancing_metrics" } } ```
pubsub
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:15:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "pubsub": { "subscription": { "retained_acked_bytes_by_region": { "bytes": 0 }, "unacked_bytes_by_region": { "bytes": 0 }, "oldest_unacked_message_age_by_region": { "value": 0 }, "oldest_retained_acked_message_age_by_region": { "value": 0 } } }, "labels": { "resource": { "subscription_id": "filebeat-gcp-audit" }, "metrics": { "region": "northamerica-northeast2" } } }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.pubsub" }, "service": { "type": "gcp" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "family": "debian", "type": "linux", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 434512489, "agent_id_status": "verified", "ingested": "2023-11-28T11:19:18Z", "module": "gcp", "dataset": "gcp.pubsub" } } ```
redis
```json { "cloud": { "instance": { "name": "redis1", "id": "projects/elastic-obs-integrations-dev/locations/us-central1/instances/redis1" }, "provider": "gcp", "machine": { "type": "BASIC" }, "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "type": "metricbeat", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:15:26.810Z", "ecs": { "version": "8.0.0" }, "gcp": { "metrics": { "stats": { "reject_connections_count": {} } }, "redis": { "stats": { "reject_connections": { "count": 0 } } }, "labels": { "resource": { "region": "us-central1", "node_id": "node-0" }, "metrics": { "role": "primary" } } }, "service": { "type": "gcp" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.redis" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "family": "debian", "type": "linux", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 1381430449, "agent_id_status": "verified", "ingested": "2023-11-28T11:19:39Z", "module": "gcp", "dataset": "gcp.redis" } } ```
storage
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "type": "metricbeat", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:10:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "storage": { "storage": { "object": { "count": 6 } } }, "labels": { "resource": { "bucket_name": "dataproc-temp-us-central1-774712120909-e8nuxv73", "location": "us-central1" }, "metrics": { "storage_class": "REGIONAL" } } }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.storage" }, "service": { "type": "gcp" }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 815694657, "agent_id_status": "verified", "ingested": "2023-11-28T11:20:18Z", "module": "gcp", "dataset": "gcp.storage" } } ```
cloudrun_metrics
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "type": "metricbeat", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:18:21.590Z", "ecs": { "version": "8.0.0" }, "gcp": { "cloudrun_metrics": { "container": { "billable_instance_time": 2.7 } }, "labels": { "resource": { "revision_name": "damien-test-hello-00001-ped", "service_name": "damien-test-hello", "location": "us-central1", "configuration_name": "damien-test-hello" } } }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.cloudrun_metrics" }, "service": { "type": "gcp" }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "ip": [ "172.22.0.7" ], "containerized": true, "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 503983018, "agent_id_status": "verified", "ingested": "2023-11-28T11:21:34Z", "module": "gcp", "dataset": "gcp.cloudrun_metrics" } } ```
cloudsql_mysql
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-observability", "id": "elastic-observability" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:20:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "cloudsql_mysql": { "database": { "instance_state": false } }, "labels": { "resource": { "database_id": "elastic-observability:mysql", "region": "us-central" }, "cloudsql": { "name": "mysql", "version": "8.0.31" }, "metrics": { "state": "RUNNABLE" } } }, "service": { "type": "gcp" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.cloudsql_mysql" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "family": "debian", "type": "linux", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 1074356670, "agent_id_status": "verified", "ingested": "2023-11-28T11:23:29Z", "module": "gcp", "dataset": "gcp.cloudsql_mysql" } } ```
cloudsql_postgresql
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-observability", "id": "elastic-observability" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:20:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "cloudsql_postgresql": { "database": { "num_backends": { "count": 2 } } }, "labels": { "resource": { "database_id": "elastic-observability:postgres", "region": "us-central" }, "cloudsql": { "name": "postgres", "version": "14" }, "metrics": { "database": "cloudsqladmin" } } }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.cloudsql_postgresql" }, "service": { "type": "gcp" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 1140922692, "agent_id_status": "verified", "ingested": "2023-11-28T11:23:18Z", "module": "gcp", "dataset": "gcp.cloudsql_postgresql" } } ```
cloudsql_sqlserver
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-observability", "id": "elastic-observability" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "type": "metricbeat", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:20:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "cloudsql_sqlserver": { "database": { "disk": { "write_ops": { "count": 110 }, "bytes_used": { "bytes": 651927552 }, "read_ops": { "count": 0 }, "quota": { "bytes": 105089261568 }, "utilization": { "pct": 0.006203560119015185 } }, "memory": { "quota": { "bytes": 27331235840 }, "total_usage": { "bytes": 3081568256 }, "usage": { "bytes": 2311217152 }, "utilization": { "pct": 0.08456321424798038 } }, "cpu": { "usage_time": { "sec": 11.019461885036435 }, "utilization": { "pct": 0.045281727500696436 }, "reserved_cores": { "count": 4 } }, "up": 1, "network": { "received_bytes": { "count": 315087 }, "connections": { "count": 9 }, "sent_bytes": { "count": 3681782 } }, "uptime": { "sec": 60 } } }, "labels": { "resource": { "database_id": "elastic-observability:ms-sql", "region": "us-west1" }, "cloudsql": { "name": "sqlserver", "version": "2019_standard" } } }, "service": { "type": "gcp" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.cloudsql_sqlserver" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "ip": [ "172.22.0.7" ], "containerized": true, "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 1086857363, "agent_id_status": "verified", "ingested": "2023-11-28T11:23:29Z", "module": "gcp", "dataset": "gcp.cloudsql_sqlserver" } } ```
firestore
```json { "cloud": { "provider": "gcp", "account": { "name": "robust-catalyst-399814", "id": "robust-catalyst-399814" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "type": "metricbeat", "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:28:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "firestore": { "document": { "delete": { "count": 18 } } }, "labels": { "metrics": { "module": "__unknown__", "version": "__unknown__" } } }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.firestore" }, "service": { "type": "gcp" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 397651706, "agent_id_status": "verified", "ingested": "2023-11-28T11:32:39Z", "module": "gcp", "dataset": "gcp.firestore" } } ```
dataproc
```json { "cloud": { "provider": "gcp", "account": { "name": "elastic-obs-integrations-dev", "id": "elastic-obs-integrations-dev" } }, "agent": { "name": "docker-fleet-agent", "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "ephemeral_id": "1f9a913c-dd2f-4347-a5fc-5746e4b9f819", "type": "metricbeat", "version": "8.12.0" }, "@timestamp": "2023-11-28T11:47:00.000Z", "ecs": { "version": "8.0.0" }, "gcp": { "labels": { "resource": { "cluster_name": "cluster-aa34", "cluster_uuid": "89f8d591-cb18-4778-9c14-8be71fb578f1", "region": "us-central1" }, "metrics": { "operation_type": "START_CLUSTER", "error_type": "UNKNOWN_ERROR" } }, "dataproc": { "cluster": { "operation": { "failed": { "count": 0 } } } } }, "service": { "type": "gcp" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "gcp.dataproc" }, "elastic_agent": { "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296", "version": "8.12.0", "snapshot": true }, "host": { "hostname": "docker-fleet-agent", "os": { "kernel": "5.10.102.1-microsoft-standard-WSL2", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": true, "ip": [ "172.22.0.7" ], "name": "docker-fleet-agent", "id": "d03b51e638e64b05b5cf16c41d2058c0", "mac": [ "02-42-AC-16-00-07" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "metrics" }, "event": { "duration": 539443435, "agent_id_status": "verified", "ingested": "2023-11-28T11:50:03Z", "module": "gcp", "dataset": "gcp.dataproc" } } ```
Some data streams have the region/AZ under gcp.labels.
Add metric type:
Set dimension fields:
Firestore blocked by:
Enable TSDB by default: