search

elastic / kibana

Your window into the Elastic Stack
https://www.elastic.co/products/kibana
Other
19.68k stars 8.23k forks source link

[Fleet] Agent is not upgradeable in 8.6.1&2 #152106

Open zez3 opened 1 year ago

zez3 commented 1 year ago

After the stack update to 8.6.1 or 8.6.2 all previous(8.6.0 or bellow) enrolled Agents are not longer up-gradable through Fleet.

image

elasticmachine commented 1 year ago

Pinging @elastic/fleet (Team:Fleet)

kpollich commented 1 year ago

Here's Fleet's logic for determining whether agents are concerned upgradeable or not - the source of the error will be in this chunk of code:

https://github.com/elastic/kibana/blob/1af75fd5001113ea4008052a067f336bd5655fa6/x-pack/plugins/fleet/common/services/is_agent_upgradeable.ts#L14-L38

It'd be helpful to get the output of the following query truncated to include only affected agents if possible:

GET .fleet-agents/_search

Note - you'll need to be logged in as an account with system index access to perform this query.

We'd be looking for the version and upgradeable values in the agent's local_metadata property here to determine whether they're somehow flagged as ineligible for upgrades.

It'd also be good to confirm whether there's another pending upgrade for these agents already, or if they're in the process of being unenrolled.

zez3 commented 1 year ago

Something is definitely wrong. I can see only a few of my agents listed there. Most quite old ones, like from 8.5.x version and a lot missing. I see one in particular that I fresh installed and re-enrolled.

        "_index": ".fleet-agents-7",
        "_id": "497c0259-0c19-454c-b751-dda86db3e600",
        "_score": 1,
        "_ignored": [
          "local_metadata.host.mac.keyword"
        ],
        "_source": {
          "policy_coordinator_idx": 1,
          "outputs": {
            "default": {
              "api_key": "...redacted...",
              "permissions_hash": "...redacted...",
              "type": "elasticsearch",
              "to_retire_api_key_ids": [
                {
                  "retired_at": "2022-11-02T12:47:52Z",
                  "id": "KrS_N4QBR6WfNyBXt1oq"
                }
              ],
              "api_key_id": "...redacted..."
            }
          },
          "default_api_key": null,
          "agent": {
            "id": "497c0259-0c19-454c-b751-dda86db3e600",
            "version": "8.6.0"
          },
          "enrolled_at": "2022-04-28T17:24:23Z",
          "policy_id": "b779e7a0-3b68-11ec-a8f2-9bf66f2cd210",
          "last_checkin_status": "online",
          "active": false,
          "local_metadata": {
            "os": {
              "kernel": "10.0.14393.5648 (rs1_release.230105-1654)",
              "name": "Windows Server 2016 Datacenter",
              "family": "windows",
              "version": "10.0",
              "platform": "windows",
              "full": "Windows Server 2016 Datacenter(10.0)"
            },
            "elastic": {
              "agent": {
                "build.original": "8.6.0 (build: b79a5db77b5d6ffab9855234f8371d9e53978a24 at 2023-01-04 22:53:22 +0000 UTC)",
                "upgradeable": true,
                "log_level": "info",
                "id": "497c0259-0c19-454c-b751-dda86db3e600",
                "version": "8.6.0",
                "snapshot": false
              }
            },
            "host": {
              "hostname": "idprtg01",
              "ip": [
                "2001:620:400:254::86/64",
                "2001:620:400:254:9042:d08e:e20c:b40c/64",
                "fe80::9042:d08e:e20c:b40c/64",
                "x.x.x.x/24",
                "::1/128",
                "127.0.0.1/8",
                "fe80::200:5efe:825c:fe56/128"
              ],
              "name": "idprtg01",
              "id": "f9e7bc9f-7d7f-4723-bf6d-d27db62b3434",
              "mac": [
                "00:50:56:ff:ff:ff",
                "00:00:00:00:00:00:00:e0"
              ],
              "architecture": "x86_64"
            }
          },
          "type": "PERMANENT",
          "last_checkin": "2023-02-23T13:41:36Z",
          "upgrade_started_at": "2023-02-21T14:48:06.560Z",
          "access_api_key_id": "...redacted...",
          "default_api_key_id": null,
          "updated_at": "2023-02-23T13:44:00.552Z",
          "policy_revision_idx": 49,
          "policy_output_permissions_hash": null,
          "action_seq_no": [
            3132
          ],
          "upgraded_at": null,
          "tags": [
            "netsec"
          ],
          "last_checkin_message": "Running",
          "upgrade_status": null,
          "components": [
            {
              "id": "log-default",
              "units": [
                {
                  "id": "log-default-logfile-auditd-f814099f-5111-41bc-a460-0a61bf7188bf",
                  "type": "input",
                  "message": "Starting: spawned pid '5672'",
                  "status": "STARTING"
                },
                {
                  "id": "log-default-logfile-system-1332303d-46d7-4160-a4c7-83d056f2798d",
                  "type": "input",
                  "message": "Starting: spawned pid '5672'",
                  "status": "STARTING"
                },
                {
                  "id": "log-default",
                  "type": "output",
                  "message": "Starting: spawned pid '5672'",
                  "status": "STARTING"
                }
              ],
              "type": "log",
              "message": "Starting: spawned pid '5672'",
              "status": "STARTING"
            },
            {
              "id": "winlog-default",
              "units": [
                {
                  "id": "winlog-default",
                  "type": "output",
                  "message": "Starting: spawned pid '8092'",
                  "status": "STARTING"
                },
                {
                  "id": "winlog-default-winlog-system-1332303d-46d7-4160-a4c7-83d056f2798d",
                  "type": "input",
                  "message": "Starting: spawned pid '8092'",
                  "status": "STARTING"
                }
              ],
              "type": "winlog",
              "message": "Starting: spawned pid '8092'",
              "status": "STARTING"
            },
            {
              "id": "filestream-monitoring",
              "units": [
                {
                  "id": "filestream-monitoring-filestream-monitoring-agent",
                  "type": "input",
                  "message": "Starting: spawned pid '8172'",
                  "status": "STARTING"
                },
                {
                  "id": "filestream-monitoring",
                  "type": "output",
                  "message": "Starting: spawned pid '8172'",
                  "status": "STARTING"
                }
              ],
              "type": "filestream",
              "message": "Starting: spawned pid '8172'",
              "status": "STARTING"
            },
            {
              "id": "beat/metrics-monitoring",
              "units": [
                {
                  "id": "beat/metrics-monitoring-metrics-monitoring-beats",
                  "type": "input",
                  "message": "Starting: spawned pid '6064'",
                  "status": "STARTING"
                },
                {
                  "id": "beat/metrics-monitoring",
                  "type": "output",
                  "message": "Starting: spawned pid '6064'",
                  "status": "STARTING"
                }
              ],
              "type": "beat/metrics",
              "message": "Starting: spawned pid '6064'",
              "status": "STARTING"
            },
            {
              "id": "http/metrics-monitoring",
              "units": [
                {
                  "id": "http/metrics-monitoring",
                  "type": "output",
                  "message": "Starting: spawned pid '7348'",
                  "status": "STARTING"
                },
                {
                  "id": "http/metrics-monitoring-metrics-monitoring-agent",
                  "type": "input",
                  "message": "Starting: spawned pid '7348'",
                  "status": "STARTING"
                }
              ],
              "type": "http/metrics",
              "message": "Starting: spawned pid '7348'",
              "status": "STARTING"
            },
            {
              "id": "system/metrics-default",
              "units": [
                {
                  "id": "system/metrics-default",
                  "type": "output",
                  "message": "Starting: spawned pid '6100'",
                  "status": "STARTING"
                },
                {
                  "id": "system/metrics-default-system/metrics-system-1332303d-46d7-4160-a4c7-83d056f2798d",
                  "type": "input",
                  "message": "Starting: spawned pid '6100'",
                  "status": "STARTING"
                }
              ],
              "type": "system/metrics",
              "message": "Starting: spawned pid '6100'",
              "status": "STARTING"
            }
          ],
          "unenrolled_at": "2023-02-23T13:44:00.552Z"
        }
      }

Do I need perhaps the output of index .fleet-agent-8(not present) ? or some other one.

In fleet if I oven in browser the Web Dev Tools

{
    "item": {
        "id": "2455b336-793d-453e-8c55-35c2a3ddbbea",
        "outputs": {
            "default": {
                "api_key": "...redacted...",
                "permissions_hash": "...redacted...",
                "type": "elasticsearch",
                "api_key_id": "...redacted..."
            }
        },
        "policy_coordinator_idx": 1,
        "agent": {
            "id": "2455b336-793d-453e-8c55-35c2a3ddbbea",
            "version": "8.6.2"
        },
        "enrolled_at": "2023-02-23T13:43:09Z",
        "components": [
            {
                "id": "system/metrics-default",
                "units": [
                    {
                        "id": "system/metrics-default-system/metrics-system-1332303d-46d7-4160-a4c7-83d056f2798d",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "system/metrics-default",
                        "type": "output",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    }
                ],
                "type": "system/metrics",
                "message": "Healthy: communicating with pid '7188'",
                "status": "HEALTHY"
            },
            {
                "id": "log-default",
                "units": [
                    {
                        "id": "log-default-logfile-system-1332303d-46d7-4160-a4c7-83d056f2798d",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "log-default",
                        "type": "output",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "log-default-logfile-auditd-f814099f-5111-41bc-a460-0a61bf7188bf",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    }
                ],
                "type": "log",
                "message": "Healthy: communicating with pid '5100'",
                "status": "HEALTHY"
            },
            {
                "id": "winlog-default",
                "units": [
                    {
                        "id": "winlog-default",
                        "type": "output",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "winlog-default-winlog-system-1332303d-46d7-4160-a4c7-83d056f2798d",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    }
                ],
                "type": "winlog",
                "message": "Healthy: communicating with pid '7992'",
                "status": "HEALTHY"
            },
            {
                "id": "filestream-monitoring",
                "units": [
                    {
                        "id": "filestream-monitoring",
                        "type": "output",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "filestream-monitoring-filestream-monitoring-agent",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    }
                ],
                "type": "filestream",
                "message": "Healthy: communicating with pid '5692'",
                "status": "HEALTHY"
            },
            {
                "id": "beat/metrics-monitoring",
                "units": [
                    {
                        "id": "beat/metrics-monitoring",
                        "type": "output",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "beat/metrics-monitoring-metrics-monitoring-beats",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    }
                ],
                "type": "beat/metrics",
                "message": "Healthy: communicating with pid '2984'",
                "status": "HEALTHY"
            },
            {
                "id": "http/metrics-monitoring",
                "units": [
                    {
                        "id": "http/metrics-monitoring-metrics-monitoring-agent",
                        "type": "input",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    },
                    {
                        "id": "http/metrics-monitoring",
                        "type": "output",
                        "message": "Healthy",
                        "status": "HEALTHY"
                    }
                ],
                "type": "http/metrics",
                "message": "Healthy: communicating with pid '7364'",
                "status": "HEALTHY"
            }
        ],
        "policy_id": "b779e7a0-3b68-11ec-a8f2-9bf66f2cd210",
        "last_checkin_message": "Running",
        "last_checkin_status": "online",
        "active": true,
        "local_metadata": {
            "os": {
                "kernel": "10.0.14393.5648 (rs1_release.230105-1654)",
                "name": "Windows Server 2016 Datacenter",
                "family": "windows",
                "version": "10.0",
                "platform": "windows",
                "full": "Windows Server 2016 Datacenter(10.0)"
            },
            "elastic": {
                "agent": {
                    "build.original": "8.6.2 (build: 913c02bea9b13dec4d5c5f3057b5b397344e3298 at 2023-02-13 16:51:46 +0000 UTC)",
                    "upgradeable": true,
                    "log_level": "info",
                    "id": "2455b336-793d-453e-8c55-35c2a3ddbbea",
                    "version": "8.6.2",
                    "snapshot": false
                }
            },
            "host": {
                "hostname": "idprtg01",
                "ip": [
                    "...redacted...",
                    "...redacted.../64",
                    "fe80::9042:d08e:e20c:b40c/64",
                    "...redacted.../24",
                    "::1/128",
                    "127.0.0.1/8",
                    "fe80::200:5efe:825c:fe56/128"
                ],
                "name": "idprtg01",
                "id": "f9e7bc9f-7d7f-4723-bf6d-d27db62b3434",
                "mac": [
                    "00:50:56:ff:ff:ff",
                    "00:00:00:00:00:00:00:e0"
                ],
                "architecture": "x86_64"
            }
        },
        "type": "PERMANENT",
        "last_checkin": "2023-02-24T19:43:11Z",
        "access_api_key_id": "x5ODfoYB3QzpE_EfdGzc",
        "tags": [
            "netsec"
        ],
        "updated_at": "2023-02-24T19:43:17Z",
        "policy_revision_idx": 49,
        "action_seq_no": [
            3186
        ],
        "policy_revision": 49,
        "status": "online",
        "packages": [
        ]
    }
}