Switching agent that is executing fleet server in cloud to debug logging causes Shutdown

[scunningham]

• Version: snapshot-8.0.0
• Operating System: linux

Steps

1. Create vanilla 8.0.0-snapshot cloud deployment in staging.

2. Navigate to the fleet ux

3. Select the agent running the fleet server and switch its logging setting to debug

4. Observer that the agent fails and is restarted:

Relevant log entry:

{
  "_index": "cluster-agent-filebeat-6.8.6-gcp-us-central1-2021.08.09",
  "_type": "doc",
  "_id": "nYnXjVhXGqgEc7HDInSbAof9ub4=",
  "_version": 1,
  "_score": null,
  "fields": {
    "ece.cluster": [
      "a6d647d4997f496b844e5d6ca20ec758"
    ],
    "docker.container.labels.description": [
      "Agent manages other beats based on configuration provided."
    ],
    "docker.container.labels.co.elastic.cloud.allocator.kind": [
      "apm"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.zone": [
      "us-central1-a"
    ],
    "docker.container.labels.url": [
      "https://www.elastic.co/beats/elastic-agent"
    ],
    "source": [
      "/logs/allocator/containers/apm/a6d647d4997f496b844e5d6ca20ec758/instance-0000000000/elastic-agent/data/logs/elastic-agent-json.log-20210809175338"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.instance_id": [
      "instance-0000000000"
    ],
    "meta.cloud.availability_zone": [
      "projects/460785783115/zones/us-central1-a"
    ],
    "docker.container.labels.org.label-schema.version": [
      "8.0.0-alpha1"
    ],
    "log.level": [
      "info"
    ],
    "host.name": [
      "gi-8027428028426182032"
    ],
    "docker.container.name": [
      "fac-a6d647d4997f496b844e5d6ca20ec758-instance-0000000000"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.type": [
      "apm"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.type_version": [
      "8.0.0-alpha1"
    ],
    "ece.runner": [
      "gi-8027428028426182032"
    ],
    "input_type": [
      "agent"
    ],
    "input.type": [
      "log"
    ],
    "tags": [
      "agent",
      "logging"
    ],
    "meta.cloud.machine_type": [
      "projects/460785783115/machineTypes/n2-custom-32-69632"
    ],
    "beat.name": [
      "gi-8027428028426182032"
    ],
    "docker.container.labels.vendor": [
      "Elastic"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.user_id": [
      "1744046644"
    ],
    "meta.cloud.project_id": [
      "cloud-staging-168820"
    ],
    "ecs.version": [
      "1.6.0"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.cluster_id": [
      "a6d647d4997f496b844e5d6ca20ec758"
    ],
    "docker.container.labels.io.k8s.description": [
      "Agent manages other beats based on configuration provided."
    ],
    "docker.container.labels.org.label-schema.vcs-ref": [
      "afe17b833cc2a73452eb8538e20f81b60a9d1a46"
    ],
    "docker.container.labels.summary": [
      "elastic-agent"
    ],
    "docker.container.labels.version": [
      "8.0.0-alpha1"
    ],
    "docker.container.id": [
      "c73d79bc4eacb66360d053da0c88ff7e91d0fb0a01c340528578a845877b1320"
    ],
    "docker.container.labels.name": [
      "elastic-agent"
    ],
    "ece.zone": [
      "us-central1-a"
    ],
    "meta.cloud.instance_id": [
      "8027428028426182032"
    ],
    "docker.container.labels.license": [
      "Elastic License"
    ],
    "ece.region": [
      "gcp-us-central1"
    ],
    "meta.cloud.instance_name": [
      "allocator-stg-gcp-uc1-a-generic-n2-68x32x45-384aga-c2tr"
    ],
    "prospector.type": [
      "log"
    ],
    "@version": [
      "1"
    ],
    "beat.version": [
      "6.8.6"
    ],
    "docker.container.labels.maintainer": [
      "infra@elastic.co"
    ],
    "log.origin.file.line": [
      181
    ],
    "docker.container.labels.io.k8s.display-name": [
      "Elastic-Agent image"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.monitoring_port": [
      "21402"
    ],
    "message": [
      "Signaling application to stop because of shutdown: fleet-server--8.0.0-alpha1"
    ],
    "meta.cloud.provider": [
      "gce"
    ],
    "beat.hostname": [
      "allocator-stg-gcp-uc1-a-generic-n2-68x32x45-384aga-c2tr"
    ],
    "log.origin.file.name": [
      "process/app.go"
    ],
    "@timestamp": [
      "2021-08-09T17:57:38.968Z"
    ],
    "ece.roles": [
      "allocator-generic-n2-68x32x45",
      "services-forwarder",
      "beats-runner"
    ],
    "log.file.path": [
      "/logs/allocator/containers/apm/a6d647d4997f496b844e5d6ca20ec758/instance-0000000000/elastic-agent/data/logs/elastic-agent-json.log-20210809175338"
    ],
    "docker.container.image": [
      "docker.elastic.co/cloud-ci/apm:8.0.0-alpha1-5d2ed18b"
    ],
    "docker.container.labels.release": [
      "1"
    ],
    "ece.instance": [
      "instance-0000000000"
    ],
    "docker.container.labels.co.elastic.cloud.allocator.id": [
      "gi-8027428028426182032"
    ]
  },
  "highlight": {
    "ece.cluster": [
      "@kibana-highlighted-field@a6d647d4997f496b844e5d6ca20ec758@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1628531858968
  ]
}

[elasticmachine]

Pinging @elastic/fleet (Team:Fleet)

[elasticmachine]

Pinging @elastic/agent (Team:Agent)

[botelastic[bot]]

Hi! We just realized that we haven't looked into this issue in a while. We're sorry!

We're labeling this issue as Stale to make it hit our filters and make sure we get back to it as soon as possible. In the meantime, it'd be extremely helpful if you could take a look at it as well and confirm its relevance. A simple comment with a nice emoji will be enough :+1. Thank you for your contribution!

[ph]

I don't think this issue is valid anymore, will close it.