search

hashicorp / nomad-autoscaler

Nomad Autoscaler brings autoscaling to your Nomad workloads.
Mozilla Public License 2.0
427 stars 84 forks source link

double scaling action #610

Closed margiran closed 10 months ago

margiran commented 1 year ago

Nomad Autoscaler version

0.3.7+ent

Operating system and Environment details

Ubuntu 22.04.1 LTS Nomad 1.4.3

Issue

Nomad Autoscaler runs two scaling actions at the same time for the same policy. it seems the same policy managed to get into two different workers at the same time and it tried to run two scaling actions

2023-01-10T10:18:08.583Z [INFO] policy_eval.worker: scaling target: id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg from=3 to=2 reason="scaling down because factor is 0.367619" meta=map[nomad_policy_id:15ef06d8-8d84-e0b7-5a5e-6720497bff2f]
2023-01-10T10:18:09.425Z [INFO] policy_eval.worker: scaling target: id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg from=3 to=2 reason="scaling down because factor is 0.367619" meta=map[nomad_policy_id:15ef06d8-8d84-e0b7-5a5e-6720497bff2f]

Reproduction steps

Deploy sample infrastructure ( https://developer.hashicorp.com/nomad/tutorials/autoscaler/horizontal-cluster-scaling ) Run the jobs listed below. Then Generate traffic and make the autoscaler perform scaling action a few times

AutoScaler config:

job "autoscaler" {
  datacenters = ["dc1"]

  group "autoscaler" {
    count = 1

    network {
      port "http" {}
    }

    task "autoscaler" {
      driver = "docker"

      config {
        image   = "hashicorp/nomad-autoscaler:0.3.7"
        command = "nomad-autoscaler"

        args = [
          "agent",
          "-config",
          "${NOMAD_TASK_DIR}/config.hcl",
          "-http-bind-address",
          "0.0.0.0",
          "-http-bind-port",
          "${NOMAD_PORT_http}",
          "-policy-dir",
          "${NOMAD_TASK_DIR}/policies/",
          "-log-level",
          "DEBUG"
        ]

        ports = ["http"]
      }

      template {
        data = <<EOF
nomad {
  address = "http://{{env "attr.unique.network.ip-address" }}:4646"
}

apm "prometheus" {
  driver = "prometheus"
  config = {
    address = "http://{{ range service "prometheus" }}{{ .Address }}:{{ .Port }}{{ end }}"
  }
}
apm "nomad" {
  driver = "nomad-apm"
  config = {
    address = "http://{{env "attr.unique.network.ip-address" }}:4646"
  }
}

target "aws-asg" {
  driver = "aws-asg"
  config = {
    aws_region = "{{ $x := env "attr.platform.aws.placement.availability-zone" }}{{ $length := len $x |subtract 1 }}{{ slice $x 0 $length}}"
  }
}
policy_eval {
  ack_timeout    = "2m"
  delivery_limit = 1

  workers = {
    cluster    = 10
    horizontal = 10
  }
}
strategy "target-value" {
  driver = "target-value"
}
EOF

        destination = "${NOMAD_TASK_DIR}/config.hcl"
      }

      template {
        data = <<EOF
scaling "cluster_policy" {
  enabled = true
  min     = 1
  max     = 6

  policy {
    cooldown            = "2m"
    evaluation_interval = "1m"

    check "cpu_allocated_percentage" {
      source = "prometheus"
      query  = "sum(nomad_client_allocated_cpu{node_class=\"hashistack\"}*100/(nomad_client_unallocated_cpu{node_class=\"hashistack\"}+nomad_client_allocated_cpu{node_class=\"hashistack\"}))/count(nomad_client_allocated_cpu{node_class=\"hashistack\"})"
      query_window = "2m"
      strategy "target-value" {
        target = 70
        threshold = 0.1

      }
    }

    check "mem_allocated_percentage" {
      source = "prometheus"
      query  = "sum(nomad_client_allocated_memory{node_class=\"hashistack\"}*100/(nomad_client_unallocated_memory{node_class=\"hashistack\"}+nomad_client_allocated_memory{node_class=\"hashistack\"}))/count(nomad_client_allocated_memory{node_class=\"hashistack\"})"
      query_window = "2m"
      strategy "target-value" {
        target = 70
        threshold = 0.1
      }
    }

    target "aws-asg" {
      dry-run             = "false"
      aws_asg_name        = "hashistack-nomad_client"
      node_class          = "hashistack"
      node_drain_deadline = "2m"
    }
  }
}
EOF

        destination = "${NOMAD_TASK_DIR}/policies/hashistack.hcl"
      }

      resources {
        cpu    = 60
        memory = 128
      }

      service {
        name = "autoscaler"
        port = "http"

        check {
          type     = "http"
          path     = "/v1/health"
          interval = "5s"
          timeout  = "2s"
        }
      }
    }
  }
}

Job spec:

job "webapp" {
  datacenters = ["dc1"]

  group "demo" {
    count = 1

    network {
      port "webapp_http" {}
      port "toxiproxy_webapp" {}
    }

    scaling {
      enabled = true
      min     = 1
      max     = 20

      policy {
        cooldown            = "1m"
        evaluation_interval = "30s"

        check "avg_sessions" {
          source = "prometheus"
          query  = "sum(traefik_entrypoint_open_connections{entrypoint=\"webapp\"})/scalar(nomad_nomad_job_summary_running{task_group=\"demo\"})"

          strategy "target-value" {
            target = 10
          }
        }
      }
    }

    task "webapp" {
      driver = "docker"
       config {
        image = "hashicorp/demo-webapp-lb-guide"
        ports = ["webapp_http"]
      }

      env {
        PORT    = "${NOMAD_PORT_webapp_http}"
        NODE_IP = "${NOMAD_IP_webapp_http}"
      }

      resources {
        cpu    = 500
        memory = 256
      }

      service {
        name = "webapp"
        port = "webapp_http"

        check {
          type     = "http"
          path     = "/"
          interval = "3s"
          timeout  = "1s"
        }
      }
    }

    task "toxiproxy" {
      driver = "docker"

      lifecycle {
        hook    = "prestart"
        sidecar = true
      }

      config {
        image      = "shopify/toxiproxy:2.1.4"
        entrypoint = ["/entrypoint.sh"]
        ports      = ["toxiproxy_webapp"]

        volumes = [
          "local/entrypoint.sh:/entrypoint.sh",
        ]
      }

      template {
        data = <<EOH
#!/bin/sh

set -ex

/go/bin/toxiproxy -host 0.0.0.0  &

while ! wget --spider -q http://localhost:8474/version; do
  echo "toxiproxy not ready yet"
  sleep 0.2
done

/go/bin/toxiproxy-cli create webapp -l 0.0.0.0:${NOMAD_PORT_toxiproxy_webapp} -u ${NOMAD_ADDR_webapp_http}
/go/bin/toxiproxy-cli toxic add -n latency -t latency -a latency=1000 -a jitter=500 webapp
tail -f /dev/null
        EOH

        destination = "local/entrypoint.sh"
        perms       = "755"
      }

      resources {
        cpu    = 100
        memory = 32
      }

      service {
        name = "toxiproxy-webapp"
        port = "toxiproxy_webapp"

        tags = [
          "traefik.enable=true",
          "traefik.http.routers.webapp.entrypoints=webapp",
          "traefik.http.routers.webapp.rule=Path(`/`)",
        ]
      }
    }
  }
}

Expected Result

Have a single scaling event and should honor the cooldown time.

Actual Result

Nomad Autoscaler runs two scaling actions at the same time for the same policy.

Autoscaler Server logs

2023-01-10T10:18:06.815Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
2023-01-10T10:18:08.430Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=5301857e-79d9-b7c7-db53-1fdda9241a83 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:08.430Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=5301857e-79d9-b7c7-db53-1fdda9241a83 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:08.430Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=5301857e-79d9-b7c7-db53-1fdda9241a83 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f token=8888b50e-5d2d-e491-622e-0a0be1e96c28
2023-01-10T10:18:08.430Z [DEBUG] policy_eval.worker: received policy for evaluation: id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg
2023-01-10T10:18:08.430Z [DEBUG] policy_eval.worker: fetching current count: id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg
2023-01-10T10:18:08.573Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu_allocated_percentage id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg
2023-01-10T10:18:08.573Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu_allocated_percentage id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg query=sum(nomad_client_allocated_cpu{node_class="hashistack"}*100/(nomad_client_unallocated_cpu{node_class="hashistack"}+nomad_client_allocated_cpu{node_class="hashistack"}))/count(nomad_client_allocated_cpu{node_class="hashistack"}) source=prometheus
2023-01-10T10:18:08.573Z [DEBUG] internal_plugin.prometheus: querying Prometheus: query=sum(nomad_client_allocated_cpu{node_class="hashistack"}*100/(nomad_client_unallocated_cpu{node_class="hashistack"}+nomad_client_allocated_cpu{node_class="hashistack"}))/count(nomad_client_allocated_cpu{node_class="hashistack"}) range="{2023-01-10 10:16:08.573863691 +0000 UTC m=+1501.770682933 2023-01-10 10:18:08.573863691 +0000 UTC m=+1621.770682933}"
2023-01-10T10:18:08.578Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu_allocated_percentage id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg count=3
2023-01-10T10:18:08.578Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=mem_allocated_percentage id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg
2023-01-10T10:18:08.578Z [DEBUG] policy_eval.worker.check_handler: querying source: check=mem_allocated_percentage id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg query=sum(nomad_client_allocated_memory{node_class="hashistack"}*100/(nomad_client_unallocated_memory{node_class="hashistack"}+nomad_client_allocated_memory{node_class="hashistack"}))/count(nomad_client_allocated_memory{node_class="hashistack"}) source=prometheus
2023-01-10T10:18:08.578Z [DEBUG] internal_plugin.prometheus: querying Prometheus: query=sum(nomad_client_allocated_memory{node_class="hashistack"}*100/(nomad_client_unallocated_memory{node_class="hashistack"}+nomad_client_allocated_memory{node_class="hashistack"}))/count(nomad_client_allocated_memory{node_class="hashistack"}) range="{2023-01-10 10:16:08.578622936 +0000 UTC m=+1501.775442179 2023-01-10 10:18:08.578622936 +0000 UTC m=+1621.775442179}"
2023-01-10T10:18:08.583Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=mem_allocated_percentage id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg count=3
2023-01-10T10:18:08.583Z [DEBUG] policy_eval.worker: check cpu_allocated_percentage selected in group : id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg direction=down count=2
2023-01-10T10:18:08.583Z [DEBUG] policy_eval.worker: check cpu_allocated_percentage selected: id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg direction=down count=2
**2023-01-10T10:18:08.583Z [INFO]  policy_eval.worker: scaling target: id=94d49fa5-7323-5d58-2030-d51969e5f21f policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg from=3 to=2 reason="scaling down because factor is 0.367619" meta=map[nomad_policy_id:15ef06d8-8d84-e0b7-5a5e-6720497bff2f]**
2023-01-10T10:18:08.637Z [DEBUG] internal_plugin.aws-asg: found healthy instance: action=scale_in asg_name=hashistack-nomad_client instance_id=i-083f535b901eb9326
2023-01-10T10:18:08.637Z [DEBUG] internal_plugin.aws-asg: found healthy instance: action=scale_in asg_name=hashistack-nomad_client instance_id=i-08a077fae143403d4
2023-01-10T10:18:08.637Z [DEBUG] internal_plugin.aws-asg: found healthy instance: action=scale_in asg_name=hashistack-nomad_client instance_id=i-0d47bdd6348b74764
2023-01-10T10:18:08.637Z [DEBUG] internal_plugin.aws-asg: performing node pool filtering: node_class=hashistack
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: found node: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 datacenter=dc1 node_class=hashistack status=ready eligibility=eligible draining=false
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: found node: node_id=4265833c-892c-2126-7530-cb724c53878a datacenter=dc1 node_class=hashistack status=down eligibility=ineligible draining=false
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: found node: node_id=5ef967de-6f32-babe-8dcd-9e31a2a030c8 datacenter=dc1 node_class=hashistack status=down eligibility=ineligible draining=false
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: found node: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d datacenter=dc1 node_class=hashistack status=ready eligibility=eligible draining=false
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: found node: node_id=0432aca7-896a-6945-78ef-b1fdca11a2cb datacenter=dc1 node_class=hashistack status=ready eligibility=eligible draining=false
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: node passed filter criteria: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3
2023-01-10T10:18:08.643Z [DEBUG] internal_plugin.aws-asg: node passed filter criteria: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d
2023-01-10T10:18:08.646Z [DEBUG] internal_plugin.aws-asg: identified remote provider ID for node: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 remote_id=i-08a077fae143403d4
2023-01-10T10:18:08.650Z [DEBUG] internal_plugin.aws-asg: identified remote provider ID for node: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d remote_id=i-0d47bdd6348b74764
2023-01-10T10:18:08.650Z [DEBUG] internal_plugin.aws-asg: node is part of the policy target: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 remote_id=i-08a077fae143403d4
2023-01-10T10:18:08.650Z [DEBUG] internal_plugin.aws-asg: node is part of the policy target: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d remote_id=i-0d47bdd6348b74764
2023-01-10T10:18:08.650Z [DEBUG] internal_plugin.aws-asg: performing node selection: selector_strategy=least_busy
2023-01-10T10:18:08.697Z [DEBUG] internal_plugin.aws-asg: node selected for removal: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 remote_id=i-08a077fae143403d4
2023-01-10T10:18:08.698Z [INFO]  internal_plugin.aws-asg: triggering drain on node: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 deadline=2m0s
2023-01-10T10:18:08.720Z [DEBUG] internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="Alloc "fe2c660b-2a25-2c6a-7be1-cb612093fe4b" marked for migration"
2023-01-10T10:18:08.724Z [INFO]  internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="Drain complete for node ac91e3f5-b8d8-3b8b-2216-6e8486105cf3"
2023-01-10T10:18:08.731Z [DEBUG] internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="Alloc "fe2c660b-2a25-2c6a-7be1-cb612093fe4b" draining"
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: policy already enqueued: eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f token=2d680a31-c9c5-d91c-ad9e-eb3814fbbfb3
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.worker: received policy for evaluation: id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.worker: fetching current count: id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg
2023-01-10T10:18:09.407Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu_allocated_percentage id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg
2023-01-10T10:18:09.407Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu_allocated_percentage id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg query=sum(nomad_client_allocated_cpu{node_class="hashistack"}*100/(nomad_client_unallocated_cpu{node_class="hashistack"}+nomad_client_allocated_cpu{node_class="hashistack"}))/count(nomad_client_allocated_cpu{node_class="hashistack"}) source=prometheus
2023-01-10T10:18:09.407Z [DEBUG] internal_plugin.prometheus: querying Prometheus: query=sum(nomad_client_allocated_cpu{node_class="hashistack"}*100/(nomad_client_unallocated_cpu{node_class="hashistack"}+nomad_client_allocated_cpu{node_class="hashistack"}))/count(nomad_client_allocated_cpu{node_class="hashistack"}) range="{2023-01-10 10:16:09.407593727 +0000 UTC m=+1502.604412969 2023-01-10 10:18:09.407593727 +0000 UTC m=+1622.604412969}"
2023-01-10T10:18:09.414Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu_allocated_percentage id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg count=3
2023-01-10T10:18:09.414Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=mem_allocated_percentage id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg
2023-01-10T10:18:09.415Z [DEBUG] policy_eval.worker.check_handler: querying source: check=mem_allocated_percentage id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg query=sum(nomad_client_allocated_memory{node_class="hashistack"}*100/(nomad_client_unallocated_memory{node_class="hashistack"}+nomad_client_allocated_memory{node_class="hashistack"}))/count(nomad_client_allocated_memory{node_class="hashistack"}) source=prometheus
2023-01-10T10:18:09.415Z [DEBUG] internal_plugin.prometheus: querying Prometheus: query=sum(nomad_client_allocated_memory{node_class="hashistack"}*100/(nomad_client_unallocated_memory{node_class="hashistack"}+nomad_client_allocated_memory{node_class="hashistack"}))/count(nomad_client_allocated_memory{node_class="hashistack"}) range="{2023-01-10 10:16:09.415029252 +0000 UTC m=+1502.611848497 2023-01-10 10:18:09.415029252 +0000 UTC m=+1622.611848497}"
2023-01-10T10:18:09.425Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=mem_allocated_percentage id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster source=prometheus strategy=target-value target=aws-asg count=3
2023-01-10T10:18:09.425Z [DEBUG] policy_eval.worker: check cpu_allocated_percentage selected in group : id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg direction=down count=2
2023-01-10T10:18:09.425Z [DEBUG] policy_eval.worker: check cpu_allocated_percentage selected: id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg direction=down count=2
**2023-01-10T10:18:09.425Z [INFO]  policy_eval.worker: scaling target: id=cd33920e-d566-e685-0b3c-4e5ae5406d67 policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster target=aws-asg from=3 to=2 reason="scaling down because factor is 0.367619" meta=map[nomad_policy_id:15ef06d8-8d84-e0b7-5a5e-6720497bff2f]**
2023-01-10T10:18:09.529Z [DEBUG] internal_plugin.aws-asg: found healthy instance: action=scale_in asg_name=hashistack-nomad_client instance_id=i-083f535b901eb9326
2023-01-10T10:18:09.529Z [DEBUG] internal_plugin.aws-asg: found healthy instance: action=scale_in asg_name=hashistack-nomad_client instance_id=i-08a077fae143403d4
2023-01-10T10:18:09.529Z [DEBUG] internal_plugin.aws-asg: found healthy instance: action=scale_in asg_name=hashistack-nomad_client instance_id=i-0d47bdd6348b74764
2023-01-10T10:18:09.529Z [DEBUG] internal_plugin.aws-asg: performing node pool filtering: node_class=hashistack
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: found node: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 datacenter=dc1 node_class=hashistack status=ready eligibility=ineligible draining=false
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: found node: node_id=4265833c-892c-2126-7530-cb724c53878a datacenter=dc1 node_class=hashistack status=down eligibility=ineligible draining=false
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: found node: node_id=5ef967de-6f32-babe-8dcd-9e31a2a030c8 datacenter=dc1 node_class=hashistack status=down eligibility=ineligible draining=false
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: found node: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d datacenter=dc1 node_class=hashistack status=ready eligibility=eligible draining=false
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: found node: node_id=0432aca7-896a-6945-78ef-b1fdca11a2cb datacenter=dc1 node_class=hashistack status=ready eligibility=eligible draining=false
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: node passed filter criteria: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3
2023-01-10T10:18:09.536Z [DEBUG] internal_plugin.aws-asg: node passed filter criteria: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d
2023-01-10T10:18:09.540Z [DEBUG] internal_plugin.aws-asg: identified remote provider ID for node: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 remote_id=i-08a077fae143403d4
2023-01-10T10:18:09.544Z [DEBUG] internal_plugin.aws-asg: identified remote provider ID for node: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d remote_id=i-0d47bdd6348b74764
2023-01-10T10:18:09.544Z [DEBUG] internal_plugin.aws-asg: node is part of the policy target: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 remote_id=i-08a077fae143403d4
2023-01-10T10:18:09.544Z [DEBUG] internal_plugin.aws-asg: node is part of the policy target: node_id=1498d0f4-05d8-7c0e-7c76-0d9ac382d91d remote_id=i-0d47bdd6348b74764
2023-01-10T10:18:09.544Z [DEBUG] internal_plugin.aws-asg: performing node selection: selector_strategy=least_busy
2023-01-10T10:18:09.545Z [DEBUG] internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="Alloc "fe2c660b-2a25-2c6a-7be1-cb612093fe4b" status running -> complete"
2023-01-10T10:18:09.546Z [INFO]  internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="All allocations on node "ac91e3f5-b8d8-3b8b-2216-6e8486105cf3" have stopped"
2023-01-10T10:18:09.546Z [DEBUG] internal_plugin.aws-asg: node drain complete: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3
2023-01-10T10:18:09.546Z [DEBUG] internal_plugin.aws-asg: pre scale-in tasks now complete
2023-01-10T10:18:09.619Z [DEBUG] internal_plugin.aws-asg: node selected for removal: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 remote_id=i-08a077fae143403d4
2023-01-10T10:18:09.619Z [INFO]  internal_plugin.aws-asg: triggering drain on node: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 deadline=2m0s
2023-01-10T10:18:09.630Z [INFO]  internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="Drain complete for node ac91e3f5-b8d8-3b8b-2216-6e8486105cf3"
2023-01-10T10:18:09.636Z [INFO]  internal_plugin.aws-asg: received node drain message: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3 msg="All allocations on node "ac91e3f5-b8d8-3b8b-2216-6e8486105cf3" have stopped"
2023-01-10T10:18:09.636Z [DEBUG] internal_plugin.aws-asg: node drain complete: node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3
2023-01-10T10:18:09.636Z [DEBUG] internal_plugin.aws-asg: pre scale-in tasks now complete
2023-01-10T10:18:09.838Z [DEBUG] internal_plugin.aws-asg: successfully terminated instance in ASG: action=scale_in asg_name=hashistack-nomad_client instance_id=i-08a077fae143403d4 node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3
2023-01-10T10:18:09.838Z [DEBUG] internal_plugin.aws-asg: ensuring AWS ASG activities complete
2023-01-10T10:18:09.917Z [DEBUG] internal_plugin.aws-asg: successfully terminated instance in ASG: action=scale_in asg_name=hashistack-nomad_client instance_id=i-08a077fae143403d4 node_id=ac91e3f5-b8d8-3b8b-2216-6e8486105cf3
2023-01-10T10:18:09.917Z [DEBUG] internal_plugin.aws-asg: ensuring AWS ASG activities complete
lgfa29 commented 1 year ago

Hi @margiran 👋

Thanks for the report. This part is very suspucious to me:

2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: policy already enqueued: eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f queue=cluster token=""
2023-01-10T10:18:09.294Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=6345dc20-f673-1865-ec84-cb8c8be7d74c policy_id=15ef06d8-8d84-e0b7-5a5e-6720497bff2f token=2d680a31-c9c5-d91c-ad9e-eb3814fbbfb3

I suspect we have a bug in this part of the code where a policy is currently being evaluated (so it hasn't been ACK'ed or NACK'ed) and is present in b.enqueuedPolicies, but since it has been dequeued it's not in pending anymore: https://github.com/hashicorp/nomad-autoscaler/blob/954f26e8c56526d62f78d6991f5ff2a29dc6fa9f/policyeval/broker.go#L148-L169

My first thought is that the return on line 167 needs to be outside the if condition, if the policy is already enqueued we shouldn't queue it again. But I would need more time to think through it.

janory commented 1 year ago

Hi @lgfa29 👋

We think, that we are also affected by this problem. Please check out this example where a double scaling action happened in our case:

2023-07-18T13:27:47.775Z [DEBUG] policy_eval.worker: successfully submitted scaling action to target: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg desired_count=14
--
2023-07-18T13:27:47.775Z [INFO] internal_plugin.aws-asg: successfully performed and verified scaling out: action=scale_out asg_name=maya-box-asg desired_count=14
2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: eval ack'd: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: ack eval: eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f token=209f37a9-d329-1707-d300-94622d061947 eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f token=209f37a9-d329-1707-d300-94622d061947
2023-07-18T13:27:31.873Z [DEBUG] policy_eval.worker: policy evaluation complete: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
2023-07-18T13:27:31.873Z [DEBUG] policy_manager.policy_handler: scaling policy has been placed into cooldown: policy_id=4a1d5af4-323a-d939-d208-18672288565c cooldown=4m0s
2023-07-18T13:27:31.873Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:31.829Z [DEBUG] policy_eval.worker: successfully submitted scaling action to target: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg desired_count=11
2023-07-18T13:27:31.829Z [INFO] internal_plugin.aws-asg: successfully performed and verified scaling out: action=scale_out asg_name=maya-box-asg desired_count=11
2023-07-18T13:27:31.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:27.228Z [INFO] policy_eval.worker: scaling target: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg from=11 to=14 reason="scaling up because factor is 1.183979" meta=map[nomad_policy_id:4a1d5af4-323a-d939-d208-18672288565c]
2023-07-18T13:27:27.228Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=14
2023-07-18T13:27:27.228Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected in group : id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=14
2023-07-18T13:27:27.228Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=memory-allocated-percentage current_count=11 new_count=14 metric_value=71.03871091033841 metric_time="2023-07-18 13:27:27.227956029 +0000 UTC m=+1037.429037121" factor=1.1839785151723068 direction=up
2023-07-18T13:27:27.228Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=11
2023-07-18T13:27:27.228Z [TRACE] policy_eval.worker.check_handler: metric result: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:27.227956029 +0000 UTC m=+1037.429037121" value=71.03871091033841
2023-07-18T13:27:27.227Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=985100 allocated_memory=1234147 allocatable_cpu=3110400 allocatable_memory=1737288
2023-07-18T13:27:24.673Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_memory/box/class
2023-07-18T13:27:24.672Z [DEBUG] policy_eval.worker.check_handler: querying source: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_memory/box/class source=nomad-apm
2023-07-18T13:27:24.672Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
2023-07-18T13:27:24.672Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=cpu-allocated-percentage current_count=11 new_count=12 metric_value=31.909079218106996 metric_time="2023-07-18 13:27:24.672761177 +0000 UTC m=+1034.873842267" factor=1.0636359739369 direction=up
2023-07-18T13:27:24.672Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=11
2023-07-18T13:27:24.672Z [TRACE] policy_eval.worker.check_handler: metric result: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:24.672761177 +0000 UTC m=+1034.873842267" value=31.909079218106996
2023-07-18T13:27:24.672Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=992500 allocated_memory=1243406 allocatable_cpu=3110400 allocatable_memory=1737288
2023-07-18T13:27:22.182Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_cpu/box/class
2023-07-18T13:27:22.182Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_cpu/box/class source=nomad-apm
2023-07-18T13:27:22.182Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
2023-07-18T13:27:21.913Z [DEBUG] policy_eval.worker: fetching current count: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
2023-07-18T13:27:21.912Z [DEBUG] policy_eval.worker: received policy for evaluation: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c token=31cbaf4f-161d-1f33-758f-fccb1fcd22c7
2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: policy already enqueued: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
2023-07-18T13:27:21.912Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:21.311Z [INFO] policy_eval.worker: scaling target: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg from=9 to=11 reason="scaling up because factor is 1.148448" meta=map[nomad_policy_id:4a1d5af4-323a-d939-d208-18672288565c]
2023-07-18T13:27:21.311Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=11
2023-07-18T13:27:21.311Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected in group : id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=11
2023-07-18T13:27:21.311Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=memory-allocated-percentage current_count=9 new_count=11 metric_value=68.90688245126887 metric_time="2023-07-18 13:27:21.311640085 +0000 UTC m=+1031.512721175" factor=1.1484480408544813 direction=up
2023-07-18T13:27:21.311Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=9
2023-07-18T13:27:21.311Z [TRACE] policy_eval.worker.check_handler: metric result: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:21.311640085 +0000 UTC m=+1031.512721175" value=68.90688245126887
2023-07-18T13:27:21.311Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=955500 allocated_memory=1197111 allocatable_cpu=3110400 allocatable_memory=1737288
2023-07-18T13:27:17.503Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_memory/box/class
2023-07-18T13:27:17.503Z [DEBUG] policy_eval.worker.check_handler: querying source: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_memory/box/class source=nomad-apm
2023-07-18T13:27:17.503Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
2023-07-18T13:27:17.503Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=cpu-allocated-percentage current_count=9 new_count=10 metric_value=30.481610082304528 metric_time="2023-07-18 13:27:17.503668056 +0000 UTC m=+1027.704749146" factor=1.016053669410151 direction=up
2023-07-18T13:27:17.503Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=9
2023-07-18T13:27:17.503Z [TRACE] policy_eval.worker.check_handler: metric result: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:17.503668056 +0000 UTC m=+1027.704749146" value=30.481610082304528
2023-07-18T13:27:17.503Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=948100 allocated_memory=1187852 allocatable_cpu=3110400 allocatable_memory=1737288
2023-07-18T13:27:14.768Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_cpu/box/class
2023-07-18T13:27:14.768Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_cpu/box/class source=nomad-apm
2023-07-18T13:27:14.768Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
2023-07-18T13:27:14.496Z [DEBUG] policy_eval.worker: fetching current count: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
2023-07-18T13:27:14.495Z [DEBUG] policy_eval.worker: received policy for evaluation: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
2023-07-18T13:27:14.495Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f policy_id=4a1d5af4-323a-d939-d208-18672288565c token=209f37a9-d329-1707-d300-94622d061947
2023-07-18T13:27:14.495Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
2023-07-18T13:27:14.495Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
2023-07-18T13:27:14.495Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:14.249Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:14.249Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
2023-07-18T13:27:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
2023-07-18T13:26:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection

At 2023-07-18T13:27:21.311Z the autoscaler came to the conclusion to scale the AWS ASG from 9 to 11 instances.

I think at this point we should enter into the cooldown phase, but instead we got a new tick at 2023-07-18T13:27:21.642Z and although there was a policy already enqueued: message at 2023-07-18T13:27:21.912Z, the autoscaler moved on and scaled again from 11 to 14 instances.

After looking at the logs a bit closer, the incorrect queue evaluation might not be the only problem here. In our case the evaluation_interval is configured to 10s, but there was only ~7 seconds between the two ticks:

...
2023-07-18T13:27:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
...
2023-07-18T13:27:14.249Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
...
lgfa29 commented 1 year ago

Thanks for the extra info @janory.

I think everything between 

2023-07-18T13:27:21.311Z [INFO] policy_eval.worker: scaling target: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg from=9 to=11 reason="scaling up because factor is 1.148448" meta=map[nomad_policy_id:4a1d5af4-323a-d939-d208-18672288565c]

And 

2023-07-18T13:27:31.829Z [DEBUG] policy_eval.worker: successfully submitted scaling action to target: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg desired_count=11

Should not have happened since the eval was not ACK'ed yet.

The time skew between policy evals hints me that there are two policy handlers active for the same policy, possibly the result of botched reload operation 🤔

If you still have access to older log entries would you be able to check if you can find any reload events?

janory commented 1 year ago

Thanks for the prompt response @lgfa29.

I've searched for the word "reload" in our logs, but I haven't found any events. But I can provide you the full trace since the autoscaler agent came up.

Hint:

The first column is the allocation id of the autoscaler agent. You can see some teardown messages from the previous allocation of the autoscaler agent. I am also sending below our policy config, it might help you with the investigation.

Logs:


"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:37:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:36:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:44.532Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:44.532Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:44.532Z [DEBUG] policy_eval.broker: eval ack'd: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:44.532Z [DEBUG] policy_eval.broker: ack eval: eval_id=82ba46cd-dce3-1ad9-f564-85fcec599076 token=defe6410-5d1f-dcb8-6ce6-b94cc6bb72c6 eval_id=82ba46cd-dce3-1ad9-f564-85fcec599076 token=defe6410-5d1f-dcb8-6ce6-b94cc6bb72c6
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:44.532Z [DEBUG] policy_eval.worker: policy evaluation complete: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:44.532Z [DEBUG] policy_manager.policy_handler: scaling policy has been placed into cooldown: policy_id=4a1d5af4-323a-d939-d208-18672288565c cooldown=4m0s
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:35:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:34:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:33:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:32:11.706Z [DEBUG] policy_eval.worker: successfully submitted scaling action to target: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg desired_count=16
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:32:11.706Z [INFO] internal_plugin.aws-asg: successfully performed and verified scaling out: action=scale_out asg_name=maya-box-asg desired_count=16
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:32:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.058Z [INFO] policy_eval.worker: scaling target: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg from=14 to=16 reason="scaling up because factor is 1.116090" meta=map[nomad_policy_id:4a1d5af4-323a-d939-d208-18672288565c]
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.058Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=16
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.058Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected in group : id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=16
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.058Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=memory-allocated-percentage current_count=14 new_count=16 metric_value=66.96539581890197 metric_time="2023-07-18 13:31:51.057939641 +0000 UTC m=+1301.259020730" factor=1.116089930315033 direction=up
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.058Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=memory-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=14
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.058Z [TRACE] policy_eval.worker.check_handler: metric result: check=memory-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:31:51.057939641 +0000 UTC m=+1301.259020730" value=66.96539581890197
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:51.057Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=1444400 allocated_memory=1809705 allocatable_cpu=4838400 allocatable_memory=2702448
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.725Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_memory/box/class
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.725Z [DEBUG] policy_eval.worker.check_handler: querying source: check=memory-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_memory/box/class source=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.725Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=memory-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.725Z [DEBUG] policy_eval.worker.check_handler: nothing to do: check=cpu-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.724Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=cpu-allocated-percentage current_count=14 new_count=14 metric_value=29.699900793650794 metric_time="2023-07-18 13:31:47.724784801 +0000 UTC m=+1297.925865890" factor=0.9899966931216931 direction=down
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.724Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=14
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.724Z [TRACE] policy_eval.worker.check_handler: metric result: check=cpu-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:31:47.724784801 +0000 UTC m=+1297.925865890" value=29.699900793650794
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:47.724Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=1437000 allocated_memory=1800446 allocatable_cpu=4838400 allocatable_memory=2702448
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.780Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_cpu/box/class
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.780Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_cpu/box/class source=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.780Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu-allocated-percentage id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.532Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.532Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.532Z [DEBUG] policy_eval.broker: eval ack'd: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.532Z [DEBUG] policy_eval.broker: ack eval: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 token=31cbaf4f-161d-1f33-758f-fccb1fcd22c7 eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 token=31cbaf4f-161d-1f33-758f-fccb1fcd22c7
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.532Z [DEBUG] policy_eval.worker: fetching current count: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [DEBUG] policy_eval.worker: received policy for evaluation: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=82ba46cd-dce3-1ad9-f564-85fcec599076 policy_id=4a1d5af4-323a-d939-d208-18672288565c token=defe6410-5d1f-dcb8-6ce6-b94cc6bb72c6
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=82ba46cd-dce3-1ad9-f564-85fcec599076 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=82ba46cd-dce3-1ad9-f564-85fcec599076 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [DEBUG] policy_eval.worker: policy evaluation complete: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [DEBUG] policy_manager.policy_handler: scaling policy has been placed into cooldown: policy_id=4a1d5af4-323a-d939-d208-18672288565c cooldown=4m0s
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.531Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.269Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:44.269Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:32.142Z [DEBUG] policy_manager.policy_handler: scaling policy has been placed into cooldown: policy_id=4a1d5af4-323a-d939-d208-18672288565c cooldown=12.125728588s
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:32.142Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:31.874Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:31.874Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:31:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:30:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:29:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:28:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:47.775Z [DEBUG] policy_eval.worker: successfully submitted scaling action to target: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg desired_count=14
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:47.775Z [INFO] internal_plugin.aws-asg: successfully performed and verified scaling out: action=scale_out asg_name=maya-box-asg desired_count=14
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: eval ack'd: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [DEBUG] policy_eval.broker: ack eval: eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f token=209f37a9-d329-1707-d300-94622d061947 eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f token=209f37a9-d329-1707-d300-94622d061947
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [DEBUG] policy_eval.worker: policy evaluation complete: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [DEBUG] policy_manager.policy_handler: scaling policy has been placed into cooldown: policy_id=4a1d5af4-323a-d939-d208-18672288565c cooldown=4m0s
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.873Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.829Z [DEBUG] policy_eval.worker: successfully submitted scaling action to target: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg desired_count=11
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.829Z [INFO] internal_plugin.aws-asg: successfully performed and verified scaling out: action=scale_out asg_name=maya-box-asg desired_count=11
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.228Z [INFO] policy_eval.worker: scaling target: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg from=11 to=14 reason="scaling up because factor is 1.183979" meta=map[nomad_policy_id:4a1d5af4-323a-d939-d208-18672288565c]
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.228Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=14
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.228Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected in group : id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=14
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.228Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=memory-allocated-percentage current_count=11 new_count=14 metric_value=71.03871091033841 metric_time="2023-07-18 13:27:27.227956029 +0000 UTC m=+1037.429037121" factor=1.1839785151723068 direction=up
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.228Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=11
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.228Z [TRACE] policy_eval.worker.check_handler: metric result: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:27.227956029 +0000 UTC m=+1037.429037121" value=71.03871091033841
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:27.227Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=985100 allocated_memory=1234147 allocatable_cpu=3110400 allocatable_memory=1737288
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.673Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_memory/box/class
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.672Z [DEBUG] policy_eval.worker.check_handler: querying source: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_memory/box/class source=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.672Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=memory-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.672Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=cpu-allocated-percentage current_count=11 new_count=12 metric_value=31.909079218106996 metric_time="2023-07-18 13:27:24.672761177 +0000 UTC m=+1034.873842267" factor=1.0636359739369 direction=up
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.672Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=11
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.672Z [TRACE] policy_eval.worker.check_handler: metric result: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:24.672761177 +0000 UTC m=+1034.873842267" value=31.909079218106996
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:24.672Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=992500 allocated_memory=1243406 allocatable_cpu=3110400 allocatable_memory=1737288
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:22.182Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_cpu/box/class
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:22.182Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_cpu/box/class source=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:22.182Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu-allocated-percentage id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.913Z [DEBUG] policy_eval.worker: fetching current count: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.912Z [DEBUG] policy_eval.worker: received policy for evaluation: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c token=31cbaf4f-161d-1f33-758f-fccb1fcd22c7
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: policy already enqueued: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.912Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=32991ffe-4f99-9147-5c43-0cf81dc98956 policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.912Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [INFO] policy_eval.worker: scaling target: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg from=9 to=11 reason="scaling up because factor is 1.148448" meta=map[nomad_policy_id:4a1d5af4-323a-d939-d208-18672288565c]
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=11
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [DEBUG] policy_eval.worker: check memory-allocated-percentage selected in group : id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg direction=up count=11
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=memory-allocated-percentage current_count=9 new_count=11 metric_value=68.90688245126887 metric_time="2023-07-18 13:27:21.311640085 +0000 UTC m=+1031.512721175" factor=1.1484480408544813 direction=up
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=9
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [TRACE] policy_eval.worker.check_handler: metric result: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:21.311640085 +0000 UTC m=+1031.512721175" value=68.90688245126887
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:21.311Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=955500 allocated_memory=1197111 allocatable_cpu=3110400 allocatable_memory=1737288
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_memory/box/class
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [DEBUG] policy_eval.worker.check_handler: querying source: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_memory/box/class source=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=memory-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [TRACE] internal_plugin.target-value: calculated scaling strategy results: check_name=cpu-allocated-percentage current_count=9 new_count=10 metric_value=30.481610082304528 metric_time="2023-07-18 13:27:17.503668056 +0000 UTC m=+1027.704749146" factor=1.016053669410151 direction=up
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [DEBUG] policy_eval.worker.check_handler: calculating new count: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg count=9
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [TRACE] policy_eval.worker.check_handler: metric result: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg ts="2023-07-18 13:27:17.503668056 +0000 UTC m=+1027.704749146" value=30.481610082304528
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:17.503Z [DEBUG] internal_plugin.nomad-apm: collected node pool resource data: allocated_cpu=948100 allocated_memory=1187852 allocatable_cpu=3110400 allocatable_memory=1737288
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.768Z [DEBUG] internal_plugin.nomad-apm: performing node pool APM query: query=node_percentage-allocated_cpu/box/class
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.768Z [DEBUG] policy_eval.worker.check_handler: querying source: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg query=node_percentage-allocated_cpu/box/class source=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.768Z [DEBUG] policy_eval.worker.check_handler: received policy check for evaluation: check=cpu-allocated-percentage id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster source=nomad-apm strategy=target-value target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.496Z [DEBUG] policy_eval.worker: fetching current count: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.495Z [DEBUG] policy_eval.worker: received policy for evaluation: id=8448f264-8874-08ee-5a61-4b56104083ba policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster target=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.495Z [DEBUG] policy_eval.broker: eval dequeued: queue=cluster eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f policy_id=4a1d5af4-323a-d939-d208-18672288565c token=209f37a9-d329-1707-d300-94622d061947
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.495Z [DEBUG] policy_eval.broker: eval enqueued: eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.495Z [DEBUG] policy_eval.broker: enqueue eval: eval_id=b7df2861-bc44-138e-c26a-daf0b8d8b55f policy_id=4a1d5af4-323a-d939-d208-18672288565c queue=cluster token=""
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.495Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.249Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:14.249Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:27:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:26:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:25:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:24:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:21.891Z [DEBUG] policy_manager.policy_handler: scaling policy has been placed into cooldown: policy_id=4a1d5af4-323a-d939-d208-18672288565c cooldown=3m52.357589015s
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:21.891Z [TRACE] policy_manager.policy_handler: sending policy for evaluation: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:11.928Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:01.945Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:23:01.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:51.933Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:41.886Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:31.869Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:21.950Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:11.927Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:11.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:11.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:01.991Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:22:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:51.948Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:51.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:51.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:41.870Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:41.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:41.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:31.936Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:21.882Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:11.933Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:01.945Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:21:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:51.951Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:51.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:51.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:41.958Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:41.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:41.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:31.914Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:21.913Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:11.889Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:01.956Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:20:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:51.868Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:41.910Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:31.914Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:21.928Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:11.872Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:01.936Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:19:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:51.924Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:41.914Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:31.892Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:21.871Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:11.979Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:02.007Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:18:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:51.907Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:51.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:51.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:41.940Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:31.949Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:31.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:21.907Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:11.978Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:11.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:11.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:01.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:01.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:17:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:51.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:51.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:41.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:41.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:41.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:41.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:31.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:31.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:21.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:21.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:11.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:11.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:11.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:11.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:01.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:01.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:16:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:51.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:51.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:41.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:41.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:31.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:31.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:21.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:21.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:11.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:11.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:11.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:11.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:01.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:01.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:01.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:15:01.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:51.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:51.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:51.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:51.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:41.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:41.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:41.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:41.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:31.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:31.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:31.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:21.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:21.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:11.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:11.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:01.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:01.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:14:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:51.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:51.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:51.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:51.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:41.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:41.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:41.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:41.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:31.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:31.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:21.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:21.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:21.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:21.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:11.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:11.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:11.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:11.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:01.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:01.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:01.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:13:01.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:51.647Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:51.647Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:51.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:51.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:41.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:41.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:31.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:31.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:31.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:21.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:21.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:11.648Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:11.648Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:09.811Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:01.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:01.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:01.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:12:01.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:51.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:51.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:41.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:41.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:41.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:41.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:31.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:31.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:31.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:31.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:21.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:21.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:11.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:11.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:11.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:11.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:09.810Z [DEBUG] internal_plugin.nomad-target: triggering run of handler garbage collection
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:01.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:01.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:01.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:11:01.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:51.647Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:51.647Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:51.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:51.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:41.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:41.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:41.641Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:41.641Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:31.645Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:31.645Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:31.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:31.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:21.646Z [TRACE] policy_manager.policy_handler: target is not ready: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:21.646Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 872ae150-f1a2-12b1-2197-cd32a3b49546 is draining"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:21.642Z [TRACE] policy_manager.policy_handler: getting target status: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:21.642Z [TRACE] policy_manager.policy_handler: tick: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent: context closed, shutting down eval handler
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=target-value
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=pass-through
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=nomad-apm
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=aws-asg
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=nomad-target
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=fixed-value
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent.plugin_manager: shutting down plugin: plugin_name=threshold
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:10:06.743Z [INFO] agent: caught signal: signal=terminated
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=59912941-3a05-bbd1-32f4-ab0dd1e096f7 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=c6da0437-4355-f384-aedb-195dda6fffe7 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=455981a0-b9d9-c000-b7a4-f65abc779920 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=503fbe17-bd26-87b8-5b9d-c32c388e5a04 queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=83796c75-e8aa-3a03-8745-276efd5a962d queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=e37e1c9d-c2ab-d38d-0903-a926035f5308 queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=50c0e764-e911-187a-a73c-0524e44b4ef3 queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [DEBUG] policy_eval.worker: starting worker: id=4ae0cb33-b250-efe0-2a28-57fafb7e8352 queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [TRACE] policy_manager.policy_handler: received policy: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [INFO] policy_manager.policy_handler: policy modified: policy_id=4a1d5af4-323a-d939-d208-18672288565c modification="min value set to 1 since scaling cluster to 0 is not supported by the Nomad APM"
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.841Z [INFO] file_policy_source: starting file policy monitor: file=/local/policies/box-asg-scaling-policy.hcl name=box-asg-scaling-policy policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.840Z [TRACE] policy_manager.policy_handler: starting policy handler: policy_id=4a1d5af4-323a-d939-d208-18672288565c
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.840Z [TRACE] policy_manager: creating new handler: policy_id=4a1d5af4-323a-d939-d208-18672288565c policy_source=file
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.840Z [TRACE] policy_manager: received policy IDs listing: num=1 policy_source=file
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=d7671229-dd34-91ce-b76b-4337cd9a5df8 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=97b18944-2ddb-f23b-1618-fa681d591364 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=64837d42-f98e-5f76-4834-e0b169788c12 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=55391bcb-4862-5641-0756-66465fbd1706 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=77a0fc2b-5001-c5c6-8661-caef255a38f4 queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=ffb9a342-3d06-09e9-5f37-c473ef2e3c2d queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=6db4a1b1-7e21-857a-e36b-1f164c159177 queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=c6a1fbe1-75df-d1c0-0472-19363adc29ab queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=d5d753e3-b2cf-e0ef-8c58-900fbbbeb0df queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=69966e50-a17f-bb5d-33d3-b097db390417 queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: waiting for eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.broker: dequeue eval: queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.839Z [DEBUG] policy_eval.worker: starting worker: id=8448f264-8874-08ee-5a61-4b56104083ba queue=cluster
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.838Z [DEBUG] file_policy_source: starting file policy source ID monitor
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.838Z [INFO] policy_manager: starting policy source: source=file
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.838Z [DEBUG] policy_eval.broker: waiting for eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.838Z [DEBUG] policy_eval.broker: dequeue eval: queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.838Z [DEBUG] policy_eval.worker: starting worker: id=d591d948-e4a4-20fa-23fa-b1c119ee80be queue=horizontal
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.838Z [INFO] policy_eval: starting workers: cluster=10 horizontal=10
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.832Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=target-value
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.832Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=pass-through
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.832Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=fixed-value
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.832Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=threshold
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.832Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=nomad-apm
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.832Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=aws-asg
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.809Z [TRACE] internal_plugin.aws-asg: Using default AWS credential chain
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.809Z [DEBUG] internal_plugin.aws-asg: setting AWS region for client: region=us-east-1
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.809Z [INFO] agent.plugin_manager: successfully launched and dispensed plugin: plugin_name=nomad-target
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.809Z [INFO] agent.http_server: server now listening for connections: address=0.0.0.0:22193
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Nomad Autoscaler agent started! Log data will stream in below:
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent:
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Version: v0.3.7 (90ad44d)
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Policies: /local/policies/
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Plugins: //plugins
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Log Level: TRACE
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Bind Addrs: 0.0.0.0
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent:
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Nomad Autoscaler agent configuration:
"3ee46194-d35e-9735-6d74-22b74a3b9b46"  2023-07-18T13:10:09.808Z [INFO] agent: Starting Nomad Autoscaler agent
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:08:39.914Z [INFO] internal_plugin.aws-asg: successfully purged Nomad node: node_id=ccf0fb44-b1e2-a0b0-d9fb-9ced07fe3d77 nomad_evals=["1a9acec8-7f3c-3fba-2091-9e9a98212e7d", "174ea3af-4086-c381-4ce0-aea684c9660f", "583ac626-3752-35d0-785a-2dcd52f4664c", "fdff5bca-6610-602c-3e25-90079cd8f98f", "48a64b76-0d69-becc-4c76-a49d2881d080", "ae9e35c8-b2ff-6d8f-c138-79b52b1e7de2", "bb429b75-19ee-8789-987f-145ed729ebd1", "c21974f9-6276-b0ed-4a16-5ec7b6d179ce", "75451a05-3a1e-e5fe-6e92-f89f501dda14", "cdf301d6-427a-1fa8-5940-394fd6ee48bd", "95d017f2-b90d-188d-ee44-ec466e2386e1", "e0d33597-15cc-8756-38ae-29e74beeca79", "2589b989-c360-9e5b-bece-026cc897d44a", "4a35f39b-87f7-bb10-56e2-e500f553537a", "2143a466-36b4-6a03-524c-129760662c61", "b733834a-5b20-a517-5890-eaa75a9d9b8b", "ac7cc2cc-efc2-8ed4-a71d-6bd6a67982cd", "9a6c49a4-616a-b188-c2a0-151732b0012f", "d455d37f-7662-bab2-b4e3-5e14e298c872", "3d254cb2-2b03-bec9-5b3a-a8f4ee0f35fb", "b271e128-98f6-e0f5-5fab-00ba94a2a210", "fd2ff18c-74f1-9ab6-8e5e-1fec2bfb7460", "d309869b-2d73-0e05-d88e-7ead5765a5c9", "014a9672-b58f-f6f7-0e0a-00e962c9d30e", "07fe6253-d6b8-fe2c-2489-f7d99215612a", "fd427a78-2920-0767-477f-412e09a3dc68", "2c54f84e-0ca2-8791-8c6d-f9fc64dac429", "5cdf2d76-8413-7cbe-2b27-5d0b2b3ebe7e", "865c04a0-77ea-acc5-ccd4-3134135a80c5", "b7ecea6b-a1c3-e31a-4b9b-fb64e0e73441", "335941fe-a191-5bbb-2ebe-a9cde20f3316", "4dfc572e-74b8-cedb-745f-8289b691ad82", "3f81451b-220f-1dc3-119c-e4bb57252f3c", "ee55515b-e22a-79aa-8e33-72afa7bdd266", "6287cc7a-af75-5006-054c-fefa995e5275", "178f378e-26ef-4e68-d3fc-e9721be0f0d8", "5e6fddf9-a8cf-a58a-3c38-3edc0892d01e", "cd020fb8-1294-d9c6-bf45-31a038990607", "30b867d6-52b7-7d24-d40a-ff744bb394b9"]
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:08:39.876Z [INFO] internal_plugin.aws-asg: successfully purged Nomad node: node_id=66e6e80c-06b3-bba7-2636-0aa63b0cc0a5 nomad_evals=["262399d8-e5ec-fd06-345a-bf019f9abe70", "f06fee7c-30b0-c419-f5eb-2da85a9e2d6b", "dd4acbd7-226f-1b5b-ca1f-f1f783b21dab", "e07ac7b8-2731-7f69-1df6-52846ab3bf46", "e7fbe618-305d-1d70-b97d-3843a0ede10c", "5b03c266-dfdf-873d-e7ed-579746695f52", "45b67bd1-7604-30d0-16ba-a4bc95a41e0e", "4517cc6e-9f4c-6a5b-6b2c-75e56a07d44a", "8d14657c-6229-7cbc-a41c-3455db22be91", "d2d94378-3226-5430-3ec9-08969bac9691", "47fcdcf0-08cd-86fc-ee1c-c6af1dd7fc1d", "b3795554-dc17-10b4-e866-76fb81648128", "732aec51-f41b-02af-e6c0-0f10ca6fe6c0", "8bd6701c-4b69-e3cf-9498-6ef1451541fa", "4b7c9582-ba79-3d46-5f80-1f07f6bdc0d8", "ce75751d-9aae-6d0e-e4a9-ead510d7fe16", "7322ba75-8214-ccce-7974-c5c87dff6d8d", "3b153486-75b7-2f5d-528e-f33570a5e8d2", "9c02450b-2b36-4e39-ee35-de9d469b4056", "f1b7d484-7de2-953e-a0ac-06196b1e4b7a", "b5b503d2-f6cd-041c-afa5-7be5426c3667", "076ab3f7-c94a-4ed5-f315-843c339298da", "cf1e3291-91c5-7ae5-1504-efe6d83b96ef", "da5d24e9-d043-7807-9d75-61ee35c70182", "8a353ea0-13dd-913e-b816-48c0e5dcd6d6", "2b0284e2-496d-569f-d8c2-44a6f7d38d07", "0be38074-b8b6-5f72-b16d-78087c694e1d", "346a0221-8b63-b605-69af-6e62097d7c91", "3de1d903-3060-63a0-7946-4a9dfa82372b", "9384f810-f350-efd6-e193-43c1f7aa01db", "e77dfe58-3d22-15db-89bc-4622b0083325", "bc828299-e4a3-df76-3fdd-4de247674ca6", "30f6d771-b274-0204-c613-ed6336c1b6b1", "49c5729d-d78d-7ca3-eb78-4c4754f97b6f", "ae3cba45-c797-2a6f-3bfb-e270be25976c", "71409aab-1911-59df-0cac-0c35629d74a2", "0f533d24-498a-0ec1-9fc5-3318d1280c09", "e60d52ab-a834-cdbf-338c-6aa8fce84be7", "2592c645-9f68-c140-9e5c-462450d50e90", "1dcf5f2b-0660-e3d6-2c3c-65c9a9e71009", "a5da59c9-e63e-ab64-d15e-690faa04789a", "0b2e496b-3d8a-a0b2-d014-554dce0a8ade", "330ab963-afe5-0f43-7691-a0905e91b277", "180379a5-abbb-6f3d-08c2-3404129ab30a", "40863a55-f3e5-31d0-aaa2-bc7b0910746f", "ceb6e04d-7cda-ba5e-b052-61e5926f8ab9", "8f46b00e-1368-437a-a42d-b0b6f80b3de0", "5684485f-9f35-9406-f79a-f5d90235a73d", "4c651786-36c5-f581-c396-99530a119e10", "93d3bbe5-6ec1-3357-9fa2-49c09b1de479", "963d0b55-d440-98d1-839f-5ff359864c0d", "76b9456a-c8cc-ff92-904e-b8a5cc7990c3", "3f06862e-771a-08e4-5c19-f600fb11b09f", "944ab4cb-2f75-e1a5-db9a-d9238cc4bd8f", "4998d006-c333-a72f-08a3-276fa5f7df00", "c4e91355-6b8c-24c7-f4f0-bf823139aa3d", "dca2edd1-8f75-4a59-191e-7576bac59c9a", "75d435fd-3802-b6e5-bbb0-ac167f212edf", "6cfbe807-1d65-8831-f4b3-ee549984174e", "4fe4b83e-f36f-a4a0-6427-6d98db06d2cd", "a45e9c03-3f23-742b-6ddc-1a0fa073fa10", "97fdfa72-9532-bfaa-f0a2-a10c080decfd", "2e916ce1-931f-73e2-d14f-fbc2f73e0e83", "6865ca61-beff-0151-1238-c26ea797e7d9", "0640caa0-0b13-a64d-984e-5f12fd2faf3a", "b829c445-08c3-edd5-aa7d-afa0c5743bbd", "f13b4a93-a99a-3c04-52c1-d6bda39dbde1", "482edaea-5017-8518-bb4a-f2aec670c0bd", "fd7b4ca3-3ab3-2492-fca2-a8069a5f7e4b", "93da445c-b9c1-6e67-ab41-a70e51ee014c", "12bf60ff-af6f-b9da-266b-7b31e90c60ef", "ae72f384-982d-d1e3-b63b-c98cdb9084cf", "9cf545fc-9079-7adf-fe8a-62f76769c3b2", "bf06aced-07f4-fc5c-03b6-6106810b454b", "0548ef1b-14b7-5501-e329-8ac4e79feb53", "1bbccede-58fb-7749-42a0-452a3abfb847", "e5e96ee8-70c4-3fd7-6f55-3579d827d922", "46b53a86-3a16-97e5-0354-d920eae58896", "4df4def3-0419-eb77-5d92-5fcf2c36d088", "9942513b-7663-106e-32fe-84b999ac963b", "96a545bb-affa-56af-ab95-6adb851b37ed", "6b387198-f8b2-8768-1adf-6286ca4e1b9b", "6c732cfd-bc0d-62ad-e17d-4ddbd8477f04", "47467257-58a4-5123-6381-c243474b369f", "88f6739f-795f-1421-8b52-9e81c5c20c98", "bb72983a-52d2-8a12-1dd6-6ec89e5cf427", "206155bf-6fed-0231-3708-3abecd5363b9", "81869e10-0363-89ff-acbe-f975a5a843b1", "3ce6c210-321f-c295-a79b-bf9df7ce0e2f", "602f5660-4a9e-6879-e470-1ae9180bd785", "4625468a-d128-75c1-152a-b4370c45ba6f", "d34182d9-ac60-1263-bdd3-d4221b1049b1", "56e9137d-ae5a-66a3-b23e-381333f69def", "6a2fe8e1-e26b-29e5-332f-b1a1be38b2e6", "ff413b48-cdbc-1364-92f0-7bb2b032cd0c", "82f5b4ab-05e7-d27e-0bcd-d4fa1a9c8dbf", "2a8ce8c0-39b8-4d29-2756-2e482766b6cb", "449d5fb7-54aa-e054-b201-185164ef877c", "434685b4-b71f-9624-9be8-f693f71630c7", "315cddaa-720f-6cf1-5f49-cd83ffd7f90b", "f9b4cfc7-d043-fbef-95c3-af43f03511ad", "f2a7ed4e-51f8-8ec1-2e46-02944e9a536b", "cf5b84f8-9e81-dccb-67b2-1d3bf88c58ea", "b0739965-a024-1175-d233-49a491a4d379", "32d161da-8414-2712-4046-77cbd3027254", "24ebf687-3b41-ffc9-557e-b0fc3551f58a", "be76c94a-361c-082c-2877-5d9bed4840b0", "53b90bc1-701a-d8bd-015f-c4d097f6e137", "83e89ab0-baad-d7a2-c292-4c642ad2746c", "db089cff-7eeb-de3f-99bf-57567d47201a", "f7dd20cf-c671-6ac1-fb7e-01df1dd4dfa4", "6cb878e7-7672-15b6-c74c-56f04d9a7c4a", "d158d7ee-bf63-520a-ce7b-5468f58277a4", "aa2a97e4-62c3-a307-224c-419ca4db80c7", "bb73ff94-ac5c-febf-6ec3-7d5166638b2a", "c845b7b3-7d01-50db-2b6d-7c96daf8b1c3", "41780993-0bdb-af12-8024-6d62d186453d", "4a53af58-91cc-b643-cc1b-e817eb834182", "085d6f29-c11f-64e6-3062-1ccdc86c10e4", "16afa11a-bbb1-b4aa-f023-029c19522144", "465d9e85-0a41-9812-f689-82172bea0d01", "8d7b5e5f-ec75-8558-3851-9a4b2193738e", "740a8bf9-02f4-8bef-6474-14af45c661f2", "ffeceb47-19e3-c865-823a-f014b0eb61b8", "5abe7891-2437-3bee-e5f2-2d3ccafff2e8", "d959f238-85b1-5b2b-cb10-3af2798ba1f9", "cd69de0d-a0d0-e140-a9cf-1f6bbbf7655a", "36ea253e-c9ee-b29c-56de-634e94b77e52", "b2112ac7-2892-dad0-8fd4-995561139ab9", "a5fde31e-0eae-a047-26c6-0f68e4b370ef", "429ad747-7972-0177-ba50-4466dd193ff3", "d09ad638-e9cd-b43f-aec1-54af61fe1ba2", "95536af6-d739-6674-4677-20f89bdd8643", "4ac84f30-5433-6803-e765-c2884b6e875d", "ea388e0f-4ec4-6cdd-2f82-bacaf34f15d0", "8fa3f9d1-b64a-723c-9313-cf410111c5db", "0bb915f6-f8ba-1b78-742d-7db414200920", "da4eaeee-575f-7aab-f410-f90e29618817", "cac5671a-ccab-0f44-b864-32c112cff7c4", "e8546d99-c728-0db8-900c-73b28a97ff37", "b5529839-4552-31ff-a5ff-d9bc7c012ff8", "87654310-442c-26ac-1f8f-3716839b0b65", "ee49e6b5-c9b7-23d1-8696-0a2b081c2ac3", "f84dad69-d48b-f19b-cea1-99f26287ed1e", "e89c1543-b1ca-84d1-d626-a5b30af39bf4", "607fd1d3-0eaf-dc1b-2c77-03387d34222c", "c8450372-d599-13bb-9125-9f9b704372c9", "b3a24e78-ab7d-2e11-0263-89f04a057712", "937849a2-f2af-d059-d636-f553e0ad63ee", "36584bec-8d06-9688-5314-fc764d71263e", "7ef8418b-b568-4fdb-6c13-2e2d1682b62b", "8fc27889-afdb-e4fa-8075-d951b675ca71", "ec859df8-0eca-c280-4144-a75f4af03d33", "3089332b-1965-e983-694f-3a95e9de97c0", "72545af8-51eb-da1e-5dd4-2c7cb26db3b4", "cd9a1303-93d5-5c79-3471-91b8fdbe3f1c", "bccc2fcb-a01d-d15f-263b-97e125896bf5", "441d2ad3-7c68-9a2d-5e36-c5ad92ebea49", "69d812b6-07b1-e567-11fb-7668d551f2bc", "d69dedb2-f0b3-0fdc-30c8-ba5799c9ee97", "28580a50-1dcc-ccc2-4f9e-2e6a653bce1d", "c432004f-d355-aa85-a80e-c255760e4a38", "7f316a67-558a-9863-f180-aab57c02e851", "e56890dd-6765-b595-4335-13ac838a2ef7", "0ebb0dc7-db44-5319-c214-8c0121f6a9be", "00b38c18-9272-0ad7-cc12-2d202ab2950e", "b0347794-c00c-f3d1-10e6-9517b46666c2", "9fecc959-4382-164c-86c0-e17ff317d142", "faba1cf2-8431-9cee-fd33-0a59dbb0a444", "7b6dcbc5-3a33-a3d3-a3a0-03ebdddcb01f", "6b77c2b1-9859-5ccd-e5ad-d98be09f7309", "e358b89e-398f-838e-b2f0-e94675a1dd91", "bb19b2fd-9b18-33e1-d77f-a0c638cd1e26", "81ab801f-8566-97eb-a24b-b425339222cf", "be26a9d0-ac92-e0d8-9644-aa29595ff14b", "db02ce4f-1c15-bb88-6fde-7a5ceab7067f", "661009f3-719a-aa59-e53c-2594549e5d34", "6d4e4131-9e1c-ac74-9ebb-f866aa1058f4", "de568052-e33d-7046-d1fb-d9f281528f39", "8c3ca6fd-6023-e9a2-99b3-aa493bc9afde", "bb19aca2-530a-3eea-b904-bfae633f280d", "9882f2b6-a7ec-606a-4d6f-e78819b5f903", "8cf2f97a-a81a-6c62-d2d5-0e275caec9d4", "6c4d939e-dc68-ac52-8190-bf0256d059bf", "bed70f97-7d4a-3cc4-ae84-c6c7ad192a2d", "d7be4829-842b-3ac2-3f27-c41fbadc9ea9", "38c8809d-0271-503e-4b64-fc352424aa8b", "aa73cc48-664d-a7bf-7975-cf237b5a6182", "24419221-1476-d1e1-ca79-d0c360f4d6ee", "d0fcc0e4-7429-d067-6e90-d8312e0e89a4", "654e9ba7-5c24-98d6-b8e1-5406194b6b60", "ca2e23f1-e24b-878d-4535-d685906db967", "3941dc38-f3af-f1bd-e1e0-c5e8ee6e0df7", "94397835-1ab4-0316-50bb-afd7b8243d6d", "6e27cc5b-11b8-9df0-6dcd-12f140128e11", "dd8c8075-1f46-ea1b-0510-206da48a7ca9", "ef61d485-0d43-3d02-3a91-8a886a9288b9", "d4cdabe7-16ee-e697-a2e0-77d8f1f750ea", "a4663100-461a-3c0a-8d86-f4aa2157412e", "40341fc5-bb1e-08d1-eb8b-91a62c25541c", "55968c33-3138-24d3-c00c-c7354dff3763", "81dade8b-3785-b89a-ebb3-353d712fe9c9", "b349957a-cc60-0331-87c4-9df53d252c05", "f7858549-42ac-eefd-d670-6420467925e8", "26a05c06-b48c-9254-2907-2a83428bba83", "79e11381-6f36-2200-cdce-b49e4543bbaf", "cad7995e-09ac-8bf6-b7e0-a4e57f110fba", "2969552b-7983-d424-6e19-d285276a35f3", "729b6d14-1825-951b-15d9-e595e9c41a2a", "24907b41-f6a3-61d2-b831-31034ce7d0a3", "3072ed76-9373-434f-7547-168772c59b47", "a3da7f23-06a4-92ce-8deb-bff7337dfd22", "2fd28150-bc40-2a77-3482-6af2cbd1883b", "08fb232a-1d3a-726d-25e5-6a2afa4a7cdf", "4a505465-f10f-ff53-c3c9-4fe620055fa8", "8296c0e3-cef3-b689-dbfb-59350a1f4405", "40d89bd7-dee9-b988-367a-eae660097a02", "78c548d1-d2c3-a4bd-091f-b90197bb1742", "d5cd70c2-21f6-ab6c-df4a-5b2de551af47", "e8b9cd44-4175-7dd4-6a0c-fb72b6cf5e10", "ac714e73-4b40-b84b-8cd6-36ad8a58f36f", "d5e77cc5-9f10-b13d-ad33-77a7ef4c5746", "55d07b10-4dee-258f-81a1-fde66229b327", "91ed1a80-27b4-2132-5c5a-5c1731bb4069", "a3786d6b-3367-6075-f536-3a6f92464722", "7d993706-0694-7098-f812-9cab7b39afa8", "be08da8f-b9f1-4c95-a5ca-075812511bdc", "4151be3d-8ab8-1f5c-d37f-b2705152b7d4", "43e9834b-7ff6-060c-b06a-488013a7dd7c", "1284ff76-5670-393c-3a30-32405bbdc16a", "10afde08-ef6a-e1f7-132a-23797ed09d6b", "b5e02de5-45a8-f56e-d3fa-773f6eea50ed", "e06a4e6e-1d14-dc5f-5341-25194b45ed5d", "a19e56d3-d67e-c8f0-ab99-7041b0b2cd06", "6857852e-f0b3-a2bb-0cbc-29ec3043272f", "ee586261-b2ac-b37e-a833-9db413e74843", "ae1feec5-ae7b-038e-d291-6a5089b0d091", "6ffcaa9d-74ab-7e32-c783-f547882524d5", "15bf0024-cc1d-c4ff-9410-c121f398b4b4", "6cf118c0-087b-3bba-a4f3-3f631b66a124", "eb15d03f-ca09-8f14-2d29-2ed0782a6938", "0092f61e-f2a3-e259-1bd6-feaf4670a1df", "2ee681fd-0175-2bf6-49e2-4ce2e82cdeeb", "5184cae9-96a3-32ec-4d1e-76b0fd32f8ea", "5ca40162-9c98-022e-53dc-156ee3c69835", "22d83849-d23f-736d-d035-b2d19fca490f", "8ad66515-6b33-2561-3800-7a677f42a500", "dfca2bae-0392-db8f-8e2f-055666852543", "4bfd8c5a-3893-d5ca-54c2-15a053512250", "d3fc0637-165d-d97c-7617-e93713f58156", "f3eb9bdd-2d18-5c1e-a591-043d8f160204", "bc0d0ce3-27cf-293d-694c-618fa9b870cc", "a7580613-ba4e-cedf-da28-51b15c560df7", "12fbf00e-c0f3-0c61-dd47-1f9e75adaabc", "7844a054-59f1-b44a-1dec-59e12b4e82ba", "31dbba6f-b6f3-804a-5ffc-9c40f3b95b17", "36768630-0bb9-e74e-941f-3e9af5c5406b", "0fb1567a-b6aa-2d56-be05-76da0ad98d09", "e0884486-0a7b-3b2b-e9d9-f37bacdbed33", "f807a736-83d0-a3d8-6f99-075eba67d353", "e5cddde0-ea6b-2aa2-2871-0b4be69ecd6f", "542e14ca-23b1-7ae9-93fc-a0cab1bc32d3", "e309b32e-1aab-3b92-1f88-52e083610981", "14d4c174-3306-4100-e1c9-619ae8aaa47a", "9d4d0e58-6b82-9328-b7b2-0b3ed9d3bf82", "4d7ed6d8-f881-8e45-e055-b05c3c9c0f3f", "4db0d256-cb48-ca44-134c-86be478dbe65", "c77bc214-da96-fada-a8eb-555897c074c9", "16d35e5a-2f4c-20df-d5ba-b5becdaa6f4c", "3678266e-8ba3-b753-9214-16b740a8e047", "45e04dcb-8a8e-660c-0155-b0a54b352f89", "52dd44c6-439b-1051-0920-ac7eb04fb9dc", "efbb9ff2-f8d8-6c33-db96-4bc1c0a8a44d", "8a0172d5-38a9-b322-dedc-383ef35ab1e4", "86f69da5-96df-9298-5759-796b5205f439", "922af09c-8098-48a6-4dbe-b63a33bdb99b", "71286262-8be2-b158-e8f0-1cfd906c69ad", "6df8d019-6095-9cc8-dfc6-6c1cfa81f981", "0938d768-60ed-be5e-1f59-84222fa78eb7", "004e16a7-93c4-8fae-4e40-0954a41860b5", "187d116a-0bed-1e84-163e-b83199227b45", "54787908-eba1-4bcb-6571-e7fc650beafb", "7db1d388-38b6-03ea-2e60-724663239f8f", "38ea9725-c922-5f49-da4e-9b4c791c593d", "3e492d07-9775-d167-b2a9-9f75d9ad1569", "a6fa4d5c-9735-7811-187b-269dee007832", "fd088971-4135-663a-006e-2a59064018f5", "2d61a0c9-617f-b6ab-8036-27d78317b090", "6b269ccb-7d9b-298c-6a33-7df53954fd58", "28524333-81b4-9323-fac8-df6f291dcdda", "078216e3-48e2-4d42-3e4f-ab5b966c9b7f", "8288b74d-8989-9908-6b1d-a34c005cc867", "18cf337c-4842-8825-e1e8-a6d379bdb23a", "fdc8f2c0-dec4-b11a-89aa-b87ee3722b38", "596c8b6d-77b0-502c-0753-56c86fd0b2ee", "c9581ea7-4ea9-79f4-3051-9d1020d35d5a", "ecb57cd0-e72c-8b0f-a08e-196278007998", "ad635c91-dd3c-71e2-5048-f8aab4e25b56", "2e392582-b5c4-ee46-fa2b-2e405a327318", "ecbbae31-55fb-373c-93ff-7964f320eb92", "f14fcd49-1717-2caf-567b-8b4eba435b59", "97e8590f-84e9-2670-e046-524ab5875e02", "c1f891f4-6b25-2502-13d7-5e41987bedbd", "e3029fc7-51ef-42d7-7acd-01204fcbaa0f", "b426a59f-73b2-c6bc-8cd8-f90ecebd156f", "6a3e8c57-9038-e40b-c20f-f5ca499be6f8", "8d653d1b-ae7f-cf87-95c5-000e8dbaf20e", "e9c54bde-4da1-fca0-0253-0b73cc93b2a9", "0630243a-781a-4838-73e3-cad0e7daa81f", "539f6880-146e-8103-4005-3e185d313f5b", "9fb6f800-56cc-f076-ffcb-9b5f9530128f", "1cdcf1f4-f0d6-6191-6448-7a641e450d9a", "cda62c47-b424-afa5-cf29-35de852f0275", "b364b287-bb0c-070e-bf34-ac9b0db57786", "fd3afc28-6766-449b-3a97-9d885abeb22b", "02af7d90-6ecc-c1fe-0431-87aa942e9b75", "4d54af55-a9e6-cd02-9b8f-ced1a3b50473", "83ebac89-a5c4-b881-45b5-5b488b3ff74a", "51f64d06-8ca5-99f5-0f52-a485930cb81a", "c2097b7b-79be-b3eb-7093-eb1590f00839", "446846e0-490a-c4e9-55a9-25f8142d2342", "7b1c36c5-14ec-d3ad-efef-1a195f4a533a", "3cb86a33-d524-581c-0506-9ef0b819ff38", "add635ad-cbe8-856d-82d5-ff844b61b126", "93134310-9e91-77fa-2c64-4dc067e586de", "7090a2e6-237f-9cd8-c284-b36170e7d523", "6163ecfe-b9bb-61e1-738e-388127a5a43a", "436936bd-550a-8c3a-5d15-aeedcf52bab6", "9cbcf85c-2441-0eef-2b33-e46f3aa02bec", "aeabf201-48e9-462b-158b-cdfbca0437dd", "6144c96f-a321-3452-965a-983d31112133", "65d68626-4fb8-e014-4a2d-189947439493", "64b400c1-6dc8-d31d-d405-bf598bcd5b17", "6ff5c72f-ae34-382d-6050-296d5e4ee1cf", "e8c91aef-149b-c7be-2b90-833611053e72", "e883cabc-04da-5366-28ce-dbd12906b58b", "752d06dd-8543-d4c8-d7a9-36aa2ccfed5e", "2735040d-815d-19ca-ef05-91a2bd489485", "a22214ca-c30d-7e7f-de31-20398c3dc2f3", "501b182d-bb89-274a-190d-9576d27ad5c3", "1a80e1ae-238f-1d99-f693-cc92e913c61d", "a03f3737-aab4-36e7-3b96-bd5b462da296", "26ef23ac-d9ac-72c6-ad2b-ad1852f9cf72", "24b6a4a0-b599-b537-1d97-a48be886532f", "9bc67b1c-52d3-9e71-833a-45d3c052dccd", "59e7188f-386b-9a2e-a649-961b8958dd54", "857f2db1-9802-5577-2285-0d844346783b", "c4b9a772-4256-587e-70bd-fbb426be0298", "f1092af6-3a20-1793-bb4a-4c94c2c30e99", "64a72a9a-dbff-440d-3cb0-adb744e0d166", "56a1c31e-823b-80cd-229e-fb84a0f7f62a", "a134967b-0a0d-dab3-6691-907296ad5fab", "f50f73e2-9a83-4974-2e66-b37d1022bb68", "390c9dc8-1347-4765-15bf-b3de7cceac3c", "e0c771f4-4fa7-4dc8-3395-2b74754926ce", "9a68ef80-45b3-e861-184f-0a9fdde132bf", "38b9c71f-ea2c-592f-306a-bf20f9a62f86", "744d5fbc-46e7-0a76-5da8-080f09f01ef7", "d39d387b-a8e8-ff2f-03e5-a0a48919d3fc", "cfe4548d-9dfe-20ce-511f-dccf3a98fc18", "f0288c9c-c154-44f2-dde0-4ba44128484c", "9201ca3f-7475-dfbd-6a44-0ba0c8a443b1", "1ad3dd28-84b9-4f7b-fab5-78eb69e9ad1b", "2a13edd7-47d8-6143-bde5-1ff2b97a428d", "0aaffb8a-fe84-2f6a-bda8-f4617ea19458", "d05df4ae-2fd8-a32c-f161-d5671618fa35", "ca995765-72ba-0549-dbde-b25cfbb2e3d4", "af3e9361-f74c-29a1-8e4f-e792d94cb0e7", "7bb1dfe5-fb5b-0dd6-f9f8-88e57a95f81c", "fb6e3d2b-0d42-3ac0-2ebd-4481eeb96906", "96ab43cf-4993-20ed-cf7b-8fc022493b2f", "1b979e42-7664-0003-833d-7139038e318a", "8f6ee68a-69c7-a887-7434-41bc9db30c26", "7bc76120-15d1-a43b-b1e9-1fd180fed206", "45862d25-d7bd-af3b-4831-c98848a4b94d", "a5d6a605-91ac-07d1-1f22-a00d0ce04d8e", "f9cfa3a3-085e-27d4-c7f8-4817914d9567", "40b003f4-5ab9-1eec-463e-927a2d5816a7", "cdb36e56-9c79-6d2e-d967-3c7157291252", "327ba461-6b53-8a06-4745-0eb8e9768c12", "01e219f9-4baa-ce4e-1b6d-34ecd0462edf", "5e215178-bcdb-3f7b-c6dd-99a0711b0f80", "6fb26f12-dbb4-22f8-a244-8affa2aa9b5a", "3efd6ec2-dba0-1ea2-148f-bf669d44536b", "0f0d9f16-572f-8c61-5da3-4d094f6c4a0f", "b09a5ccb-a916-42c5-b68c-90f484abe555", "876ec709-7de2-9746-4fc6-f5f94bfecab2", "0983a934-654f-d8ca-7fa7-e0042e7dece9", "4163b9ee-03d7-e21d-e29f-896b85d04cac", "f69315d8-924d-6442-2146-5a43af95489d", "b93c0d96-9386-da47-feac-b579523069c7", "d642d04a-4343-24e8-0ebb-74cb4ad2d47a", "96b04a5c-687d-4cc6-843c-e517ee6ec92b", "140f4b61-5386-054d-ce82-a05300b793a8", "daa15287-e2ee-8e0f-1483-e09eab73b45f", "2206b79c-8547-f3d6-447c-1c7e9477ffe7", "10749c7b-0d6d-909f-1c50-6bc0739e59f5", "f69cf15e-34c2-695a-fda2-2382c65fe087", "aaccbba5-7063-8360-2ca1-c3dbf4b5fd17", "574152a6-2a08-e6b0-e2e0-f0efc1fbf5a1", "d60bea40-a047-8104-3141-a26bb2a4305f", "7e9577a4-ab48-f1e6-eb53-527d74990e7b", "363ecfc4-24b9-97e2-2b85-ab02e1fe00fe", "5e325390-92a5-1680-cb8b-1ab48bff3559", "727ec8f7-e3ed-6159-bac6-5c1411434025", "d20cee27-a98f-76d5-e886-67a90d5ca7fe", "2599f72f-1710-b85f-2852-73e8f45d13b3", "6908d31b-7218-9d2d-aa39-f9899a7cfc6e", "48f00126-74ad-d3d3-000d-64381394a337", "b6acd376-6d2f-ecc9-c8d1-7d728bfd0d46", "55f3cf29-2507-3dde-990c-a866c32820b8", "f30e1bd6-d1bd-2231-2801-de3d3a522ca2", "e5260017-fdfe-9df4-24af-59d35fbd2d72", "d8116503-3e22-9415-fa9d-722017561158", "e167c111-1ad0-58af-8037-ccf40207dfd4", "80047732-f9a9-6d86-ecb8-bf511f27335e", "dd97d53d-2a07-abb3-da8f-09566973aac2", "7fa77d3f-ae40-2607-10c1-13cb88c16ecd", "735717f2-bf1e-d45c-0c6c-baa1e7aa25cf", "03dfffdc-cd7c-37d3-6dfc-ce29fbe5cb63", "3fd7e68a-1634-56d8-453a-fc3f85e44c00", "260fd0ad-9ebb-4a93-bd6c-27a3651ade2c", "6a24ee63-5d21-4e58-93fd-86a0d6e0ea65", "02ce1938-fc37-e530-99aa-6da44b1da1c9", "368d733f-5210-c3a7-7220-c5b488573ccd", "fb790aa8-d3e7-a778-330f-efd5bfae89b0", "f12a36be-c327-727e-4525-0460afdc4036", "3c6a729d-d80f-9d4e-cade-cf7b38593459", "f80d34ef-a570-dc18-f1b2-e01cc8dc9118", "317584ce-be1b-2bdf-0170-6b9cd8ed91bb", "0eace6de-fb66-428f-3742-0c1db42c77f8", "6340fcd4-92e9-1d54-00f9-2941411e83b7", "4253536b-6a49-7f6f-3325-41e54ecea38e", "a6ee83fd-570f-aa01-7845-e57756c6098b", "efa89b38-74b1-40d2-f5bf-455f6555a299", "93c279bf-b31f-36de-bb40-e047d73ac777", "1225b828-c7d6-6cf4-0d41-78d2fa93c1a7", "3a7245f1-a7b4-74bf-57c9-cfa66b058944", "68b5df23-cb4a-2b4e-1649-2d437606c3c6", "012f4866-64b7-1d4e-6592-f75b5173b90b", "11a03dab-1443-2e1d-a8f0-0f6de5150d40", "0c2351fa-6c08-948c-22ce-f65fc330c3ba", "e012e944-b10c-e90e-fc75-4e9767c91d41", "ad59b3f1-4b07-79d6-7eec-fa14bbc3c744", "bc403ce6-9409-472f-fdf0-32d77fb90768", "1b729741-2102-ecc8-85be-dd2e87e70444", "5aa8b196-90bf-aa0f-ba3b-d7a40477930e", "700eb754-2149-9ea7-b1b2-a7c2a62e9b8b", "f835a0e9-2314-6e3f-c481-9ba705280259", "e2b3a075-129e-5bc5-297b-41bce6f37e64", "032f5a8a-8122-2a1b-f2dc-72853e3979e6", "afd667db-e8bf-b3a8-80ea-ba2e17ebe9ec", "6fab27e6-0d42-10c9-6a04-f5041d769b28", "d26928d8-4065-ed5b-437f-7ee1b5135cda", "df9997b9-4f38-910a-4f81-4b5bae0f080f", "232f203f-0c5f-a909-cd0f-bc9022a176d1", "e3a01937-2003-a77b-a59a-b5bd20701084", "8d4fd995-6911-818f-ad64-501885ae781a", "f8f66520-37d5-8f0b-ca74-c3d714d8a7d7", "59eec781-6e59-fdc1-3325-7e7e1aa3d994", "dc324012-8444-c155-8a40-61265b282c7e", "c13083fa-968b-6c2a-773e-83006a672b69", "425e195a-8226-3f73-fc8d-e7b3b9189389", "5336ff1c-f8fd-ce6a-df63-99070e7d4cd4", "6ee6cbf0-e534-c340-cc0a-b027331bd066", "4e7ee0fc-26f3-03a7-8b08-5e0a048463ea", "a93c4210-ef80-757c-8318-950e47ca89b5", "d06f99f5-5940-89e9-8247-a2909e59ef3b", "337b2d74-eadf-79cb-751f-f26380cf65a1", "8490c7bd-ac64-28a4-f7ff-f6b36f893b97", "b6517419-d13e-8284-51fb-9a6353478169", "ab8cde1d-9bf0-2c30-2bc9-c54928286bd2", "b79728f7-3d75-aa08-de5b-b7da957f9284", "c93adf30-decc-3a33-cdfa-9c786fcfb287", "75aeebbb-7a1b-5f1f-91da-6f3ed0fe9079", "371290ae-0275-7aa6-3b5f-f3c72d7ae40c", "eb0422d7-1394-6d6a-9737-7ed9e68840ac", "570b53c1-f58d-67ce-9c96-66ad72f67454", "3c98b5b0-40f4-c706-fde1-fa4a932dfb3a", "4b679f72-bd2d-5125-cab6-9580e80f6b5d", "f6aa8299-c43b-c181-e4af-22d085add5b8", "ff928575-8334-68c5-707a-f3792d6162c5", "a2ddf296-6324-0963-2c7b-2429ce34c9d7", "1be13270-29c0-5bcc-becc-e1eca01eea0f", "051705ce-5297-9318-2c95-72ecbe5bee67", "ef5f460c-da95-9b1d-7934-250ad33f1035", "cfc9a05a-92e8-23be-385d-23809f01f346", "233da7c1-75c2-fea9-4080-2d67b4efd67e", "fafdd07d-816b-d770-ad74-b41ec4edf147", "6b05cab1-19f5-2306-2452-a2d51be2f1bf", "2bef2894-ac31-af39-30de-33e838f18edb", "436047ee-8f51-1d49-0a63-dd26b25036b0", "6daa7753-80fd-be7d-6e8f-713afc5b22f2", "e340e313-959f-fb9a-a8d7-eb534b46b2b8", "2c77d2e0-fb5c-e15f-bc4b-3c0dbf0551e0", "481b942e-bc0d-0281-12fe-d4cd0a44b3da", "24df2bcc-fd14-722a-04dd-aa7a0f79bf40", "b92227a4-46b2-34bc-45c5-ce4fc056f40b", "ed4778d5-7a10-6010-aa26-6f811704c454", "cc91429d-3706-5071-8c59-c29d5b6757ec", "417feccc-1346-aa14-e928-51b76696ae72", "a4d4a78f-8d66-903a-64e9-b033545ff34d", "ae79c803-ee6f-0a4a-a704-2169acf8fa19", "1766009b-357b-a7a2-caba-469536b11112", "44f791d7-88c3-3e6f-bfed-cd5cf8fe449c", "b47373ae-a6ec-1ce0-d7d2-6455ed95cf32", "d3aaa6d4-68f0-cbfd-d233-7fceddc5d501", "0e568431-f4bb-2147-ee0d-9dbc0c5e2190", "ffb4c9a7-8960-b769-4f74-ea36e3aab17e", "d538d6ec-7bad-31fa-671e-2d2751aa25b3", "d2a7a915-c79e-1732-fffb-35eeb56d1508", "13961f33-21ce-fbf5-9bf6-1b12e7fa4812", "2fe90709-4da0-9779-4872-f6922df3feb7", "d1660944-29a6-d8c7-f4be-f9584dd34415", "3d5fa6e6-3fa2-f1ce-675a-efcdc416cb69", "0984be7e-baa8-365a-9233-59cf1302a6d5", "69fc1c16-09c9-02e1-89c3-cf307d9e64e9", "0859ac85-7340-42d7-3f3c-f6be2366365d", "ac1aee70-18be-0b74-c2e5-ea78b07fab55", "df670b90-166d-5015-4163-bcf307841601", "df0b06b5-b02b-a510-453d-87f408d533ae", "75f921a4-168d-7a4d-c377-ad96ab8c98c7", "efbfe56d-39c5-b998-7875-14d534970e98", "2cd9b5da-d2dd-fe7b-6690-b0d98adda9db", "87dfab06-4213-f6aa-fb5c-e213e98e9388", "81f361c8-7ae0-1aa9-c966-346236fd6468", "1db6c5d5-2526-791b-dc99-bfd66288d6af", "5002c95c-37ad-65aa-1236-f833838b3e3d", "3bfca838-ea8e-fcb8-b5d3-153f1aa35f37", "61586a99-dba4-a426-b32f-870dae77a135", "02401980-48ce-630c-8584-edba8a2decbc", "28ddb6cf-5272-5a3e-b9f7-affd71d61a49", "fe0fa55a-eaa9-b9ea-5972-8eb68ee1e62d", "4e32e7b9-2870-cbfe-857e-4e6ada1c9694", "fe8e359b-4633-0d88-9324-5d6b33370458", "877dd0ab-3afd-7d9f-fe26-d4b15a2825d0", "aa422ccf-76ec-a2c8-54e1-a4c7bc3137f6", "b6dd1994-9831-23da-057d-06043d90731b", "aca871fd-e8b0-3e22-a0bf-a95484de37c4", "9ca0ef9d-2cb6-c95c-85a5-fd6aa94834a2", "1d379f3a-4286-4da7-1fe2-938139c42cea", "77523afd-1ccc-e3af-d58a-b397b91da784", "8f1a1b7c-5c36-1c29-5491-e4b51c42697a", "c179def4-6ebd-30b9-2a81-aee7bf7b54e2", "b1f12255-1510-05d5-bbb9-98aff579692d", "5af9f45d-79cd-0c80-f837-d9073c2be69d", "30cd1934-3cf7-5c6f-508c-8f9e5d873559", "b82d489e-cd51-da0b-e73b-fb3f496558c8", "97ac14d5-fad8-a31d-424d-0c27d24837f8", "a7a79f3e-cc76-a116-7425-d4842ce1b613", "94ef7223-65c3-14fb-5411-94893524b11d", "536611ff-0494-99b3-45f8-b0447aa2d87c", "b5992a43-ea24-64a2-e1f5-2789a3c2fe79", "b3019c7d-3510-6f85-d082-3e68014adf5c", "38cfd7c0-0a4e-401f-a310-61167fb3b062", "8abf2de3-76a8-6d95-0116-a5ece8213628", "b96b4fbd-3899-4e45-a300-837f69c1654b", "de0f6c57-a7f1-4a80-f861-bb522a337ae1", "1758cdd1-2072-9908-63cc-529de8fa725f", "03db45a4-b465-4670-5211-f5dd9f603656", "60c79d3b-494f-d661-3a3b-46a9ed9069b2", "f78b864b-cc60-71af-2b8d-c1c71f750c29", "3e0188bd-2eaf-7257-3c63-aeac47462c14", "036bcf2d-1b19-4755-1eb9-9c29529c6cea", "f67ad20a-cc32-3c15-82f8-3662d164737c", "da2b4227-4555-ca70-73c1-74c86ce0a332", "44a7ac51-a446-4e28-3ef3-3f17e319cd96", "17babc4a-2832-010c-92e4-af508d3e0d82", "da22e543-0070-4d7c-5388-d96230fe560b", "1093c1df-0274-7cbf-7a78-f2a8c4459648", "1e9a7e14-1674-95e0-9e4e-41e604f3c15b", "6f33214f-9cc5-5b05-c8db-37c3e0b85fae", "40385d7b-3d4c-58fd-5691-69f8baa2592f", "00fddf66-1a7d-9bc4-defa-9d59788f5fee", "a86de9bd-7c1c-c0dc-f48c-2cea6b17016d", "2c10cfc4-75c8-9904-f019-70df0a1eff3b", "7e007fe3-e1cd-a099-ec15-30235995e678", "76b70532-b743-3341-5ac6-58a6f05eaf88", "9ad9550d-587e-f4cf-7e8a-9cc1acdf579f", "cc72aee9-bf72-6492-c789-62fee3ab8164", "97aa9bb2-a3e2-df23-6cf3-b2f22d0f8337", "a0410060-276f-e3c8-a5d0-ca9473acfff4", "d54a3359-6767-cf2b-1edc-c0c42657a891", "72f8488e-150e-1df1-f520-46f05c9bc392", "1d130554-b663-2b72-4874-884585d4413f", "8a7465dc-f6a4-51e9-8085-460b13a9b999", "259559fb-d58f-826f-46bc-19f8f5bd11a8", "0de2d859-b1d2-aa43-c7e6-0bdd1c3a979f", "dc86c970-7aff-a88e-2c90-6f34271a1f89", "b02b904b-0238-7cc2-0f8d-6899aec64488", "b677f609-69e2-8cc1-eafc-2a60bb9412f0", "cc8a1f90-b469-0475-1f2a-110d30ecf70f", "786d5931-4fde-233f-4cdb-b44c505ad541", "79bbd729-4646-8c95-fd73-a8aa409b918c", "9f0d4eb8-37ef-c6c5-7ea8-3d5728bb7640", "3141ad12-ff78-1dd7-b377-365ce1a4d27d", "3d49b1bd-9e1b-6a11-9165-d8fd537f8b85", "bd38838d-33e0-bfc4-6bc8-dca3caa52684", "7f15cc1a-31cc-0f01-96bb-3f6ee84a7294", "f46fb0ce-1e16-784f-9d7b-5bb93150a946", "44a55283-7f96-0152-b737-05ebeb82b86e", "785ad412-bee9-02d3-40c0-87f54d257dea", "9d20301a-cbe1-7451-a8c2-14df67a8d781", "f86f5128-a0dc-a868-287a-d8f43a770fb7", "b4995d82-69d7-e85e-67f9-ce78d72533b9", "b9c44d99-b4d8-607f-c9f6-17f72bc5d928", "43706c77-9f87-b1e6-447c-6d7d0bfaa4e1", "1045905e-85c5-f16b-1c9d-55f174dd888d", "938a3f94-08f6-e0fe-ec0a-c3a6ec9a819a", "73814772-ad6c-d294-9711-98a4cf694f44", "6c55e843-97f7-fd90-02fd-af97b96e8267", "597ab0ed-728a-3d37-5e67-1994c08b280c", "b70c8048-1a44-743b-b215-661b99c2196e", "c51a4ef8-05ef-1c31-649f-694528a5a507", "27815667-7147-1c76-ea25-71156e150115", "b348296c-d374-4d3d-ffb4-b9164191d289", "a7f206fd-64a7-48d9-c65a-b1b4c4019e98", "3cde7cbd-1f9c-1375-df53-aa74676a12a2", "21780984-b34e-56be-2b28-e9068b10323d", "5529c02c-ecc6-949f-4291-ce7b7639a83f", "1c124947-4cd3-306b-0edb-1da33bd623d6", "ccd2839b-d244-3dd5-bce2-b23cbc691683", "bce8bf28-482b-0339-78f6-a001b9fcbe54", "b063804d-ae7c-9578-e1c0-b7a1cca850d4", "82c6a62e-a97e-2e50-9f2e-d8324029ac08", "46d6c43d-db83-2595-9be1-8d7b254b59b6", "e238a710-975c-bee5-aca6-369265536ca9", "38ebeb4f-8a60-2912-c519-15acb6b83c7f", "32a784c1-f466-d5df-fb44-42a9e1af93c5", "0e8d7882-e866-187b-873e-3075599fefa7", "aeafd0cc-c9ad-e0e8-42b4-a822d3ec68f4", "c22126e8-11ea-8608-c61a-4e7565d35bb3", "7f34fc3d-1cb8-5422-7222-d45bf34f12dc", "6d139a5c-00f9-cbb5-6c75-f93881c9c7fc", "62d20d09-bc36-da43-f38b-c1776843c283", "50113b71-b30b-ab78-0b7b-30373ce73005", "d507fc9b-0642-bf07-4572-080088083d6b", "45723b56-9286-2227-fb31-d5226558e45b", "ea46ad85-96a9-ee87-09ca-e0c6aab67f2d", "10407659-f984-fd18-b7cc-4b5c58ccf3f8", "5ff65d82-ff97-68fc-aa89-9f0aeaf2b138", "c7610cfe-e476-d396-8cb0-ec9549a52c09", "a220ea0d-c098-49bc-5ba6-8d92e349ac2a", "6e84c681-58c5-bd5f-6fd3-4e8dbaf81a7c", "f547a824-544d-ecab-142b-c18fc673a3fb", "ab89e5d7-9942-a7a6-c7b7-c60dc7997286", "37b335ef-92d3-e1c0-f234-e7c249b8eb9a", "4950bec3-6946-38b5-7168-94874f22f26d", "a8c81701-62ac-4978-8c03-7b34abed1f1a", "88a4e2c1-cde9-3636-ee40-ad053b7c1860", "cd09dd00-dd2b-3c79-731b-53713ba90be5", "24b1d78c-aa67-12bf-07ae-581f65755a79", "f83465f9-fba1-1f7c-71ab-8538d827862a", "e27b7009-ad70-ffe6-00d5-97bec35fc9d2", "4d0d2643-ad35-36a2-df13-5cd58dabdb21", "48a73004-62d6-96ad-92c7-c2467dbdc689", "c7030d7e-5390-9632-a557-36f7a96747be", "725c7788-e2be-f15a-2b5a-bc49aab09f7c", "a31b3763-8a24-8b7b-4e2c-d1ce819ff913", "5e04ab30-d659-9fab-7b8c-762630040bc9", "b1e17570-9c29-0253-caa3-73838be297cf", "68d56af9-74b6-427a-f6d2-ccab1ed7843e", "98974a4c-f5b7-aff7-a3f5-587ba7507860", "1eebb5b6-8571-a36d-4955-010f3d8efb14", "a2d4a4d0-df07-4e70-0bbf-c58be2455ead", "48656ae7-b459-c54b-8e80-b27040a5c2d3", "995c571a-8055-c4f3-2693-f85f9d80267a", "0e6f8a39-ce3b-e9e0-0db3-2d8a97bfee1b", "8311cde9-f068-2f84-3950-975b6ccbdf6b", "3476ec83-9c51-858b-4c48-956e04f030a6", "921815db-f98d-d99a-3d4e-8c5dcadfc673", "466fe019-40a0-5bf6-18e6-d729b9b9fa41", "21d7a1c3-c74c-8e3a-ef2f-d38e19abe9b2", "f6100b89-e32f-7ea8-b2fd-3ea2b338b0fe", "261aaecc-b593-0600-43df-8aa3488d4c6f", "ebabe7ad-6fc6-46f4-15f5-a008f61aebc6", "6e8d8e27-f789-3e72-866e-9f1231d358a5", "49f47142-6059-205a-c87a-3972d121b39f", "a9876ef0-d99a-e3ce-ca16-fd7d8b10a4fe", "9961edb8-5f0c-e4e0-44b2-df67d7ba05d5", "84a088e8-29e2-f57c-8448-49e21079d314", "ae917e36-f853-745d-aa25-003fa4fbd97f", "2a67b211-d229-d8f7-aa9c-5f55c067d100", "cafe1398-cdaf-f2d3-ccd0-58b51e49f085", "de0fe123-175e-5a3a-6e7a-9b7fba8ecfd0", "4744cf43-82f7-ba53-f46e-ff5c482a7f30", "69497ee9-a647-fe00-ecfe-c21c4de8c6a3", "8e22c3b1-1563-5b5d-1d63-4159b8167658", "3ba344a2-2aab-5170-b21f-68f859b1361c", "7fb55215-f18b-654f-436b-463ded421c7d", "8246d0fb-7306-d5f9-e54c-d2a42aa00d4c", "9ac875ae-1ccc-b105-537c-3e970e0c3677", "51bf49a6-7b45-bb88-1da3-4af446f91442", "eff4d021-1fd1-0651-5e5b-5c3fcce35c23"]
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:08:39.338Z [INFO] internal_plugin.aws-asg: successfully purged Nomad node: node_id=95099deb-446b-4121-585d-1c90c29eb897 nomad_evals=["71eb9ad1-a085-a39a-aa4a-5921c56a606f", "4eb3e5f0-cb3b-37a3-fe28-fb1e4ead272b", "c71765e7-5517-7845-f7e8-67f784d17f55", "d8688347-7e25-39f0-cf6c-65a88bc9d9bc", "5755ed27-40a4-d293-07b1-e6c9782dd23f", "2be99064-dbd0-1422-33e6-94639ca49499", "3c2bcc84-7634-ccdd-e9ba-acdaa63943d6", "edf0a85a-0f0b-9c35-b39f-17c9e82a9803", "b890329a-8728-f8aa-28bb-98fd41d5e3a4", "65a6dbb8-8283-81fa-9e99-fafc5a57fb79", "511b07ad-ff37-f5e4-ddc5-8159e3eb3650", "3db3d0ed-8809-7148-eaf3-5c07b99e29b3", "21bb35a4-cdf9-3733-8a70-6b4d03ed477a", "4585d487-0ef3-1daf-68b8-287966130602", "0d70922f-522e-caca-2a24-ccc489dfcca9", "b5ed2cc9-2b1b-7983-b7ed-9889320cf4f2", "5bee7c2e-1b11-291d-f488-a88d075c71b6", "003ebae7-87b1-28cb-f7ce-f41eeca9cfc3", "35921d4b-2e87-db3c-da34-6f3fbef7c437", "0f892d53-9b82-e5b1-5ac5-89af215e4674", "426cb097-f9ce-9d6e-3db9-0cfd70a545d8", "693ff05f-9959-b016-40a9-77c4f04de8f1", "82409180-f6fb-5ac0-e9f1-f93565a895b2", "7280c1b9-12c6-7896-319a-ac09e5dacafd", "bf817db4-4bec-e8ec-9c93-130f268182bf", "01ccf37d-8a57-f4e2-5c9f-98bbf57af119", "21c2a2d2-4403-74fe-fec0-1b805be442e9", "a1310043-5b57-2f2a-8d85-9717c8311a28", "1c24d9c6-627e-8db9-0071-089d96ac94f7", "573a4658-0f13-4656-e208-af565f0796fc", "1e9f5699-94c0-b09a-09ef-8558567f88b1", "83f8b04b-95cc-e5e7-2d3c-60ae1e978e16", "a93e92b5-0c42-a84b-3e15-cb6e8fd9964a", "efcab778-4261-1f8f-dbbf-0407f0c3d692", "ccb67dd2-4544-465e-7887-76ebcf3480e9", "09af764d-7106-0064-5d1e-d45822046d2e", "6e5d7724-0f7d-4bda-a006-3beedb09761f", "411de831-88d6-0387-bb0d-db79e6d25748", "eb436cdd-2b16-0c98-b521-cb0a799262c4", "4e0b2097-77cc-dde6-e1bc-cc089bd7a329", "74f9d4f5-87fa-278f-0564-34c362e37956", "93a26924-3cbd-c11b-664b-9b1271dde872", "117487f3-582e-a965-2cf7-bfed3decf0ec", "cd8ec60d-5272-c9e6-0f92-4b1b205385cc", "e65f13b6-d0ca-28eb-a070-be2229705b45", "921a99ff-7183-db9b-9b79-49dbd2138d5e", "a43ff54d-3f50-ada5-b586-14515e566b47", "0519028a-228c-2b7a-efd2-e1152bb0c61c", "2614d3c4-b456-4119-9516-052b7443489e", "a9912753-3dbd-47b5-97e9-5d43d04c3ac1", "48ee8094-ad9a-0674-d118-83dfef57bb68", "1dcb4f8c-4160-71b1-5d09-c569c42fdaca", "22efbc4b-1e00-5cad-45d4-c63334028950", "6079157d-315c-c74e-bf3b-14f113611c10", "4a180e43-e422-2920-ea80-14f66719f75e", "cb9e74b7-a0f7-b104-201b-91f93bdb2e36", "1551f4a1-7964-432e-e18f-8f26bbc370b1", "a8ad8c4c-8c24-84c7-603d-2b6c98f839a7", "bd451eee-64ec-43cd-0862-2bcb723dd57c", "7b2c166d-4f47-203e-5ade-ec2ab5c89d88", "d0d267e5-2c80-fa18-9816-e42bdd696fca", "5afd53c9-b712-88d9-2a1c-d8dc3a98e56b", "38f5545f-3f0c-a634-faff-595f676aa3b4", "bdcdf1cd-5ac5-f7d0-4d05-c72df0d403b9", "a43cc3ab-1719-dace-c7c2-05e8d070988d", "7861f7e5-ba96-094b-c4dd-efc4b71a2244", "3e7ac96b-fa75-1bcd-6efa-9c29b742b4d1", "de6ca036-3450-121d-364e-a311d0048275", "48259f05-5f3a-0996-f1cd-dc412a2bb196", "93a18608-bba5-9cc4-c969-3802b626b8d9", "e6aa757c-2b58-2195-3900-cd19aafe7cc1", "855b41a7-9575-f542-bb2c-618ffd9b3087", "e3bf7383-8c86-7b33-c4f4-af96f5fdafe9", "5347a84e-14e8-eb44-e1be-3dcd2fab664a", "14c0fabb-df33-5543-f36a-ed8ad162775a", "effe8af4-4c3d-24bd-54c3-5b50dbe69058", "3143573f-c9f7-cf6f-c247-2d01d61cf778", "5c8ab74a-f98c-00dd-c365-0553a3b9b89b", "face08dd-5613-b637-d021-5326aa0f572a", "64f0a601-de80-2419-fd4f-9e28f930d7c8", "0d8493d5-5eae-2dee-35a3-6617cbfbe3a2", "189d6e6c-fd55-7011-dbb5-105f786c8668", "15ba7fbd-c435-25b8-cf54-b71ff29dc1e9", "e5abf57f-0fa6-a7f7-be8b-cc4955ab4b43", "60c3f770-36bc-481a-44e2-93d4f1dc53bc", "4e8acba7-0802-5efe-121e-0736d849f26b", "cd7a6c93-279d-079e-98eb-8b8b5f701654", "082bb3e1-0cbb-2ef4-34e1-06c3a95d84b0", "173d0229-513b-1a91-f213-bd4d37965484", "af34f395-4a26-4b2a-1d7f-d07bb9b194dd", "9154aa58-03c6-fd72-7ba5-f9e263c16fcc", "634ca19b-a80c-ec30-a959-663397191e09", "2ad816ef-0cf9-6612-9225-1a2578cdc5c4", "f807691d-cbf1-98ae-9213-2b32ad28dffc", "e059da93-d4dc-2e36-fe73-542b79774335", "04feb04c-2fd1-5f11-b594-98ce0b3858d2", "b7ec8c80-9514-2100-7df6-611729f436cd", "4c73f030-1ec6-446b-e792-e9932bc7a8ef", "78e432cc-a1cd-77d8-1d21-a4d8767c7c4a", "3d713546-f425-927e-35e3-92db214d331d", "286e96bc-c871-4d1e-094f-62edb26592c7", "11c766fa-ea27-e6af-f324-adb0a262533b", "acd737d4-325f-83e8-8cf9-f0cc0b33ef9a", "530917c5-6959-b039-3a46-e5d17fadb2c8", "4d3eabd4-20ec-2f89-a43b-ca13c675eedd", "a6a81114-d709-e598-389e-b141b009ff81", "26b7d4eb-340a-6704-4304-ee138ba5c8ff", "93854ccb-5bfe-7412-493d-e9f8efe1e215", "15598a6c-e3e0-77d4-0de4-98ee3db544c5", "9f3bc35d-a2dd-7159-6d37-10212ec2a131", "2fd0c160-fb23-c484-49d4-141485de87c2", "539bd5f9-0768-baae-728c-dd76ae75e0fd", "191922b7-2a63-7683-c699-89dec3b267bc", "6c77047c-fa85-f5de-85f3-ed1c82a79d4e", "437c9857-6bb3-8192-9795-c8be8317ec71", "198249f7-a5ae-f1e7-a981-c2ef3581f68b", "c093be9b-a67a-6167-96b8-f93d62067e37", "8b2e5774-47f5-011b-b1d3-3cd28973811d", "d533af1b-841e-45ea-7533-2f805c2dae5b", "753485e2-5939-0e2c-a789-6b91eef2b204", "a10bcf7a-479b-a499-7196-155bc404d75e", "2ea55f8e-3482-d4f0-3a9d-ed30c0d56b58", "a5150c4b-9054-3b30-4fcc-6ad71fa0cfd1", "b03356ad-ef18-433a-da2d-f3f8f32bafe3", "c6d8fc46-a380-9e1a-27e0-e9057d070521", "7d6185b6-7c59-4265-d308-f2c88d5c421c", "198f5074-72b7-c339-e5d8-43f4f8caa8b4", "053f8d5a-cfda-3881-619c-7ab6864f5e94", "c608cc61-93ce-2b1c-86bb-7018206765b4", "32f1c34c-5f30-e817-5846-7c5da9a2ad26", "67628a63-fd8e-14dc-8bd9-03b87f8d9f32", "7ff517af-b76a-a585-f48a-033bebf29e2d", "8ee7520c-3054-0054-ac31-3b093f233509", "1b176162-a47b-13fd-9667-49640e03eb0b", "62e61443-c815-16be-b452-a0aca65bc2b9", "e5420d52-9366-a3ce-4a92-825034deaf71", "0b1aed5f-cf38-6445-8ae1-1931e3abac05", "22374623-be95-9f73-53fb-6b858a9be497", "ca653216-832a-6060-a929-e434497bcf84", "1ea9452f-4063-a6fc-6249-14d8f752ed70", "1b29c04e-62db-13e3-bf59-b4afae67ee3f", "bdd40e76-c005-8f0a-8484-6254acb8433e", "2b5ad1b8-75ff-fe12-69dd-bd0d297bd2cf", "5209bea4-57e0-c6b0-b080-5fa75bedf6ce", "18e5188d-9fd1-0593-f690-432162d4dc36", "7b5f8619-22e3-ba83-9dc5-07369fd35c59", "dd37d7be-04e5-3848-aed4-3bf1d6dc9996", "7e7e00fc-a7ad-72ef-e001-efd819b44cc1", "c155cac5-7b6f-2304-7082-1e7db431c932", "d229f99f-d309-2ebc-70be-a8ee4427f657", "957b4c28-21d8-249e-1039-72154a772c78", "c089bd26-6a16-32e1-fda1-8d40c1a64c76", "5c0778ee-1fca-8e4f-860d-a0f221bfe7c4", "db52e794-8a3c-7fa0-6b41-ff4ce2306605", "9985836f-0882-2a1b-3e75-a58c794c9837", "a1e4fd10-8272-381e-e589-c2813cd09f9b", "1f7f3f42-d5e3-3023-2fd8-963154cd4d0d", "da19af24-18ef-b509-7997-9ad635f71418", "f8e891d5-d6f5-116a-0a75-efd4078170ca", "b23f21a5-2ebe-df79-35b4-c6ceec07a76d", "ddd37402-9f40-7a38-52b0-ab3ebb896b65", "a46705ad-bd73-bf4a-4657-e49112776277", "f7168a1f-ea57-3a75-c6fc-79fea7535341", "674888c8-5e3a-e4a3-3a05-5ec5ab1d5bb3", "6f5b7f62-d413-71f9-59c6-1517d719535d", "5feaf479-2ca7-ad48-bc6e-dbe56485c0a5", "8d9a339a-46e2-7415-24e2-fd88f0c35960", "028cf3f0-946f-4c68-ac17-ba310a2c323e", "3dff8cdb-b897-808d-9163-3b10620b6079", "0ad201b6-d370-94fb-17bc-8caf912d8773", "eb253142-a1b7-b97f-8bbb-26ed9f98ae45", "f9ca059b-9d4c-2554-ed31-1719b78bffe8", "a3496081-e314-29ab-965c-918f80338633", "a20e73dc-a010-9e5e-1d9c-2197b2cc14cc", "e0f1c0c3-ae2a-2ebe-cd9d-b79d51699e25", "b015d16a-8fbe-bf6e-44a7-245d60ab6abb", "439b878b-8dc5-f240-75d8-0d51bd8052b9", "c216282f-be89-e9dd-7d7d-9e24efe411ca", "9390008a-2901-7978-08fc-1b8bf2e1c1c5", "cf1b09d7-e286-04e0-beeb-c08afe4caedc", "9812098d-d78b-f512-9694-3d65df9328b9", "75e44768-fbfa-c132-a8bf-b783761802a7", "63ee99d9-bb99-5e2f-b319-e42066eecf03", "2b30f38c-2f08-5bbb-d35b-e443027d5b87", "b01ccf56-71b6-6924-fb17-ad523c7c1481", "a84f4a1c-1fb4-f45d-a80f-b4fd55c8f216", "051bf9dc-d883-e9c6-d3ff-ff2a1345aa33", "fa181578-a768-19fa-6194-212728393af3", "799587dc-a8ba-ebd8-e8a4-fcb011fc4b0e", "8160b578-6419-3096-edcd-73a0a44364f9", "846c07f2-6bf5-d326-8339-d8a9fb45cd7a", "816d1081-7067-9ae8-7873-646f07deca34", "d043e6d7-8920-8699-fa92-8615b4d9cfad", "9f0bb6a0-d520-0969-4ff6-61197e62674c", "3e565b6d-c309-e5af-c020-88dd05d518b5", "c98f4827-9515-2692-68d8-d079faec3148", "9973166e-6238-2f2e-b4ff-892fac056675", "fe46e429-124f-0f44-6580-d8bfe1eabb89", "9ed2cfa7-40f2-5bb9-cd99-77608a4dc6d4", "f4a008bb-c8d7-6154-5530-e25b5fc38cde", "dd296e24-a045-5915-c171-f1b2a7bde2de", "be478abf-b051-392f-6038-60661c595afa", "6e94db38-4d1e-96ac-5633-d964685768d0", "0dc3beec-6ded-ef70-67c7-bb0dc8f9bd8a", "d270a5c4-fef5-74fe-312a-010fec6e7e74", "f4c5266b-abf3-49d2-0769-710e55ef966b", "cc6d093d-152d-692e-e743-1baa100c386d", "394e7dfa-8780-4703-f8f2-bf0165efb3e6", "0b4d114f-5e60-1a45-5e41-075ea9ee277e", "d6c44ccb-1a80-30c5-c3f6-c48709f320f5", "e8dc159d-5486-1088-6533-372f5f1a5ed4", "30f811a4-2a23-c5e2-ac53-29d3e922b538", "15967247-c88f-2058-ccb6-597bedc9a258", "d0f66796-ca1e-5e38-e358-4808ae2c5f1f", "389d0117-2b06-3dce-c592-245b87e96aa0", "c5dfb652-1cc3-0592-a1fc-9e28856121e9", "0bb498ff-dfca-60bc-17a4-1c427e371641", "ab78cbff-e34a-e169-2421-b27a55b55c83", "1b50ffdf-b204-b728-62b0-0f134a68217b", "a814ee99-8754-0f52-c2b2-c0de3e052372", "ff5924f4-18ee-a255-4a4f-ee16c55be3b0", "4a45a9fa-6250-0d39-1e9d-426ed1b16c44", "7fd2c10c-8155-ccbb-aece-f3700841ebd6", "5188b8e9-32eb-8ea2-bcdc-39498868f25b", "55864226-423c-fafc-dbf4-191c65751f4f", "0361f097-c291-0bce-9366-33e98eaa1d21", "bd934996-aefb-62cb-793e-18e753750590", "11814741-a019-1d67-e4af-c46643adc321", "95445567-df58-a28f-19b7-be6b49433451", "60f41eae-a0a1-a5c9-845b-c4215fad7572", "92750c09-f6c9-685c-a7de-8eee727927c1", "25533870-8d1e-9fb1-848a-9dcc55fa8e95", "a9ddc506-8be1-5aa7-f00b-8af0fa33e4db", "ab7c63f2-8313-be44-d3c6-4cacbe77e495", "961ae91e-aa51-e43c-f735-6750d5e12a50", "24b7bad3-45d2-4c2f-40b8-61da28487404", "e47508d9-8993-119c-2293-3a3f6c21475b", "cf8197bc-1f6f-0403-5a43-e0360f880703", "8f03f45e-d97f-3e4c-f51e-67c124622675", "00982c4f-be01-f199-67b3-8af487daaf4b", "45f4b79a-789e-64e4-38e6-625af3fe4268", "1641f12a-5586-3252-325a-3a5f0a61ede9", "d59c89db-e903-4be6-0f8f-6ae5b0f3bbe0", "ff4932c3-751a-671a-5643-ba06df051f5f", "faa20713-3399-540c-1235-286b7a85e8b1", "ad749296-23fe-977d-5faa-972c45d5ca83", "4f79b705-a73c-3a43-8ceb-ccbe6b751092", "74779533-37f0-6984-8856-2486ce8c1543", "a1ea2b06-5317-bf1f-69ed-f065aaadff86", "edc44c39-56ea-ad84-b8e9-f9d6de32d0af", "14a70550-5449-49eb-bc68-a762389529d6", "88609760-ab2a-77de-ceb7-db9520343a06", "3726e7cb-2fa8-68ab-28a4-623177454fc5", "50192b78-b2e9-6984-3ee7-28a890eacc25", "8bc0194d-71af-c39e-eac1-0f873cf1949a", "eb3cb06f-2165-cc5b-ffc2-7a5ee6c6b9c7", "b2515f80-8031-c869-044e-a144ed35fac5", "fb11c50a-af3f-34a7-a54c-bf95550f18fd", "ecc5dae7-81e5-c97a-f41c-3b6df0cdfff2", "f89e6a22-5f9b-9999-34f0-c3ea5f046f1e", "e4c6a05e-2097-5973-a649-25384fbf37a4", "6f907b13-2d96-f522-51f7-ed3a08048d6e", "757d74df-08a9-2457-d17c-860b8e08d5a1", "ab70e492-5617-fbc6-57fa-61b234396bae", "a087a3c8-ad18-63aa-16f7-af07cc5767bd", "b82a7d13-92f2-f5bd-6da1-fe3aff1b94ff", "3d5add2e-0fc5-9a16-c661-13f0c077edda", "1073e65b-8775-5f1b-9f6a-0b3c328f96c8", "b035d3bb-bf60-9f05-ca5c-1ea425bb973d", "5813e186-812c-af2a-c2c1-e5da1bf55f9a", "32b9c2da-18ed-0682-d780-fbe9f3fba406", "dc8d4915-1e18-6d4e-ab05-9e8ec9df0503", "310f6088-3808-56c7-03ef-eb3a312e5d25", "e626d64a-cc60-08b6-f623-699257570a4c", "cae60977-7209-1167-2f8c-270bebf4b2bb", "2d25251f-aa2f-5093-08bd-e6aa2ae14b59", "85dcc633-0d78-b0c0-0a27-95e2b95bc9d5", "75df570c-2c6c-e1f2-c6b1-67bf1f117b33", "7b78ef1f-3dc7-f020-4632-55266cf319ed", "5fa8c24c-adcc-d947-790f-d59828ca150d", "f36d5e75-eb3d-5909-05e6-677f1910d8d6", "b3c7d05f-4a54-9014-1d9f-61eab25d437d", "9b186237-6a89-92cc-a9d6-2b69e0ea2de6", "514d84af-ed66-4299-7032-e35c74636a2b", "382cee1c-cf73-f91c-7cdf-72f629e6ca67", "e55de870-2a92-8859-5220-2fcdf75d0bd0", "6a6dd05b-e888-ece4-929e-ebc09cc3bd23", "ac260ff6-4c45-a939-a19a-9c741e150154", "f775cf2d-20ba-f7ba-b5b5-e75b5fc370dd", "9aa81b80-f2a6-cfe4-342b-273b40a35e33", "7cd22d75-c4ee-68b8-0372-70ece08ac7c1", "6a5718fb-cee1-5d9b-7c19-22b127cd7dd6", "ce0d5aa7-4f44-e4c8-3e49-1990cfc300ef", "0dcfac94-6e83-5df5-d04d-97d653e56680", "4be31369-ffc2-083a-07ff-9f154d146cf6", "5d85cf1f-a58b-71c2-5331-25988e9b8272", "35931b1c-2baf-6b75-6499-bad0df862de2", "2a9f8005-9a08-e6a7-7963-0a6d89efaddd", "68d42c4c-e168-43de-0df4-05cb05f0033a", "017f007f-6dfb-d1bd-c110-0b7f35ad4e3a", "49a52825-026f-a504-d8c2-a7c9111b185a", "81417f46-371e-e59b-2703-ab846a926a3a", "600872e2-8e2b-075b-f365-4638973e6fc4", "d238aa10-9cb0-a880-5af7-d186d3af2e4a", "9a2c05d7-54e1-2593-ab2b-eeeb38535122", "f8b09baa-828c-64fe-4e6a-e4d484870f47", "5e93a5c5-7e09-73c9-8b65-52f742b98cad", "2e3caa42-f4f2-88b0-6ddd-0c04574191ac", "d3d1b278-0c19-5668-f36b-3c70b5abe358", "9044643c-249e-1899-d818-2c8f830efcc1", "741fd825-f2ff-3bc9-4bb2-5c3c99d4d5f2", "0484a558-5713-3eb3-c233-fa4d5ef52072", "8fa770f2-2b1e-d7fe-7972-29190fb12018", "6ef677e2-8dcb-dd52-1368-4c42503f7c1e", "f72b2c22-9e5c-5c29-5f2b-2cb055593376", "b80a53db-378f-d7f8-b9db-d14f50150ffc", "8e4d4ca4-b680-3a00-b844-653b7d321ac9", "862eba39-bf5a-37d2-9fed-bb9429089b21", "6124c3b5-a66a-a60e-0d6b-f3f942f12533", "d4ff71fa-bcd2-7c09-933a-b5a0b3998deb", "cd924047-52f3-32c7-4eb4-b2ce01e448a8", "7f573f96-12ec-970d-151a-5d78972542a2", "9e89b701-9221-6295-ef28-594c56ffbe0a", "cf129922-a519-acde-72db-32aa28762fd2", "bac7fb94-4466-b759-744c-b5d04837460f", "701882ce-3084-19ec-621e-8dd207fd629e", "117ed84c-7f0c-a690-d2ae-df38725b316a", "125d2674-aa02-d18e-3d85-e9a1777ed5bb", "947a1a1e-008b-e756-0c43-8736167ff85b", "ac501ea8-ff8a-d5c0-48f5-dd0b33cd1730", "f32833b8-eef7-d53b-f9fb-7faf8c401c3b", "27f20a6d-87bb-1828-3a97-b7d40b0e092d", "14cfd2a5-a3bb-bdc4-8e1b-ee5063aacba1", "ba138a17-5beb-ff96-2178-c86231c9e334", "5cb729ff-f6ce-a66b-d73f-681bd19e1d42", "034999fb-1a67-5c64-8a46-9b1d6e8ff387", "505df1ae-cec8-3457-2bfb-9870e6a00b03", "90f7483f-34f5-8b35-7a51-06674dfb089f", "07e8d28b-5cec-a7f2-e814-d7357af32ac2", "174b1dae-768c-3cb1-abc5-f8c3d0755fa4", "5cbd5eb0-d8a9-4740-b6fa-f3af695143d4", "d5c36752-5778-da82-1a7b-457e4471d0ba", "d40dae9e-c994-6e88-61c6-f39a3d96bbe7", "55a59bd2-9b8b-6f83-6a8c-4eb002c2ef73", "578063c9-ec30-f07f-8518-1c9014d4dc53", "fabeb6d2-5532-b18f-b4bb-b5d9e2a94ecd", "4f989d49-db4e-ad1e-33ce-f73b5ac3be71", "53f91f39-f770-e713-6f91-2922bad30c38", "da39500e-af0b-f10a-f1c2-4de3b402e738", "2adcd87e-b56b-3815-d305-e97a30732f4e", "7bc12ee2-973b-123c-b87f-a95d5215f9ae", "e57e3f57-cbb5-4823-936c-5ff60faaa483", "b316ef26-c2c2-a7ab-eebf-078e2c2f12b7", "a0de9062-fcb1-7334-b534-f17ae6ecbed4", "417679e9-ae1c-cb50-a51e-4b8eb6b3aca3", "24d7abe0-5062-9733-f136-c3ae1a0a4c58", "b1533f46-e007-094f-b111-441456c9960a", "6bfce515-9ff0-1ab4-e216-e315a79785fd", "84d211be-0c73-36c2-053e-cf70b70bde38", "15273004-8ccd-bac3-814a-54ad9bdce328", "12af81a7-3e09-b46a-6e81-79d12fd9b8b1", "f11e1eba-0580-feb1-3010-982755935bb0", "65d22ecb-cb9a-0b08-8f55-cdc687b73fdc", "1f132f07-df92-3d80-ae91-7e50915ef97a", "0e6159f6-2a6c-1e5b-388a-3212d4148a6f", "e2036448-bbec-43bf-e703-9a57b4df6608", "2a387dfa-0b6b-845f-4d73-d9850093f349", "9225bba1-ff88-3b45-170c-856a87210a54", "70cd1c33-f115-1b4a-528e-c27a3100a7c9", "1ae5cfe2-c49f-ef8b-b7fe-59a28bf06383", "73cd9949-6ba1-c7b3-4c81-bc84b382cc19", "b31ab588-599f-5c2b-2f5f-47b5075ae9da", "f2d917d8-0c69-fcd5-7f12-774c0c8791a7", "01030b21-cfe0-7cdf-d285-08fc40f64bfd", "03e676bc-048b-5a63-8869-7c79e2edb38f", "02a50b62-b27b-2788-4126-82f7431776d0", "e0b1a2ce-d5d6-98a9-9590-857f5e16eae5", "96eb14bf-1e9b-2b47-c5b8-bdcd9841c080", "8002362b-2549-ba03-ad53-419660fe2283", "5950bd44-7d0d-24b4-b5bc-f8c7c10ff448", "f4ed094c-74f8-9402-b36a-444e11cd169d", "0e111390-a101-19af-a050-07c8671d3a9a", "2e45e135-f8af-5325-9a04-cc354af7e65c", "54c5e05d-9421-f031-2bce-27ef98114579", "29f0306a-c71b-9d75-3acd-4ca3cedd0ef3", "fedb58ef-0843-a586-4beb-8d409d67f104", "d88a1d58-b06a-a1bf-076d-68b5a1483bea", "3901b0f8-ce54-235a-6d71-c84b1db279b3", "a69a9d3c-75b5-ce41-3d4f-1e73b92a7094", "850fd66c-c511-5530-667e-ac59a0ee7e35", "34ed97ad-5a6f-4173-52cd-bf49e566d58f", "e49d1454-908e-8e18-d624-e2e276f5403f", "f12274a1-b3db-5df1-068f-bc0abef4c866", "adea7842-0959-bb16-249f-0369178cbeb1", "0db9117d-65b0-66f0-5662-4fbe8f8eb430", "115f6a2c-d749-5d88-41aa-3d5fbf57a051", "54d4abb4-8603-d51c-569d-1f56c8e24794", "0ef5dd26-d184-6bb9-1555-3c9ec699bf2d", "936e9da3-7f0f-acfa-dc61-0e88e486be3e", "ac5bbc66-5681-ae15-3afb-04cbb2a9e0cb", "54fcbace-7b0f-ffa3-7f84-db56f9e27eb7", "6ee15649-f804-e134-9674-d23a61f4462d", "ee864623-ed0d-d0c9-b56a-7a95a098aa37", "ed4b75ca-a19f-06e7-7a3a-1b1885b18f21", "d824967b-b447-218f-4bc2-3dc399b63665", "5b38ca73-fb13-0664-83aa-4391c9c5d4db", "4027e322-79f4-8d71-72d1-16cc7d84fac2", "dd081b47-3da7-2825-f586-4db863e51be7", "b9d0752e-b76a-675a-f5b7-4cbdee89a438", "8ba70f2b-3898-3e6b-fe20-60c4e2106979", "5e05f514-2ddc-916a-cfef-88f82a9a3c9a", "d6705c03-cb15-2934-399e-88dbf04dcef1", "6759f14d-fed2-7f8a-2a71-1b73a30dce42", "e0d22bf3-92ea-f94c-9b98-d8f0cc248e8d", "0a0a6ff3-20d7-05b3-1e13-135e7a29a313", "01fe2b12-54e9-b35d-2ef1-071ce2bf3487", "dec61f1c-b26d-6250-3cc8-7ad7c9aaaa38", "c2922f9d-e854-f66f-02e6-c1577815dcc5", "868c8562-c396-1937-d998-5ebb4ea8cede", "571eaae9-a3a1-e669-b7e6-01c0dcbdd09a", "69916115-75f3-7a21-2b3b-e09d69e75066", "f8646445-ff0b-94cb-7107-db44cf0bdade", "8214829d-6018-0d81-7fc2-8e9e186c5852", "c0447923-5001-7870-16c6-af8a8a15a8f5", "5f16ad61-447d-3a0c-c314-a8677b625555", "0dd67217-b24b-cb31-00aa-9d993301cdb0", "f09b3ad8-4d79-77e6-a3f3-99028d4dde10", "81bea5b7-e9b8-4564-a2c3-bbd28a10299c", "d6990467-1f89-d6dc-f5a4-1677431de16f", "2a85babf-97fd-13bc-1b79-48ca8092dc13", "738ed4df-00a1-621c-6931-8269c265c3de", "2290cb8b-8453-4712-1cbe-233fdf7fe0c0", "6e1176f2-68e1-6d98-fa65-997f417f4584", "d4039175-b584-6bcb-f628-f91a864376db", "11bf2b06-f248-0f32-a256-aefb246b00bf", "9b73cfa8-406c-7943-f808-8ccfc626bc4b", "610ac589-3e81-e341-44bf-319031d66019", "dd48d368-874a-2aa4-64e6-886e6c227c16", "989dfd52-a22c-ccbe-b2ff-e4eae945a36b", "5098168a-ed78-d149-d060-c41c5d037ccb", "dee19e54-2f6f-d6fd-f2fa-9f22956c1bec", "aa6c6cfe-d7bf-3b84-2dae-99188e638406", "02f9b474-354a-b8a1-7f67-4045a974d1b8", "6a89d646-2c8d-a77b-47d8-afe39fa1e3bb", "4d65ed26-c49e-3eba-4542-9350628ef4e0", "14196ddb-b8e1-135c-5933-122d08f22839", "00a16b07-1910-63a2-2ffc-d16a01a7435f", "f17c6bd4-2d75-6430-c2e0-22cf5b372551", "c07ce666-7549-9101-09f9-ac3a2876524b", "3b54e7f1-db95-7a5c-5ef4-518e9d730fcc", "8a123e2a-ae44-93f5-46c1-11a0ef87f7fd", "2f0756e3-a8a8-c1a3-afc3-dcff41e63477", "bc161375-791f-cd89-0a14-dab56f27f65a", "d5b1ba0d-845e-7b30-268a-d739e79fbf22", "8628804e-1dd5-2056-00b5-74df47d54874", "a7538135-c73f-ec81-31ea-5a46a9f6e146", "232f9383-597c-0633-7ef0-8e80d0950b56", "d7b53f78-d4c3-bdeb-ad70-cb5f84f72d3b", "9ce96ee0-6e8d-2148-c655-b7a16d4cb13d", "34f460a0-212a-8081-7ff9-b8776ade5f4b", "46b5de60-54e6-4608-6476-61f86f943d12", "a081776f-c78f-6016-db8c-58a04bfd2794", "18756314-3b0c-c299-7e11-ae9373097fd8", "0063ee76-482c-b7d2-0785-b809cca1de9d", "b2758410-f8d3-4e0f-c01d-f4f6460be26e", "8f90a246-a09d-d5ce-3c25-7ba89ed76d96", "9e4b4038-1d58-419e-5446-851da21f4351", "1b5ddc57-3713-a264-c71f-a448b74cc9f7", "49bf48e7-e90a-d37f-9928-9969bb38fcff", "5eba5915-56b0-e155-2d74-40f18715db83", "bafef14d-d234-483e-d9f5-e70b323538be", "e3f7fc00-318b-ec0d-0f83-8d8c8fee6583", "392cb006-a8a0-92fc-8d6f-08e503b38a13", "da57c38b-0b76-0652-2047-8c9f75396b5b", "651a183a-3332-f0b5-e868-c2cb54a0b9b8", "0ee00369-02d1-db5d-f555-803c5229fecd", "3eacfc50-057a-d947-fb6d-8f7d7d9fd2ce", "697be91a-300f-5c1e-9f1e-16f2e17e1c24", "9a49fdfa-ff6c-d304-ea55-00be7cd4b6e4", "79041ce9-1c50-2b38-f626-3d1e791381bc", "3c9be6a6-9f65-6d1c-26ee-89f4ebba8353", "c561da67-1c0e-55e0-fa91-f023dd946648", "5722b0e8-4ab9-42e1-081a-ca220f1b4417", "e5e264b5-190b-189e-19be-9bb1159f1af5", "f0d2b54a-07cc-6af0-de8f-9b453f0ade11", "a77ca7b5-3e8a-c7f1-52a9-c269ca07f588", "66c6d7d7-587b-fdf8-f22e-f3001a1276c4", "f6892ad4-ac41-36c2-fb3a-5474fd4aa067", "0178d63b-d04f-e7d4-f16e-0ec874b8cc24", "f62a932a-d7d2-305e-2f0a-d23a0a534974", "16ba2095-d7f3-db91-81d7-c84d77b72cbb", "7b1a4764-1a08-b13a-e35e-0610e898f01c", "d9b2395b-c929-e1fb-f3fb-14e2b58115cd", "af715659-cf66-0fec-f663-c7b484a92e90", "b69690f0-d826-a0c4-e176-1331c677fa71", "ff2573ad-9301-2f34-4501-0a3704d4a722", "c3a5ea19-a885-e129-a39e-28355a670331", "ae998be3-418f-5fab-ca6d-bd674b1a8712", "39ab3190-1bd2-0427-f8f3-0744a9730370", "77f41673-8cc1-35ea-d6c1-0bc65449e2fb", "b24ca0c2-df07-e414-5a1e-0f14e0036527", "ad0b97fe-5aec-e48f-1a5c-4262952d9087", "acfc0553-f937-141c-a3f2-f3c06e7a209b", "87b20ed1-3cfc-dca8-769b-8a44f03774ab", "23e0ef41-451c-cb29-ff64-b24ebd997cd9", "232ec7d7-f231-b819-42a1-15d9b87cf84d", "12f26d94-ef5e-6f68-f557-11fea017fc0b", "6e385fe9-58dc-4872-ab91-d2e4802a29b3", "1571d7a6-dfb8-7dc1-141c-17681ec137c6", "f3d60ea0-1875-4c7e-d000-29693f9b40c0", "6167919c-5901-612d-cdca-26ef13149d38", "af19400d-4fe3-2b42-c328-7b87a57ca282", "f83fdf6a-ccba-0fe3-de51-9825a42e1658", "a9e57cf0-a076-c454-fe5f-ce3c03de8b34", "230426db-0a75-8d22-ddd6-304b861c7585", "5d1ea1d4-67eb-d180-0c4f-f40d03854197", "f52e44d7-24eb-b0fb-67d0-f7d6c18b3074", "7f3059ef-9b9e-2bd4-12a8-66bf1bb4cf80", "ec24c01e-40ef-bc58-2520-bfd123ea7a5a", "48b2733d-20f3-aa9e-0a45-b8d3d4ad9560", "d8bad743-0df4-77a5-3c68-432beb60ae68", "9057746d-6ef7-25ae-25f5-a0358fb03761", "40f192f6-8c10-6ea8-21c4-6b5424cfc48c", "f8d2ed2f-06ed-8228-de96-a4ff7947bf9d", "cb164d28-58f4-0cfa-c78f-2de513650979", "7efd1269-b4a0-269c-b518-6bc77281d6a2", "0ed5d61d-8f47-75ee-19dc-767975aa6183", "59b0c128-2eb6-f143-dd31-cf2814c69ba8", "b85471f4-1258-e371-7623-c0343f569871", "e5153d67-4b41-22a2-958e-866c1c9d1256", "b021075a-f153-5f5a-2ced-987fbaa62d29", "2f3ccaea-e8c0-88c9-29ca-6bce3d415fe6", "9ce2f355-6308-2175-1983-a0457ee6908a", "2b4c3e72-33c7-39ea-d4de-578f4a182257", "7440292a-48e6-c516-7716-cef11f0bd882", "066a1e81-ee93-6e64-71ae-329f2c0756a1", "7c51f519-00a8-739e-c839-7c610659c46f", "991e3b0f-92de-55a7-c346-150ee73e7dfe", "5e1cdd08-e486-484c-5432-f7373cedd2e1", "4b88d393-d5b4-1d8b-e5d2-dbed133dde03", "be1b2cff-7d12-d988-9162-418d0ab87848", "4934f548-f5c1-401c-2117-5b0432522c45", "091621b5-f674-0497-1e88-b305fb7ed3a1", "456557cd-4ea0-74a6-5d60-bc70c88fdd0a", "7d37bbfd-2c14-cc5c-928a-708c2a1e7e2c", "411b5ac8-37e0-ad22-2c19-7530c8880ed1", "ef0c5e92-beff-d2f1-6973-0324007aa789", "664c8987-7aa9-b6a5-7f3e-467384c246bc", "22e6c62f-8ffd-bb58-83bd-e9ab4dacd6f6", "4525b7e4-64e9-eaef-45b8-59f5d9425ad0", "2f0f13ff-c50e-11aa-4093-a67b7f661b6a", "ebea82bd-8109-c26a-4063-b85b516260e4", "951f497f-f90d-27b2-bc7e-42e22194b699", "d161ff93-6284-7fe5-d6e9-859c9660949b", "6cb57847-ecf5-9db6-338f-76a75144d577", "2d2d25c1-5a0c-db75-ecb2-d0e50725faad", "64aa0e26-686f-91e5-ed11-cc9371975dfb", "c20eca66-adda-ba36-66d9-d14cebd9440a", "ffb35064-1751-d7af-0c87-6bcda1a2fbc1", "d7db4931-0a40-9873-1285-a227fd8852d0", "bdcae4e3-4daf-4006-5681-79576eae378c", "fbf53006-f34e-58f2-36a7-0e119dcab33c", "e65eb0d7-c28f-212e-ed56-1af4c7c9cfef", "b840c34c-241a-a45d-5d75-b0205aa4fa81", "ce78f61a-40bb-8c86-b58e-11e3bfe9ea6b", "ae008f33-d0dd-9eec-77c9-62790b0cab29", "dae696d5-96c7-de26-8589-d6816cb4e193", "f2784ee6-cf5f-eb48-6d89-afa46f3bd24b", "ad7fc1ad-d6e0-65e0-459d-eff6358fea24", "d7ba7c4b-58af-af9d-e289-db2424698743", "252ec9af-7f1a-1cf7-e323-4ea8560028ad", "1a28da11-8dd3-7359-7eca-a434203a91ea", "dfcfebae-e744-af38-36df-d9b85d1eea9c", "0ebf460e-e121-a2d7-6c96-40a644698fb1", "54480946-019f-4887-9a7f-db80284ab657", "ce7146f5-a0b3-9f4f-d892-38ebafbe8ac8", "a8eb5716-a5e5-4e4e-5b82-7a6369639f53", "8c714214-e5a4-460f-fb0d-ef3a16766e60", "3a598c89-ddc6-7315-c3d4-93f77b96c8c1", "95eebb70-e425-2575-e33e-bfeaa1db112e", "b9221b54-787f-bc9d-bc3d-34c5e7b851c1", "e7b7704b-80dd-3824-f882-97e8ab832c8c", "769557c8-2db0-5c38-97bd-a876cb35729d", "dbf251bd-b74b-53ef-878d-3a69b56ec948", "1a72bca9-2a89-f295-e1a0-9efac17bb041", "d585e062-ac00-2071-44d3-ea48f022237f", "ce4bc116-aa3c-7336-3c9c-49f0ea1f1951", "d6e344a5-8381-61d2-8b2b-54378ea503ea", "38133557-2a3b-4f63-842c-d386e132e8af", "e11b7068-3c18-d534-9d3f-49ef64d4a97e", "cc41cd8f-7143-c48b-5fd5-8128055889d8", "00f430f9-0333-b8dc-8494-afdd006a430a", "5868fb9c-5cb1-4b37-5089-67c7bc9fef73", "454d3d03-5833-095c-fca9-6372d59cc770", "618d7655-150d-5c6e-7bcb-49e9d17c6d15", "37288194-8f7f-baca-dd3f-eccb48fb87b5", "dca10c59-42ad-440f-44eb-05d178649639", "e31f87fe-656c-5d6d-268a-3bd7cb8cf9a6", "40dadb52-6cea-bbe0-2f82-3074ecdc394d", "9a63c804-81f7-f1d0-2162-bb96938a7a75", "514f0214-5843-de32-144f-452e99467058", "9fc1c5bf-1794-b30c-805a-49dc5f640465", "7a8e4021-c0c2-44dc-90aa-ffa727faec0a", "d9c47e97-5714-683e-2068-b600fb7e5c40", "7897ea98-e43e-b005-6f9c-6b584c91fbd0", "d06dbbfa-044a-f0af-a23d-ea83bd870f1b", "006ce6a4-2c76-d341-0fe7-e04bf69f9dd2", "259270d8-a2f5-204b-c621-617eceaf4354", "8f023813-7237-193c-d292-92e3f6aece66", "49d07965-2bd1-41b6-75bc-1a710ba34cfe", "24ccf18f-ebca-96aa-65e6-6e0110d57340", "be0ab6d2-4b98-e3cd-e086-87df1f14ec5a", "cd726c78-d020-b378-ac02-998a19d84cda", "5a3216cf-2fac-de31-aa1e-e9f6f4d7951a", "c232b315-e64c-7a35-8dda-d12b3456cc0b", "a42219cc-e130-8040-7586-ad5924add1d8", "6bf9c334-1363-718a-4c17-40ede2fc0635", "1f6fa56d-40a9-598e-0476-b8d1a4aae5d6", "92a91901-dab3-c7e5-00bf-f870be2e5a09", "cf7049e7-5c71-0578-9705-0cbfb869845a", "95b70bd7-e713-1eea-95e2-742b428481c3", "f972af47-0b7c-9410-0b74-77e2f0803d87", "f8dc5a9a-f348-1cdf-7840-629de2830a25", "6239e6be-955a-f034-bce0-dc43f33c9030", "0180315f-5464-f787-b3cb-9ca41dca3a10", "11abbbe1-8dad-7aa7-8ff8-1afa0cd447e6", "629b4bf2-37d6-9ca0-b42a-bc17965ae679", "50a62cf0-2d5e-0d6b-946b-823188d52e94", "5bea350e-f884-b2ec-0a44-db9816f88cd4", "23752531-1055-c3d8-f3a2-097f3d27fcb2", "563b26d8-9569-a129-b626-c16c6a147c15", "bc50ca96-21a3-4822-d88e-579bc957616e", "6bfec4f0-cfa9-d4ef-7cfc-8c5873e4ef7a", "b04f1261-4f06-ab2e-dc3b-26bfead9c9ff", "638d26ac-547c-d528-5451-af0b315989a5", "e83df03c-4b3d-ce1c-a014-fbe0cf7aa044", "f14945b0-ac3e-a87d-03b0-d113775e50f9", "a72cccbf-aad5-f852-af3b-2df360517dce", "2f30cb3b-7f31-6195-d753-4595379086a7", "b4ac043b-1e8c-fe87-b632-6f7504d8b555", "709fc42e-ce62-441f-8364-4e812f85fef9", "b1a44999-7c80-34ad-8908-728c8791e293", "513f6fc0-48f8-a5a5-7430-fc590ec38fc9", "4158e19d-5d70-3a64-f0b1-dd2c1a079338", "5e1af44a-3426-fded-f86a-8231e3432a64", "d6f13e74-1555-51b1-c559-06dc4e60722b", "aac66be3-b98e-4b94-148c-9632573aad9d", "8ec9bf04-3bf6-f61a-9bd6-f04f54b27f03", "c9379eb4-409e-2e9a-3e29-417a55d35289", "cabeaf1a-78cf-7092-6e58-a321b047be46", "52340f27-1bb9-b82c-469c-7a5338e64962", "d2153cc1-e053-9fa8-5bd5-d01db237d3e8", "ae7181f4-db63-0a51-ee81-e3410fe3f73c", "7111d553-b318-6363-cb8b-226f93171aad", "0c819507-5f11-5226-1d90-f10f10ff5850", "54451b85-fff1-ce68-417b-7f52e488895d", "ac7fa445-e290-0b94-f5cf-63b1291f623c", "cac6dc8a-6ce6-5b3c-67f7-0075c5f3ea4e", "00cc940b-9f36-06cb-e009-f600db397dd1", "44b9acd4-bd9c-e1be-faf3-0e57a5c818eb", "496902b2-4d6b-3b9c-184e-e125625c9997", "5720e69f-dbf2-c426-0e82-14845ad4d4c4", "2cc3adbe-6bd0-81f2-6cb9-627b70eeeea4", "9c9a02db-5b20-9707-9760-f2d03bc3d479", "2f9468df-2e35-f0d6-f848-b402417dbb74", "fbccb793-5841-585a-fd1d-3bee337db904", "0cb56163-0d2b-a2e6-00e4-725fd01103df", "5ddc78f9-5c7c-74cd-025b-0bf6c7ee54f2", "a4f7b507-4de4-7be4-b064-fe1de3269e81", "fea96029-9cce-5544-6053-825f04f6f738", "0aa5b94a-46b9-f368-0c37-de28a70c2ac1", "7d63952e-55bd-393f-3b06-d64ed2c26c45", "25b0eff4-9a38-ff21-23dd-ccd14f985565", "b6effefb-9987-a03e-bacf-f4053f84257b", "65cf5893-60d2-f1a2-eca8-abba2dba9fdc", "e780b25a-c8af-ee61-cd9d-db9a7b31ec82", "56a40e11-e3f7-c8f1-3abe-093abbb29ab2", "7d5082b3-02fb-4ce3-92cd-81c015f63047", "f3d0420f-747e-15c5-a39e-0f542ba34ee8", "8f193c91-5a37-56d9-6128-4afbafb7fc1c", "b788bc97-e7bf-327c-d4de-0f1d227b98fa", "2ebab94c-cc83-3947-7f49-5d9621336699", "f5a313e3-ac3a-4f81-12bd-1951f78419c2", "0975b818-9d78-a2c6-90ee-cfd9cc50101b", "326074cc-bef8-863e-fb80-01b3b704040a", "dc64c5ea-0171-00ef-c852-fd3dc31b72d5", "acb36859-25a3-17c0-fdf2-33ebf69635ae", "aab4f744-8dce-c9fc-12bb-9399cd11acd3", "d5acf2a7-1a37-0e26-60ff-d40541fa46ea", "7744c8b0-646d-7545-678c-76f07c7b6ff2", "4b2509f2-f6aa-73c5-99af-4f1054b9fe29", "66c998d2-8343-5a03-229e-727bce9c6d46", "40dc1977-d9c3-77d0-aa78-83b2727cc9c2", "9d2cb5cb-5239-3905-bdac-a929a6433bf2", "888febb9-8903-7446-5a9f-944b36674650", "abf3869b-b679-39c4-d191-e585fd2f1899", "8ce1292c-f35b-4ca4-87c5-7394f403c82d", "db906a9c-b92b-861f-c43c-a2882db8d482", "5dc1f2e0-3450-a523-e4d3-a0a5172b4c0a", "f82f3958-08de-5655-6a65-e527714ac410", "984e0a48-4d62-b0cd-5ff7-156ff660d641", "213ae698-5fbe-739f-1697-b0d7c4a36ac8", "384da934-1779-d386-eeb5-e883467d5776", "dfdece1f-b224-1317-a3cd-e04ec8669f9f", "dee5f5cd-b75a-5fcc-b798-48fce9bce481", "849de416-a357-c936-bc3a-17efbca13605", "1bb0cf11-afc9-a717-f1dd-2ce50fb83ff3", "07026dfc-680b-9807-2f4b-5ad90c1ade73", "a88db506-f094-bc60-53ea-999c3f8340bc", "618c11a9-7855-dc28-c4ea-f9ab2fbcad6b", "0045deef-8616-096f-f53c-2d8a924a0268", "bf7956a3-6f28-d119-47a4-83a951a8a1d1", "8ee6813c-5357-c383-6549-d798d14973a4", "fa62e132-ed33-a7e1-6741-331badfc259d", "06928554-c48d-751b-6f22-b08af077468b", "66ae3fca-d5c3-1bd6-06c1-d30dc85d032e", "6ab7744b-68a0-a0c2-4482-9fe1fde15d83", "fb86777d-ef97-7bfd-95d6-91cfaa06cde8", "43d33c16-b9aa-1a75-40b1-0e97b9316656", "efe2217d-5673-1918-0fe3-483452b038db", "d441f71b-b3dc-3c29-995d-6da05659e59c", "42d3c345-2362-e3f6-4dc9-614ba7192e7c", "93dd2fd2-041e-c000-1922-b7fbe0b7f1d5", "c92b2be8-5ba2-371c-a789-5375d52cc938", "bae7b90f-8455-85cd-c8c5-7e0e5cd642d2", "f22d0a37-49b4-51a7-8e17-b8477a037010", "ea5a0604-39cf-9e71-5cd0-b4ca4db27a56", "09919c3a-5002-eccf-a9db-e161765607f8", "76e59997-f03b-6665-421f-ce2e6ac4a4b2", "73f3c1ea-4a79-3809-019e-38c819eeaece", "9a6a7499-ec0e-026e-f8ee-85d14e4c3c6e", "c44f55df-ec30-37c0-b655-b1297205a094", "e88ce0eb-5933-3698-5bb8-7e30758809ee", "538d69c4-4332-894c-8017-cf5fbdd8802c", "1507f0aa-baf6-5b67-36aa-b2eac4fb78a0", "7883e9e2-4f0e-63d9-00dc-4a505404cde0", "cd224b1f-df5f-2ce5-f3a1-d10ab12a86c0", "0d44b86d-c0e4-0495-8630-433718376239", "4ad3b3db-1655-5456-87e2-8faceb2bc626", "5c1991d8-26f4-3949-7a60-2c7d948a5e8b", "24f88709-9526-4943-3044-0bac87433912", "f3957389-cee1-5cd3-2203-c959c6c4218c", "04122c57-fe77-b8ee-1691-09e5ff225cb0", "7384a3a7-963f-3749-6d3c-f88316bb9a66", "5d5c7c3b-28e3-637a-4ee7-18e25b4e49d9", "a317ca48-86c0-837c-0e4d-0148463efbaf", "a70d551f-7eb4-b072-6a56-8cb07c07637b", "01d58a7c-a847-09ab-d3b4-a813c8028cd8", "92514d93-9ef4-60ff-5270-cf4557ef13a4", "a96f3a24-bda1-a178-e4ce-fb36a3433c88", "78ca001b-b0de-6699-8dd0-810c7349cb4b", "7e64cf7e-8ab9-b5d6-eb15-4196e41abada", "de732aa3-5c18-db19-fb11-3374f78a976b", "d2a8e132-2f55-dd82-3161-4d455a19cf0a", "20a5390b-f6c3-901d-9931-382acaf30db9", "f1523d16-201a-70db-0ab7-4ede1daf5aac", "a474a2db-33d1-4e40-75d6-57efbc1a8aa3", "707fc405-55a8-f038-23c1-284abb021690", "98bb6e73-5c9c-b421-0207-18b2492299b1", "c86dde43-361c-4c06-84a9-bf249804d295", "46e5d4b0-236a-ef16-2efc-74f75c43c69b", "4b5fdde5-4577-81ac-aeb1-2a3873ca3be8", "5a099179-3ae0-4cde-dd1a-a658283b341d", "e517ae73-882b-4b21-c17a-a72476f378f9", "9fb63a18-ea3e-d744-b2ab-e6a44303bd04", "b1b08725-b9d1-25c3-64c1-9d4215518524", "f57ddc3a-dae6-4b48-ff66-e8e31d651b2b", "7e2dabfc-e844-f7b7-222e-1085ea1ebbef", "e2862e97-3d1b-9e72-74c2-e4bdd13078b0", "d9e4ebbc-fdfa-3979-7fc4-d9081f33a143", "7942b60b-fa91-cdb5-8f04-cfc427379156", "9d81c1dc-c36f-755a-a3ce-7f8a14a96b4d", "34f15028-c406-6c46-bee6-85fac04b633f", "83a8ab13-7741-d900-7b7b-51fa573d9840", "74d7c335-4195-f5e4-939f-57275745ecad", "92b1e830-4fd0-8b26-3a1e-a967d9efe6fb", "66fd8898-53a2-adfc-c5b4-5dee55f7dabf", "36f53743-67ff-9890-3d1e-e444736f14bc", "b358dab7-5f81-3985-e2e7-f35dc1867e3b", "8cfdb9bc-cd5e-2be5-6529-2ca8e0bf95ba", "5e6cd5cb-fad2-ea8d-de94-99c4b5c0705b", "6e1043b7-6724-5a7a-8ae7-4bc994987fd4", "8135279c-f4d8-7df1-580f-17247b73b2cb", "d4c9f91d-225f-5a9f-c117-fff6011dbcfd", "30c4d05b-dc21-9bf5-e355-1af02fb089cd", "6aa030b9-9186-9078-ca8d-21cde3fffabe", "f7554e0a-aa53-d694-fd1c-714fb685c6db", "b3ccf2f0-378a-0b06-991c-c2ce0cf80d38", "054892a2-2017-de07-a04b-bf6f4b1e6253", "c57989fd-10bd-3d7b-0adf-1155e265b12b", "e1d04aea-6b9e-fd57-1788-fb5891d1e03a", "3d887702-adfb-01fb-a3d1-6cd6a215f274", "144b4685-2286-24d4-025d-314c8e20b2da", "997b795e-c893-1435-899e-78b57a95efb9", "8ce921ef-856f-41a8-59c8-7623267234da", "b92d89db-072c-32df-28a0-0fafb7e43731", "1c587ec5-f6fb-b967-3d09-f23523aa0e2e", "02c485b4-910d-a608-bd97-33e7f4b36d8d", "2e83d180-d9ce-8f17-7835-0820b2e75901", "8313b81d-7fe9-34fb-635e-b4ce212994ef", "48c55eb3-6677-4e28-e5e7-73fbd1f167b7", "62750eb4-0d84-a3cd-f07e-6444f15828d1"]
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:08:39.182Z [INFO] internal_plugin.aws-asg: successfully purged Nomad node: node_id=481493a9-9889-fb7b-ef2a-3fd3810465e5 nomad_evals=["96dab34f-7795-5deb-57e1-572c7a98a4a6", "d07054dc-c0b3-8d03-8988-888616ee2884", "48042897-2686-5053-e480-aa58663468a4"]
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:08:39.050Z [ERROR] internal_plugin.aws-asg: failed to ensure all activities completed: action=scale_in asg_name=maya-box-asg error="reached retry limit"
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:06:16.487Z [INFO] internal_plugin.aws-asg: received node drain message: node_id=66e6e80c-06b3-bba7-2636-0aa63b0cc0a5 msg="All allocations on node "66e6e80c-06b3-bba7-2636-0aa63b0cc0a5" have stopped"
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:06:16.059Z [INFO] internal_plugin.aws-asg: received node drain message: node_id=66e6e80c-06b3-bba7-2636-0aa63b0cc0a5 msg="Drain complete for node 66e6e80c-06b3-bba7-2636-0aa63b0cc0a5"
"3dc05b38-36d3-ef8b-bf47-efbb68811b47"  2023-07-18T13:06:08.218Z [WARN] internal_plugin.aws-asg: node pool status readiness check failed: error="node 66e6e80c-06b3-bba7-2636-0aa63b0cc0a5 is draining"

Policy config:

      template {
        data        = <<EOF
        scaling "box-asg-scaling-policy" {
          enabled = true

          min     = ${BOXES_ASG_MIN_COUNT}
          max     = ${BOXES_ASG_MAX_COUNT}

          policy {

            cooldown            = "3m"
            evaluation_interval = "1m"

            check "cpu-allocated-percentage" {
              source       = "nomad-apm"
              query        = "percentage-allocated_cpu"
              strategy "target-value" {
                # Nomad only knows about half of the CPUs because of Numa.
                # Therefore we need to set here 60/2, if we want to scale at 60%.
                target = 30
              }
            }

            check "memory-allocated-percentage" {
              source       = "nomad-apm"
              query        = "percentage-allocated_memory"
              strategy "target-value" {
                target = 60
              }
            }

            target "aws-asg" {
              dry-run                       = "false"
              aws_asg_name                  = "${ASG_BOX_NAME}"
              node_class                    = "box"
              node_drain_deadline           = "1h"
              node_drain_ignore_system_jobs = "true"
              node_purge                    = "true"
              node_selector_strategy        = "least_busy"
            }

          }
        }
        EOF
        destination = "$${NOMAD_TASK_DIR}/policies/box-asg-scaling-policy.hcl"
      }

Please also note that there were no scaling events from 2023-07-18T13:10:21.646Z to 2023-07-18T13:23:21.891Z because of a draining node. This also lines up what we are seeing in the ASG Activity history:

Cause: 
At 2023-07-18T13:06:29Z instances were launched to balance instances in zones us-east-1a us-east-1c 
with other zones resulting in more than desired number of instances in the group. 
At 2023-07-18T13:06:41Z an instance was taken out of service in response 
to a difference between desired and actual capacity, shrinking the capacity from 10 to 9. 
At 2023-07-18T13:06:41Z instance i-0b3c2b3efde3b140b was selected for termination.

Start time: 2023 July 18, 03:06:41 PM +02:00    
End time: 2023 July 18, 03:23:14 PM +02:00

This means that we couldn't scale for 16 minutes. I've created a separate issue for this problem.

lgfa29 commented 10 months ago

I was finally able to reproduce this and the analysis in https://github.com/hashicorp/nomad-autoscaler/issues/610#issuecomment-1636615580 seems correct. #812 should fix this problem 🙂