SPIKE: Reduce the number of VMs required to run autoscaler

[cweibel]

In order to reduce the cost of running autoscaler, we need to reduce the number of vms required to run it

Acceptance Criteria

• [ ] GIVEN cg-deploy-autoscaler, and app-autoscaler-release/templates /app-autoscaler.yml try to create ops files and/or pipeline manipulation to deploy autoscaler as a single vms

---

Security considerations

None

Implementation sketch

• [x] Use https://github.com/cloudfoundry/app-autoscaler-release/tree/main/templates/app-autoscaler.yml as a starting point (like the pipeline does). Render the the manifest with bosh int and figure out a series of ops files to extract and merge into a single instance group.
• [x] Figure out a way to repeatedly do this without human intervention for changes to the bosh release. Check out how we did logstash router groups in CF for one possible solution https://github.com/cloud-gov/cg-deploy-cf/pull/773

[cweibel]

Ran into a blocker as noted in the comments below:

mkdir -p autoscaler/terraform-secrets
mkdir -p autoscaler/terraform-yaml
cd autoscaler

git clone https://github.com/cloudfoundry/app-autoscaler-release     #app-autoscaler-release
git clone https://github.com/cloud-gov/cg-deploy-autoscaler.git      #autoscaler-manifests

mv cg-deploy-autoscaler autoscaler-manifests

export AWS_DEFAULT_REGION="us-gov-west-1"
aws s3 cp s3://terraform-state/development/state.yml terraform-yaml/state.yml --sse AES256

bosh interpolate \
  autoscaler-manifests/bosh/varsfiles/terraform.yml \
  -l terraform-yaml/state.yml \
  > terraform-secrets/terraform.yml

bosh interpolate \
   app-autoscaler-release/templates/app-autoscaler.yml \
      -o autoscaler-manifests/bosh/opsfiles/instance-identity-cert-from-cf-dev.yml \
      -o autoscaler-manifests/bosh/opsfiles/enable-nats-tls-dev.yml \
      -o autoscaler-manifests/bosh/opsfiles/loggregator-certs-from-cf-dev.yml \
      -o app-autoscaler-release/operations/add-extra-plan.yml \
      -o autoscaler-manifests/bosh/opsfiles/enable-log-cache-dev.yml \
      -o autoscaler-manifests/bosh/opsfiles/log-cache-syslog-server-dev.yml \
      -o autoscaler-manifests/bosh/opsfiles/external-db.yml \
      -o autoscaler-manifests/bosh/opsfiles/bosh-dns-cf-deployment-name.yml \
      -o app-autoscaler-release/operations/remove-metricsserver.yml \
      -o app-autoscaler-release/operations/remove-metricsgateway.yml \
      -o app-autoscaler-release/operations/enable-scheduler-logging.yml \
      -o autoscaler-manifests/bosh/opsfiles/cf-uaa-client-secret-dev.yml \
      -o autoscaler-manifests/bosh/opsfiles/scaling-dev.yml \
      -l autoscaler-manifests/bosh/varsfiles/development.yml \
      -l terraform-secrets/terraform.yml > as-dev.yml

# bosh deploy -d app-autoscaler as-dev.yml # Yields a matching deploy

cat << EOF > remove_consumes.yml
- type: remove
  path: /name=route_registrar
EOF

echo "addons:"               > single.yml ; bosh interpolate as-dev.yml --path /addons   > single.yml
echo "domains:"             >> single.yml ; bosh interpolate as-dev.yml --path /domains  > indentme; sed 's/^/  /' indentme >> single.yml
echo "features:"            >> single.yml ; bosh interpolate as-dev.yml --path /features > indentme; sed 's/^/  /' indentme >> single.yml
cat << EOF                  >> single.yml
instance_groups:
- azs:
  - z1
  instances: 1
  name: onvmtorulethemall
  networks:
  - name: default
  stemcell: default
  vm_type: t3.large
  jobs:
EOF
bosh interpolate as-dev.yml --path /instance_groups/name=metricsforwarder/jobs   > job.yml
bosh interpolate job.yml -o remove_consumes.yml > jobs.yml
bosh interpolate as-dev.yml --path /instance_groups/name=scalingengine/jobs      > job.yml
bosh interpolate job.yml -o remove_consumes.yml >> jobs.yml
bosh interpolate as-dev.yml --path /instance_groups/name=apiserver/jobs          > job.yml
bosh interpolate job.yml -o remove_consumes.yml >> jobs.yml
bosh interpolate as-dev.yml --path /instance_groups/name=scheduler/jobs          > job.yml
bosh interpolate job.yml -o remove_consumes.yml >> jobs.yml
bosh interpolate as-dev.yml --path /instance_groups/name=operator/jobs           > job.yml
bosh interpolate job.yml -o remove_consumes.yml >> jobs.yml
bosh interpolate as-dev.yml --path /instance_groups/name=eventgenerator/jobs     >> jobs.yml
sed 's/^/  /' jobs.yml >> single.yml

echo "name: app-autoscaler" >> single.yml
echo "public_domains:"      >> single.yml ; bosh interpolate as-dev.yml --path /public_domains > indentme; sed 's/^/  /' indentme >> single.yml
echo "releases:"            >> single.yml ; bosh interpolate as-dev.yml --path /releases       >> single.yml
echo "stemcells:"           >> single.yml ; bosh interpolate as-dev.yml --path /stemcells      >> single.yml
echo "update:"              >> single.yml ; bosh interpolate as-dev.yml --path /update          > indentme; sed 's/^/  /' indentme >> single.yml
echo "variables:"           >> single.yml ; bosh interpolate as-dev.yml --path /variables      >> single.yml

bosh interpolate as-dev.yml --path /instance_groups/name=metricsforwarder/jobs/name=route_registrar/properties/route_registrar/routes   > route_registrar.yml
bosh interpolate as-dev.yml --path /instance_groups/name=scalingengine/jobs/name=route_registrar/properties/route_registrar/routes     >> route_registrar.yml
bosh interpolate as-dev.yml --path /instance_groups/name=apiserver/jobs/name=route_registrar/properties/route_registrar/routes         >> route_registrar.yml
bosh interpolate as-dev.yml --path /instance_groups/name=scheduler/jobs/name=route_registrar/properties/route_registrar/routes         >> route_registrar.yml
bosh interpolate as-dev.yml --path /instance_groups/name=operator/jobs/name=route_registrar/properties/route_registrar/routes          >> route_registrar.yml

cat > routes.yml <<EOF
- type: replace
  path: /instance_groups/name=onvmtorulethemall/jobs/name=route_registrar/properties/route_registrar/routes/-
  value:
EOF

sed 's/^/    /' route_registrar.yml > route_registrar_indented.yml
cat route_registrar_indented.yml >> routes.yml
bosh interpolate single.yml -o routes.yml > single_with_routes.yml

#blows up ^^ since the route_registrar_indented.yml is multiple arrays and the bosh int on the next line can only handle a single array replace command at a timie
# since the number of array values could change in the future, as well as their names, doing this lookup feels very very brittle

cat << EOF> onvmtorulethemall-dns.yml 
- type: remove
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=app-autoscaler.autoscalerpostgres.service.cf.internal
- type: remove
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=app-autoscaler.apiserver.service.cf.internal
- type: remove
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=app-autoscaler.autoscalerscheduler.service.cf.internal
- type: remove
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=app-autoscaler.servicebroker.service.cf.internal
- type: remove
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=app-autoscaler.eventgenerator.service.cf.internal
- type: remove
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/domain=app-autoscaler.scalingengine.service.cf.internal

- type: replace
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/-
  value:
    domain: app-autoscaler.autoscalerpostgres.service.cf.internal
    targets:
    - deployment: app-autoscaler
      domain: bosh
      instance_group: onvmtorulethemall
      network: default
      query: '*'
- type: replace
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/-
  value:
    domain: app-autoscaler.apiserver.service.cf.internal
    targets:
    - deployment: app-autoscaler
      domain: bosh
      instance_group: onvmtorulethemall
      network: default
      query: '*'
- type: replace
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/-
  value:
    domain: app-autoscaler.autoscalerscheduler.service.cf.internal
    targets:
    - deployment: app-autoscaler
      domain: bosh
      instance_group: onvmtorulethemall
      network: default
      query: '*'
- type: replace
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/-
  value:
    domain: app-autoscaler.servicebroker.service.cf.internal
    targets:
    - deployment: app-autoscaler
      domain: bosh
      instance_group: onvmtorulethemall
      network: default
      query: '*'
- type: replace
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/-
  value:
    domain: app-autoscaler.eventgenerator.service.cf.internal
    targets:
    - deployment: app-autoscaler
      domain: bosh
      instance_group: onvmtorulethemall
      network: default
      query: '*'
- type: replace
  path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties/aliases/-
  value:
    domain: app-autoscaler.scalingengine.service.cf.internal
    targets:
    - deployment: app-autoscaler
      domain: bosh
      instance_group: onvmtorulethemall
      network: default
      query: '*'
EOF

bosh interpolate single_with_routes.yml -o onvmtorulethemall-dns.yml > as-single.yml

The decision was made to stop pursuing this since future maintenance makes this process brittle