vpc leak

[Sn0rt]

/kind bug

What steps did you take and what happened:

1: prepare

guohao@buffer ~/Downloads $ export AWS_CREDENTIALS=$(aws iam create-access-key \
  --user-name bootstrapper.cluster-api-provider-aws.sigs.k8s.io)
export AWS_ACCESS_KEY_ID=$(echo $AWS_CREDENTIALS | jq .AccessKey.AccessKeyId -r)
export AWS_SECRET_ACCESS_KEY=$(echo $AWS_CREDENTIALS | jq .AccessKey.SecretAccessKey -r)
guohao@buffer ~/Downloads $ export AWS_REGION=us-east-2
guohao@buffer ~/Downloads $ rm -rf /Users/guohao/Downloads/aws/out/*
zsh: sure you want to delete all 6 files in /Users/guohao/Downloads/aws/out [yn]? y
guohao@buffer ~/Downloads $ ./aws/generate-yaml.sh
Done generating /Users/guohao/Downloads/aws/out/cluster.yaml
Done generating /Users/guohao/Downloads/aws/out/machines.yaml
Done generating /Users/guohao/Downloads/aws/out/machines-ha.yaml
Done generating /Users/guohao/Downloads/aws/out/machine-deployment.yaml
Done copying /Users/guohao/Downloads/aws/out/addons.yaml
Generated credentials
Done writing /Users/guohao/Downloads/aws/out/provider-components.yaml
WARNING: /Users/guohao/Downloads/aws/out/provider-components.yaml includes credentials

2: check the true world (query by AWS CLI

guohao@buffer ~/Downloads $ aws ec2 describe-vpcs
{
    "Vpcs": []
}

3: run clusterctl without -v=10

guohao@buffer ~/Downloads $ clusterctl create cluster \
  --bootstrap-type kind \
  --provider aws \
  -m ./aws/out/machines.yaml \
  -c ./aws/out/cluster.yaml \
  -p ./aws/out/provider-components.yaml \
  -a ./aws/out/addons.yaml

I0610 16:10:40.191381   29554 createbootstrapcluster.go:27] Preparing bootstrap cluster
I0610 16:11:07.286393   29554 clusterdeployer.go:78] Applying Cluster API stack to bootstrap cluster
I0610 16:11:07.286416   29554 applyclusterapicomponents.go:26] Applying Cluster API Provider Components
I0610 16:11:08.136889   29554 clusterdeployer.go:83] Provisioning target cluster via bootstrap cluster
I0610 16:11:08.146539   29554 applycluster.go:36] Creating cluster object test1 in namespace "default"
I0610 16:11:08.153783   29554 clusterdeployer.go:92] Creating control plane controlplane-0 in namespace "default"
I0610 16:11:08.159542   29554 applymachines.go:36] Creating machines in namespace "default"

4. inspect

   guohao@buffer ~ $ export KUBECONFIG="$(kind get kubeconfig-path --name="clusterapi")"
   guohao@buffer ~ $ kubectl  get pod --all-namespaces -o wide
   NAMESPACE             NAME                                READY   STATUS    RESTARTS   AGE   IP           NODE                       NOMINATED NODE   READINESS GATES
   aws-provider-system   aws-provider-controller-manager-0   1/1     Running   0          23s   10.244.0.2   clusterapi-control-plane   <none>           <none>
   cluster-api-system    cluster-api-controller-manager-0    1/1     Running   0          23s   10.244.0.3   clusterapi-control-plane   <none>           <none>
   kube-system           coredns-fb8b8dccf-f7gq8             1/1     Running   0          23s   10.244.0.4   clusterapi-control-plane   <none>           <none>
   kube-system           coredns-fb8b8dccf-tdgzl             1/1     Running   0          23s   10.244.0.5   clusterapi-control-plane   <none>           <none>
   kube-system           ip-masq-agent-t75vd                 1/1     Running   0          22s   172.17.0.2   clusterapi-control-plane   <none>           <none>
   kube-system           kindnet-7wzjr                       1/1     Running   0          22s   172.17.0.2   clusterapi-control-plane   <none>           <none>
   kube-system           kube-proxy-twghf                    1/1     Running   0          22s   172.17.0.2   clusterapi-control-plane   <none>           <none>

5. the log of aws-provider-controller-manager

guohao@buffer ~ $ kubectl -n aws-provider-system logs aws-provider-controller-manager-0  -f
I0610 08:11:38.699257       1 plugins.go:39] Registered cluster provisioner "aws"
I0610 08:11:38.702838       1 reflector.go:131] Starting reflector *v1alpha1.Cluster (10m0s) from pkg/cache/internal/informers_map.go:126
I0610 08:11:38.702901       1 reflector.go:169] Listing and watching *v1alpha1.Cluster from pkg/cache/internal/informers_map.go:126
I0610 08:11:38.703291       1 reflector.go:131] Starting reflector *v1alpha1.Machine (10m0s) from pkg/cache/internal/informers_map.go:126
I0610 08:11:38.703329       1 reflector.go:169] Listing and watching *v1alpha1.Machine from pkg/cache/internal/informers_map.go:126
I0610 08:11:38.969089       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:38.969092       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:11:38.979044       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:11:38.979086       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:11:38.979106       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:11:38.979480       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:11:38.979495       1 certificates.go:74] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Generating keypair for"  "user"="cluster-ca"
I0610 08:11:39.070891       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:39.070990       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:39.072116       1 machine_scope.go:161] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=1 "msg"="Patching machine"
I0610 08:11:39.077324       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:39.077388       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:39.080525       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:41.269074       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:41.269759       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:42.270294       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:42.270444       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:42.271907       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:42.271974       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:42.276180       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:42.488273       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:42.488889       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:43.489488       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:43.489697       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:43.572466       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:43.572557       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:43.768147       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:43.977629       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:43.978233       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:44.978897       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:44.979000       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:44.979684       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:44.979753       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:44.983663       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:45.193302       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:45.193918       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:46.196452       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:46.196662       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:46.268223       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:46.367578       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:46.373236       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:46.667706       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:46.668486       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:47.668741       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:47.668956       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:47.669715       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:47.669832       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:47.676664       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:47.885728       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:47.886152       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:48.268813       1 certificates.go:83] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Generating keypair"  "user"="etcd-ca"
I0610 08:11:48.886900       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:48.887045       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:48.969766       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:48.969873       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:48.975770       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:49.189128       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:49.189560       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:50.190753       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:50.190892       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:50.191847       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:50.191941       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:50.197043       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:50.479878       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:50.480446       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:51.075064       1 certificates.go:91] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Generating keypair"  "user"="front-proxy-ca"
I0610 08:11:51.567535       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:51.567644       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:51.568366       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:51.568434       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:51.572489       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:51.867636       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:51.968275       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:53.267938       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:53.268079       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:53.668429       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:53.668533       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:53.674840       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:53.885625       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:53.886480       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:56.447131       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:11:56.447361       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:56.448233       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:11:56.448319       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:11:56.468627       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:11:56.767676       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:11:56.768308       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:11:58.075765       1 certificates.go:100] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Generating service account keys"  "user"="service-account"
I0610 08:12:01.371165       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:01.371275       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:01.888854       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:12:01.889018       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:12:01.890812       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:12:01.890912       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:12:01.894927       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:12:02.178291       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:12:02.179419       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": certificate not found in config
I0610 08:12:02.468105       1 scope.go:197] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="Patching cluster"
I0610 08:12:02.478512       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:02.487821       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: cc277ef7-2f64-49e3-9426-bb87acfeee88
I0610 08:12:03.488557       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:03.488641       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:03.488686       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:03.490764       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:03.490829       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:03.490843       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:04.250226       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:04.256718       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: 51b1dbcb-43ea-4e5d-b7e3-3ec62fd27f14
I0610 08:12:05.257332       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:05.257419       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:05.257457       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:05.259171       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:05.259256       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:05.259274       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:06.237023       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:06.275214       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: ff1f9375-021b-420e-823a-5fde8e61559d
I0610 08:12:07.275875       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:07.275932       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:07.275993       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:07.278188       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:07.278273       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:07.278290       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:08.770311       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:08.777054       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: 477ee04e-653f-4aba-9006-3ce71dc12c9d
I0610 08:12:09.845979       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:09.846064       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:09.846146       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:09.847781       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:09.847864       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:09.847881       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:11.462049       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:11.468512       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: 947c77e1-9e63-4a86-aefb-363c42094177
I0610 08:12:12.397757       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:12:12.397927       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:12:12.400243       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:12:12.400333       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:12:12.407270       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:12:12.468960       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:12.469009       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:12.469035       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:12.470027       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:12.470071       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:12.470081       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:13.294531       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
I0610 08:12:13.660741       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:13.745917       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 2fd05739-3a5c-4ea6-a141-8e1414aa38cf
W0610 08:12:14.258602       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": failed to get DNS address: no classic load balancer found with name: "test1-apiserver": LoadBalancerNotFound: There is no ACTIVE Load Balancer named 'test1-apiserver'
    status code: 400, request id: 74500eba-8b57-11e9-91a3-69e538eafa47
I0610 08:12:14.746344       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:14.746430       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:14.746469       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:14.750062       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:14.750131       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:14.750149       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:15.262571       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:15.270115       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 352dd9e6-66de-4710-ab6c-1f34dbe8323a
I0610 08:12:16.271029       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:16.345138       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:16.345283       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:16.348361       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:16.348537       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:16.348562       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:17.745082       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:17.752683       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: c2b21972-bac4-4d53-95c4-4a699f14f074
I0610 08:12:18.753709       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:18.753796       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:18.753834       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:18.756315       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:18.756397       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:18.756491       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:19.853265       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:19.861290       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 510abfcc-fd39-4760-8d7f-e146527cb589
I0610 08:12:21.142446       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:21.142682       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:21.142803       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:21.145465       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:21.145551       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:21.145588       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:22.187373       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:22.194744       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 0647b86f-7e5e-476c-8d18-b1cf2f987aba
I0610 08:12:24.755198       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:24.755288       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:24.755326       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:24.758336       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:24.758400       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:24.758414       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:25.897817       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:25.906630       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 01dbbb15-9dd3-4961-b3e2-fdf5090ed605
I0610 08:12:31.027334       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:31.027427       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:31.027468       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:31.046221       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:31.046348       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:31.046366       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:32.188432       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:32.196402       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: af0b08d9-9917-4911-9fe6-cca7b97c99f5
I0610 08:12:34.739075       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:12:34.739236       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:12:34.742256       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:12:34.742343       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:12:34.750591       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:12:35.603644       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:12:36.426016       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": failed to get DNS address: no classic load balancer found with name: "test1-apiserver": LoadBalancerNotFound: There is no ACTIVE Load Balancer named 'test1-apiserver'
    status code: 400, request id: 81836c52-8b57-11e9-9159-4b3db74b1bd5
I0610 08:12:42.414141       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:12:42.414265       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:12:42.414317       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:42.416922       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:42.417028       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:12:42.417067       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:12:42.889632       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:12:42.899424       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 7c60dc50-93b0-4864-9584-a07e6d12f02a
I0610 08:13:03.380848       1 controller.go:89] Running reconcile Cluster for test1
I0610 08:13:03.381076       1 controller.go:127] reconciling cluster object test1 triggers idempotent reconcile.
I0610 08:13:03.381149       1 actuator.go:61] [cluster-actuator] "level"=0 "msg"="Reconciling Cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:13:03.422928       1 certificates.go:71] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling certificates"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:13:03.423138       1 network.go:21] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling network for cluster"  "cluster-name"="test1" "cluster-namespace"="default"
I0610 08:13:03.423212       1 vpc.go:39] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=2 "msg"="Reconciling VPC"
I0610 08:13:04.676121       1 scope.go:217] [cluster-actuator]/cluster.k8s.io/v1alpha1/default/test1 "level"=1 "msg"="updating cluster status"
E0610 08:13:04.684575       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to create vpc: VpcLimitExceeded: The maximum number of VPCs has been reached.
    status code: 400, request id: 0039b58e-8cfc-489b-8950-d948fccf98ac
I0610 08:13:17.340920       1 controller.go:113] Reconciling Machine "controlplane-0"
I0610 08:13:17.341114       1 actuator.go:425] [machine-actuator] "level"=0 "msg"="Checking if machine exists in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:13:17.345908       1 controller.go:222] Reconciling machine object controlplane-0 triggers idempotent create.
I0610 08:13:17.346008       1 actuator.go:153] [machine-actuator] "level"=0 "msg"="Creating machine in cluster"  "cluster-name"="test1" "machine-name"="controlplane-0" "machine-namespace"="default"
I0610 08:13:17.354013       1 instances.go:68] [machine-actuator]/cluster.k8s.io/v1alpha1/default/test1/controlplane-0 "level"=2 "msg"="Looking for existing machine instance by tags"
I0610 08:13:18.182976       1 actuator.go:140] [machine-actuator] "level"=2 "msg"="Will machine join the controlplane"  "machine-name"="controlplane-0" "machine-namespace"="controlplane-0" "should-join-control-plane"=false
W0610 08:13:19.013257       1 controller.go:229] Failed to create machine "controlplane-0": failed to retrieve kubeconfig while creating machine "controlplane-0": failed to get DNS address: no classic load balancer found with name: "test1-apiserver": LoadBalancerNotFound: There is no ACTIVE Load Balancer named 'test1-apiserver'
    status code: 400, request id: 9ae894e9-8b57-11e9-aed4-b1ce77670a0f

and to count the create new vpc

guohao@buffer ~ $ kubectl -n aws-provider-system logs aws-provider-controller-manager-0  | grep "create new vpc" | wc -l
      14

6. check real world by aws cli

guohao@buffer ~ $ aws ec2 describe-vpcs | grep VpcId
            "VpcId": "vpc-0d12689c3c8c0f9b5",
            "VpcId": "vpc-0fd56325f25b4f63d",
            "VpcId": "vpc-03d3e7d933a1b2438",
            "VpcId": "vpc-04f064aa2552638e2",
            "VpcId": "vpc-01feaa26646588c6f",

What did you expect to happen:

guohao@buffer ~/Downloads $ cat ./aws/out/*
# Calico Version v3.6
# https://docs.projectcalico.org/v3.6/release-notes/
---
# Source: calico/templates/calico-config.yaml
# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
apiVersion: v1
metadata:
  name: calico-config
  namespace: kube-system
data:
  # Typha is disabled.
  typha_service_name: "none"
  # Configure the Calico backend to use.
  calico_backend: "bird"

  # Configure the MTU to use
  veth_mtu: "1440"

  # The CNI network configuration to install on each node.  The special
  # values in this config will be automatically populated.
  cni_network_config: |-
    {
      "name": "k8s-pod-network",
      "cniVersion": "0.3.0",
      "plugins": [
        {
          "type": "calico",
          "log_level": "info",
          "datastore_type": "kubernetes",
          "nodename": "__KUBERNETES_NODE_NAME__",
          "mtu": __CNI_MTU__,
          "ipam": {
              "type": "calico-ipam"
          },
          "policy": {
              "type": "k8s"
          },
          "kubernetes": {
              "kubeconfig": "__KUBECONFIG_FILEPATH__"
          }
        },
        {
          "type": "portmap",
          "snat": true,
          "capabilities": {"portMappings": true}
        }
      ]
    }

---
# Source: calico/templates/kdd-crds.yaml
# Create all the CustomResourceDefinitions needed for
# Calico policy and networking mode.

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
   name: felixconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: FelixConfiguration
    plural: felixconfigurations
    singular: felixconfiguration
---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ipamblocks.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPAMBlock
    plural: ipamblocks
    singular: ipamblock

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: blockaffinities.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BlockAffinity
    plural: blockaffinities
    singular: blockaffinity

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ipamhandles.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPAMHandle
    plural: ipamhandles
    singular: ipamhandle

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ipamconfigs.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPAMConfig
    plural: ipamconfigs
    singular: ipamconfig

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgppeers.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPPeer
    plural: bgppeers
    singular: bgppeer

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgpconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPConfiguration
    plural: bgpconfigurations
    singular: bgpconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ippools.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPPool
    plural: ippools
    singular: ippool

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: hostendpoints.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: HostEndpoint
    plural: hostendpoints
    singular: hostendpoint

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: clusterinformations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: ClusterInformation
    plural: clusterinformations
    singular: clusterinformation

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworkpolicies.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkPolicy
    plural: globalnetworkpolicies
    singular: globalnetworkpolicy

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworksets.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkSet
    plural: globalnetworksets
    singular: globalnetworkset

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: networkpolicies.crd.projectcalico.org
spec:
  scope: Namespaced
  group: crd.projectcalico.org
  version: v1
  names:
    kind: NetworkPolicy
    plural: networkpolicies
    singular: networkpolicy
---
# Source: calico/templates/rbac.yaml

# Include a clusterrole for the kube-controllers component,
# and bind it to the calico-kube-controllers serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: calico-kube-controllers
rules:
  # Nodes are watched to monitor for deletions.
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - watch
      - list
      - get
  # Pods are queried to check for existence.
  - apiGroups: [""]
    resources:
      - pods
    verbs:
      - get
  # IPAM resources are manipulated when nodes are deleted.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
    verbs:
      - list
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
      - ipamblocks
      - ipamhandles
    verbs:
      - get
      - list
      - create
      - update
      - delete
  # Needs access to update clusterinformations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - clusterinformations
    verbs:
      - get
      - create
      - update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: calico-kube-controllers
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-kube-controllers
subjects:
- kind: ServiceAccount
  name: calico-kube-controllers
  namespace: kube-system
---
# Include a clusterrole for the calico-node DaemonSet,
# and bind it to the calico-node serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: calico-node
rules:
  # The CNI plugin needs to get pods, nodes, and namespaces.
  - apiGroups: [""]
    resources:
      - pods
      - nodes
      - namespaces
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - endpoints
      - services
    verbs:
      # Used to discover service IPs for advertisement.
      - watch
      - list
      # Used to discover Typhas.
      - get
  - apiGroups: [""]
    resources:
      - nodes/status
    verbs:
      # Needed for clearing NodeNetworkUnavailable flag.
      - patch
      # Calico stores some configuration information in node annotations.
      - update
  # Watch for changes to Kubernetes NetworkPolicies.
  - apiGroups: ["networking.k8s.io"]
    resources:
      - networkpolicies
    verbs:
      - watch
      - list
  # Used by Calico for policy information.
  - apiGroups: [""]
    resources:
      - pods
      - namespaces
      - serviceaccounts
    verbs:
      - list
      - watch
  # The CNI plugin patches pods/status.
  - apiGroups: [""]
    resources:
      - pods/status
    verbs:
      - patch
  # Calico monitors various CRDs for config.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - globalfelixconfigs
      - felixconfigurations
      - bgppeers
      - globalbgpconfigs
      - bgpconfigurations
      - ippools
      - ipamblocks
      - globalnetworkpolicies
      - globalnetworksets
      - networkpolicies
      - clusterinformations
      - hostendpoints
    verbs:
      - get
      - list
      - watch
  # Calico must create and update some CRDs on startup.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
      - felixconfigurations
      - clusterinformations
    verbs:
      - create
      - update
  # Calico stores some configuration information on the node.
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - get
      - list
      - watch
  # These permissions are only requried for upgrade from v2.6, and can
  # be removed after upgrade or on fresh installations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - bgpconfigurations
      - bgppeers
    verbs:
      - create
      - update
  # These permissions are required for Calico CNI to perform IPAM allocations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
      - ipamblocks
      - ipamhandles
    verbs:
      - get
      - list
      - create
      - update
      - delete
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ipamconfigs
    verbs:
      - get
  # Block affinities must also be watchable by confd for route aggregation.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
    verbs:
      - watch
  # The Calico IPAM migration needs to get daemonsets. These permissions can be
  # removed if not upgrading from an installation using host-local IPAM.
  - apiGroups: ["apps"]
    resources:
      - daemonsets
    verbs:
      - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: calico-node
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-node
subjects:
- kind: ServiceAccount
  name: calico-node
  namespace: kube-system
---

---
# Source: calico/templates/calico-node.yaml
# This manifest installs the node container, as well
# as the Calico CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
  name: calico-node
  namespace: kube-system
  labels:
    k8s-app: calico-node
spec:
  selector:
    matchLabels:
      k8s-app: calico-node
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        k8s-app: calico-node
      annotations:
        # This, along with the CriticalAddonsOnly toleration below,
        # marks the pod as a critical add-on, ensuring it gets
        # priority scheduling and that its resources are reserved
        # if it ever gets evicted.
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      nodeSelector:
        beta.kubernetes.io/os: linux
      hostNetwork: true
      tolerations:
        # Make sure calico-node gets scheduled on all nodes.
        - effect: NoSchedule
          operator: Exists
        # Mark the pod as a critical add-on for rescheduling.
        - key: CriticalAddonsOnly
          operator: Exists
        - effect: NoExecute
          operator: Exists
      serviceAccountName: calico-node
      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
      initContainers:
        # This container performs upgrade from host-local IPAM to calico-ipam.
        # It can be deleted if this is a fresh installation, or if you have already
        # upgraded to use calico-ipam.
        - name: upgrade-ipam
          image: calico/cni:v3.6.0
          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
          env:
            - name: KUBERNETES_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: CALICO_NETWORKING_BACKEND
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: calico_backend
          volumeMounts:
            - mountPath: /var/lib/cni/networks
              name: host-local-net-dir
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
        # This container installs the Calico CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
          image: calico/cni:v3.6.0
          command: ["/install-cni.sh"]
          env:
            # Name of the CNI config file to create.
            - name: CNI_CONF_NAME
              value: "10-calico.conflist"
            # The CNI network config to install on each node.
            - name: CNI_NETWORK_CONFIG
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: cni_network_config
            # Set the hostname based on the k8s node name.
            - name: KUBERNETES_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # CNI MTU Config variable
            - name: CNI_MTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # Prevents the container from sleeping forever.
            - name: SLEEP
              value: "false"
          volumeMounts:
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
            - mountPath: /host/etc/cni/net.d
              name: cni-net-dir
      containers:
        # Runs node container on each Kubernetes node.  This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
          image: calico/node:v3.6.0
          env:
            # Use Kubernetes API as the backing datastore.
            - name: DATASTORE_TYPE
              value: "kubernetes"
            # Wait for the datastore.
            - name: WAIT_FOR_DATASTORE
              value: "true"
            # Set based on the k8s node name.
            - name: NODENAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # Choose the backend to use.
            - name: CALICO_NETWORKING_BACKEND
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: calico_backend
            # Cluster type to identify the deployment type
            - name: CLUSTER_TYPE
              value: "k8s,bgp"
            # Auto-detect the BGP IP address.
            - name: IP
              value: "autodetect"
            # Enable IPIP
            - name: CALICO_IPV4POOL_IPIP
              value: "Always"
            # Set MTU for tunnel device used if ipip is enabled
            - name: FELIX_IPINIPMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
            # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within `--cluster-cidr`.
            - name: CALICO_IPV4POOL_CIDR
              value: "192.168.0.0/16"
            # Disable file logging so `kubectl logs` works.
            - name: CALICO_DISABLE_FILE_LOGGING
              value: "true"
            # Set Felix endpoint to host default action to ACCEPT.
            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
              value: "ACCEPT"
            # Disable IPv6 on Kubernetes.
            - name: FELIX_IPV6SUPPORT
              value: "false"
            # Set Felix logging to "info"
            - name: FELIX_LOGSEVERITYSCREEN
              value: "info"
            - name: FELIX_HEALTHENABLED
              value: "true"
          securityContext:
            privileged: true
          resources:
            requests:
              cpu: 250m
          livenessProbe:
            httpGet:
              path: /liveness
              port: 9099
              host: localhost
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
            exec:
              command:
              - /bin/calico-node
              - -bird-ready
              - -felix-ready
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
              name: lib-modules
              readOnly: true
            - mountPath: /run/xtables.lock
              name: xtables-lock
              readOnly: false
            - mountPath: /var/run/calico
              name: var-run-calico
              readOnly: false
            - mountPath: /var/lib/calico
              name: var-lib-calico
              readOnly: false
      volumes:
        # Used by node.
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: var-run-calico
          hostPath:
            path: /var/run/calico
        - name: var-lib-calico
          hostPath:
            path: /var/lib/calico
        - name: xtables-lock
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
        # Used to install CNI.
        - name: cni-bin-dir
          hostPath:
            path: /opt/cni/bin
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
        # Mount in the directory for host-local IPAM allocations. This is
        # used when upgrading from host-local to calico-ipam, and can be removed
        # if not using the upgrade-ipam init container.
        - name: host-local-net-dir
          hostPath:
            path: /var/lib/cni/networks
---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-node
  namespace: kube-system

---
# Source: calico/templates/calico-kube-controllers.yaml
# This manifest deploys the Calico node controller.
# See https://github.com/projectcalico/kube-controllers
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: calico-kube-controllers
  namespace: kube-system
  labels:
    k8s-app: calico-kube-controllers
  annotations:
    scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
  # The controller can only have a single active instance.
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      name: calico-kube-controllers
      namespace: kube-system
      labels:
        k8s-app: calico-kube-controllers
    spec:
      nodeSelector:
        beta.kubernetes.io/os: linux
      tolerations:
        # Mark the pod as a critical add-on for rescheduling.
        - key: CriticalAddonsOnly
          operator: Exists
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      serviceAccountName: calico-kube-controllers
      containers:
        - name: calico-kube-controllers
          image: calico/kube-controllers:v3.6.0
          env:
            # Choose which controllers to run.
            - name: ENABLED_CONTROLLERS
              value: node
            - name: DATASTORE_TYPE
              value: kubernetes
          readinessProbe:
            exec:
              command:
              - /usr/bin/check-status
              - -r

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-kube-controllers
  namespace: kube-system
---
# Source: calico/templates/calico-etcd-secrets.yaml

---
# Source: calico/templates/calico-typha.yaml

---
# Source: calico/templates/configure-canal.yaml

apiVersion: "cluster.k8s.io/v1alpha1"
kind: Cluster
metadata:
  name: test1
spec:
  clusterNetwork:
    services:
      cidrBlocks: ["10.96.0.0/12"]
    pods:
      cidrBlocks: ["192.168.0.0/16"]
    serviceDomain: "cluster.local"
  providerSpec:
    value:
      apiVersion: "awsprovider/v1alpha1"
      kind: "AWSClusterProviderSpec"
      region: "us-east-2"
      sshKeyName: "default"
apiVersion: "cluster.k8s.io/v1alpha1"
kind: MachineDeployment
metadata:
  name: sample-machinedeployment
  labels:
    cluster.k8s.io/cluster-name: test1
spec:
  replicas: 1
  selector:
    matchLabels:
      cluster.k8s.io/cluster-name: test1
      set: node
  template:
    metadata:
      labels:
        cluster.k8s.io/cluster-name: test1
        set: node
    spec:
      versions:
        kubelet: v1.13.6
      providerSpec:
        value:
          apiVersion: awsprovider/v1alpha1
          kind: AWSMachineProviderSpec
          instanceType: "t2.medium"
          iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io"
          keyName: "default"
apiVersion: "cluster.k8s.io/v1alpha1"
kind: MachineList
items:
  - apiVersion: "cluster.k8s.io/v1alpha1"
    kind: Machine
    metadata:
      name: controlplane-0
      labels:
        cluster.k8s.io/cluster-name: test1
        set: controlplane
    spec:
      versions:
        kubelet: v1.13.6
        controlPlane: v1.13.6
      providerSpec:
        value:
          apiVersion: awsprovider/v1alpha1
          kind: AWSMachineProviderSpec
          instanceType: "t2.medium"
          iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io"
          keyName: "default"
  - apiVersion: "cluster.k8s.io/v1alpha1"
    kind: Machine
    metadata:
      name: controlplane-1
      labels:
        cluster.k8s.io/cluster-name: test1
        set: controlplane
    spec:
      versions:
        kubelet: v1.13.6
        controlPlane: v1.13.6
      providerSpec:
        value:
          apiVersion: awsprovider/v1alpha1
          kind: AWSMachineProviderSpec
          instanceType: "t2.medium"
          iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io"
          keyName: "default"
  - apiVersion: "cluster.k8s.io/v1alpha1"
    kind: Machine
    metadata:
      name: controlplane-2
      labels:
        cluster.k8s.io/cluster-name: test1
        set: controlplane
    spec:
      versions:
        kubelet: v1.13.6
        controlPlane: v1.13.6
      providerSpec:
        value:
          apiVersion: awsprovider/v1alpha1
          kind: AWSMachineProviderSpec
          instanceType: "t2.medium"
          iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io"
          keyName: "default"
apiVersion: "cluster.k8s.io/v1alpha1"
kind: MachineList
items:
  - apiVersion: "cluster.k8s.io/v1alpha1"
    kind: Machine
    metadata:
      name: controlplane-0
      labels:
        cluster.k8s.io/cluster-name: test1
        set: controlplane
    spec:
      versions:
        kubelet: v1.13.6
        controlPlane: v1.13.6
      providerSpec:
        value:
          apiVersion: awsprovider/v1alpha1
          kind: AWSMachineProviderSpec
          instanceType: "t2.medium"
          iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io"
          keyName: "default"
  - apiVersion: "cluster.k8s.io/v1alpha1"
    kind: Machine
    metadata:
      generateName: node-
      labels:
        cluster.k8s.io/cluster-name: test1
        set: node
    spec:
      versions:
        kubelet: v1.13.6
      providerSpec:
        value:
          apiVersion: awsprovider/v1alpha1
          kind: AWSMachineProviderSpec
          instanceType: "t2.medium"
          iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io"
          keyName: "default"
apiVersion: v1
kind: Namespace
metadata:
  labels:
    controller-tools.k8s.io: "1.0"
  name: aws-provider-system
---
apiVersion: v1
kind: Namespace
metadata:
  labels:
    controller-tools.k8s.io: "1.0"
  name: cluster-api-system
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: awsclusterproviderspecs.awsprovider.k8s.io
spec:
  group: awsprovider.k8s.io
  names:
    kind: AWSClusterProviderSpec
    plural: awsclusterproviderspecs
  scope: Namespaced
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        caKeyPair:
          description: CAKeyPair is the key pair for ca certs.
          properties:
            cert:
              description: base64 encoded cert and key
              format: byte
              type: string
            key:
              format: byte
              type: string
          required:
          - cert
          - key
          type: object
        clusterConfiguration:
          description: ClusterConfiguration holds the cluster-wide information used
            during a kubeadm init call.
          properties:
            apiServer:
              description: APIServer contains extra settings for the API server control
                plane component
              properties:
                certSANs:
                  description: CertSANs sets extra Subject Alternative Names for the
                    API Server signing cert.
                  items:
                    type: string
                  type: array
                extraArgs:
                  description: 'ExtraArgs is an extra set of flags to pass to the
                    control plane component. TODO: This is temporary and ideally we
                    would like to switch all components to use ComponentConfig + ConfigMaps.'
                  type: object
                extraVolumes:
                  description: ExtraVolumes is an extra set of host volumes, mounted
                    to the control plane component.
                  items:
                    properties:
                      hostPath:
                        description: HostPath is the path in the host that will be
                          mounted inside the pod.
                        type: string
                      mountPath:
                        description: MountPath is the path inside the pod where hostPath
                          will be mounted.
                        type: string
                      name:
                        description: Name of the volume inside the pod template.
                        type: string
                      pathType:
                        description: PathType is the type of the HostPath.
                        type: string
                      readOnly:
                        description: ReadOnly controls write access to the volume
                        type: boolean
                    required:
                    - name
                    - hostPath
                    - mountPath
                    type: object
                  type: array
                timeoutForControlPlane:
                  description: TimeoutForControlPlane controls the timeout that we
                    use for API server to appear
                  type: object
              type: object
            apiVersion:
              description: 'APIVersion defines the versioned schema of this representation
                of an object. Servers should convert recognized schemas to the latest
                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
              type: string
            certificatesDir:
              description: CertificatesDir specifies where to store or look for all
                required certificates.
              type: string
            clusterName:
              description: The cluster name
              type: string
            controlPlaneEndpoint:
              description: 'ControlPlaneEndpoint sets a stable IP address or DNS name
                for the control plane; it can be a valid IP address or a RFC-1123
                DNS subdomain, both with optional TCP port. In case the ControlPlaneEndpoint
                is not specified, the AdvertiseAddress + BindPort are used; in case
                the ControlPlaneEndpoint is specified but without a TCP port, the
                BindPort is used. Possible usages are: e.g. In a cluster with more
                than one control plane instances, this field should be assigned the
                address of the external load balancer in front of the control plane
                instances. e.g.  in environments with enforced node recycling, the
                ControlPlaneEndpoint could be used for assigning a stable DNS to the
                control plane.'
              type: string
            controllerManager:
              description: ControllerManager contains extra settings for the controller
                manager control plane component
              properties:
                extraArgs:
                  description: 'ExtraArgs is an extra set of flags to pass to the
                    control plane component. TODO: This is temporary and ideally we
                    would like to switch all components to use ComponentConfig + ConfigMaps.'
                  type: object
                extraVolumes:
                  description: ExtraVolumes is an extra set of host volumes, mounted
                    to the control plane component.
                  items:
                    properties:
                      hostPath:
                        description: HostPath is the path in the host that will be
                          mounted inside the pod.
                        type: string
                      mountPath:
                        description: MountPath is the path inside the pod where hostPath
                          will be mounted.
                        type: string
                      name:
                        description: Name of the volume inside the pod template.
                        type: string
                      pathType:
                        description: PathType is the type of the HostPath.
                        type: string
                      readOnly:
                        description: ReadOnly controls write access to the volume
                        type: boolean
                    required:
                    - name
                    - hostPath
                    - mountPath
                    type: object
                  type: array
              type: object
            dns:
              description: DNS defines the options for the DNS add-on installed in
                the cluster.
              properties:
                imageRepository:
                  description: ImageRepository sets the container registry to pull
                    images from. if not set, the ImageRepository defined in ClusterConfiguration
                    will be used instead.
                  type: string
                imageTag:
                  description: ImageTag allows to specify a tag for the image. In
                    case this value is set, kubeadm does not change automatically
                    the version of the above components during upgrades.
                  type: string
                type:
                  description: Type defines the DNS add-on to be used
                  type: string
              required:
              - type
              type: object
            etcd:
              description: Etcd holds configuration for etcd.
              properties:
                external:
                  description: External describes how to connect to an external etcd
                    cluster Local and External are mutually exclusive
                  properties:
                    caFile:
                      description: CAFile is an SSL Certificate Authority file used
                        to secure etcd communication. Required if using a TLS connection.
                      type: string
                    certFile:
                      description: CertFile is an SSL certification file used to secure
                        etcd communication. Required if using a TLS connection.
                      type: string
                    endpoints:
                      description: Endpoints of etcd members. Required for ExternalEtcd.
                      items:
                        type: string
                      type: array
                    keyFile:
                      description: KeyFile is an SSL key file used to secure etcd
                        communication. Required if using a TLS connection.
                      type: string
                  required:
                  - endpoints
                  - caFile
                  - certFile
                  - keyFile
                  type: object
                local:
                  description: Local provides configuration knobs for configuring
                    the local etcd instance Local and External are mutually exclusive
                  properties:
                    dataDir:
                      description: DataDir is the directory etcd will place its data.
                        Defaults to "/var/lib/etcd".
                      type: string
                    extraArgs:
                      description: ExtraArgs are extra arguments provided to the etcd
                        binary when run inside a static pod.
                      type: object
                    imageRepository:
                      description: ImageRepository sets the container registry to
                        pull images from. if not set, the ImageRepository defined
                        in ClusterConfiguration will be used instead.
                      type: string
                    imageTag:
                      description: ImageTag allows to specify a tag for the image.
                        In case this value is set, kubeadm does not change automatically
                        the version of the above components during upgrades.
                      type: string
                    peerCertSANs:
                      description: PeerCertSANs sets extra Subject Alternative Names
                        for the etcd peer signing cert.
                      items:
                        type: string
                      type: array
                    serverCertSANs:
                      description: ServerCertSANs sets extra Subject Alternative Names
                        for the etcd server signing cert.
                      items:
                        type: string
                      type: array
                  required:
                  - dataDir
                  type: object
              type: object
            featureGates:
              description: FeatureGates enabled by the user.
              type: object
            imageRepository:
              description: ImageRepository sets the container registry to pull images
                from. If empty, `k8s.gcr.io` will be used by default; in case of kubernetes
                version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`)
                `gcr.io/kubernetes-ci-images` will be used as a default for control
                plane components and for kube-proxy, while `k8s.gcr.io` will be used
                for all the other images.
              type: string
            kind:
              description: 'Kind is a string value representing the REST resource
                this object represents. Servers may infer this from the endpoint the
                client submits requests to. Cannot be updated. In CamelCase. More
                info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
              type: string
            kubernetesVersion:
              description: KubernetesVersion is the target version of the control
                plane.
              type: string
            networking:
              description: Networking holds configuration for the networking topology
                of the cluster.
              properties:
                dnsDomain:
                  description: DNSDomain is the dns domain used by k8s services. Defaults
                    to "cluster.local".
                  type: string
                podSubnet:
                  description: PodSubnet is the subnet used by pods.
                  type: string
                serviceSubnet:
                  description: ServiceSubnet is the subnet used by k8s services. Defaults
                    to "10.96.0.0/12".
                  type: string
              required:
              - serviceSubnet
              - podSubnet
              - dnsDomain
              type: object
            scheduler:
              description: Scheduler contains extra settings for the scheduler control
                plane component
              properties:
                extraArgs:
                  description: 'ExtraArgs is an extra set of flags to pass to the
                    control plane component. TODO: This is temporary and ideally we
                    would like to switch all components to use ComponentConfig + ConfigMaps.'
                  type: object
                extraVolumes:
                  description: ExtraVolumes is an extra set of host volumes, mounted
                    to the control plane component.
                  items:
                    properties:
                      hostPath:
                        description: HostPath is the path in the host that will be
                          mounted inside the pod.
                        type: string
                      mountPath:
                        description: MountPath is the path inside the pod where hostPath
                          will be mounted.
                        type: string
                      name:
                        description: Name of the volume inside the pod template.
                        type: string
                      pathType:
                        description: PathType is the type of the HostPath.
                        type: string
                      readOnly:
                        description: ReadOnly controls write access to the volume
                        type: boolean
                    required:
                    - name
                    - hostPath
                    - mountPath
                    type: object
                  type: array
              type: object
            useHyperKubeImage:
              description: UseHyperKubeImage controls if hyperkube should be used
                for Kubernetes components instead of their respective separate images
              type: boolean
          required:
          - etcd
          - networking
          - kubernetesVersion
          - controlPlaneEndpoint
          - dns
          - certificatesDir
          - imageRepository
          type: object
        etcdCAKeyPair:
          description: EtcdCAKeyPair is the key pair for etcd.
          properties:
            cert:
              description: base64 encoded cert and key
              format: byte
              type: string
            key:
              format: byte
              type: string
          required:
          - cert
          - key
          type: object
        frontProxyCAKeyPair:
          description: FrontProxyCAKeyPair is the key pair for FrontProxyKeyPair.
          properties:
            cert:
              description: base64 encoded cert and key
              format: byte
              type: string
            key:
              format: byte
              type: string
          required:
          - cert
          - key
          type: object
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        networkSpec:
          description: NetworkSpec encapsulates all things related to AWS network.
          properties:
            subnets:
              description: Subnets configuration.
              items:
                properties:
                  availabilityZone:
                    description: AvailabilityZone defines the availability zone to
                      use for this subnet in the cluster's region.
                    type: string
                  cidrBlock:
                    description: CidrBlock is the CIDR block to be used when the provider
                      creates a managed VPC.
                    type: string
                  id:
                    description: ID defines a unique identifier to reference this
                      resource.
                    type: string
                  isPublic:
                    description: IsPublic defines the subnet as a public subnet. A
                      subnet is public when it is associated with a route table that
                      has a route to an internet gateway.
                    type: boolean
                  natGatewayId:
                    description: NatGatewayID is the NAT gateway id associated with
                      the subnet. Ignored unless the subnet is managed by the provider,
                      in which case this is set on the public subnet where the NAT
                      gateway resides. It is then used to determine routes for private
                      subnets in the same AZ as the public subnet.
                    type: string
                  routeTableId:
                    description: RouteTableID is the routing table id associated with
                      the subnet.
                    type: string
                  tags:
                    description: Tags is a collection of tags describing the resource.
                    type: object
                required:
                - isPublic
                - routeTableId
                type: object
              type: array
            vpc:
              description: VPC configuration.
              properties:
                cidrBlock:
                  description: CidrBlock is the CIDR block to be used when the provider
                    creates a managed VPC. Defaults to 10.0.0.0/16.
                  type: string
                id:
                  description: ID is the vpc-id of the VPC this provider should use
                    to create resources.
                  type: string
                internetGatewayId:
                  description: InternetGatewayID is the id of the internet gateway
                    associated with the VPC.
                  type: string
                tags:
                  description: Tags is a collection of tags describing the resource.
                  type: object
              type: object
          type: object
        region:
          description: The AWS Region the cluster lives in.
          type: string
        saKeyPair:
          description: SAKeyPair is the service account key pair.
          properties:
            cert:
              description: base64 encoded cert and key
              format: byte
              type: string
            key:
              format: byte
              type: string
          required:
          - cert
          - key
          type: object
        sshKeyName:
          description: SSHKeyName is the name of the ssh key to attach to the bastion
            host.
          type: string
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: awsclusterproviderstatuses.awsprovider.k8s.io
spec:
  group: awsprovider.k8s.io
  names:
    kind: AWSClusterProviderStatus
    plural: awsclusterproviderstatuses
  scope: Namespaced
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        bastion:
          properties:
            ebsOptimized:
              description: Indicates whether the instance is optimized for Amazon
                EBS I/O.
              type: boolean
            enaSupport:
              description: Specifies whether enhanced networking with ENA is enabled.
              type: boolean
            iamProfile:
              description: The name of the IAM instance profile associated with the
                instance, if applicable.
              type: string
            id:
              type: string
            imageId:
              description: The ID of the AMI used to launch the instance.
              type: string
            instanceState:
              description: The current state of the instance.
              type: string
            keyName:
              description: The name of the SSH key pair.
              type: string
            privateIp:
              description: The private IPv4 address assigned to the instance.
              type: string
            publicIp:
              description: The public IPv4 address assigned to the instance, if applicable.
              type: string
            rootDeviceSize:
              description: Specifies size (in Gi) of the root storage device
              format: int64
              type: integer
            securityGroupIds:
              description: SecurityGroupIDs are one or more security group IDs this
                instance belongs to.
              items:
                type: string
              type: array
            subnetId:
              description: The ID of the subnet of the instance.
              type: string
            tags:
              description: The tags associated with the instance.
              type: object
            type:
              description: The instance type.
              type: string
            userData:
              description: UserData is the raw data script passed to the instance
                which is run upon bootstrap. This field must not be base64 encoded
                and should only be used when running a new instance.
              type: string
          required:
          - id
          type: object
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        network:
          properties:
            apiServerElb:
              description: APIServerELB is the Kubernetes api server classic load
                balancer.
              properties:
                attributes:
                  description: Attributes defines extra attributes associated with
                    the load balancer.
                  properties:
                    idleTimeout:
                      description: IdleTimeout is time that the connection is allowed
                        to be idle (no data has been sent over the connection) before
                        it is closed by the load balancer.
                      format: int64
                      type: integer
                  type: object
                dnsName:
                  description: DNSName is the dns name of the load balancer.
                  type: string
                healthChecks:
                  description: HealthCheck is the classic elb health check associated
                    with the load balancer.
                  properties:
                    healthyThreshold:
                      format: int64
                      type: integer
                    interval:
                      format: int64
                      type: integer
                    target:
                      type: string
                    timeout:
                      format: int64
                      type: integer
                    unhealthyThreshold:
                      format: int64
                      type: integer
                  required:
                  - target
                  - interval
                  - timeout
                  - healthyThreshold
                  - unhealthyThreshold
                  type: object
                listeners:
                  description: Listeners is an array of classic elb listeners associated
                    with the load balancer. There must be at least one.
                  items:
                    properties:
                      instancePort:
                        format: int64
                        type: integer
                      instanceProtocol:
                        type: string
                      port:
                        format: int64
                        type: integer
                      protocol:
                        type: string
                    required:
                    - protocol
                    - port
                    - instanceProtocol
                    - instancePort
                    type: object
                  type: array
                name:
                  description: The name of the load balancer. It must be unique within
                    the set of load balancers defined in the region. It also serves
                    as identifier.
                  type: string
                scheme:
                  description: Scheme is the load balancer scheme, either internet-facing
                    or private.
                  type: string
                securityGroupIds:
                  description: SecurityGroupIDs is an array of security groups assigned
                    to the load balancer.
                  items:
                    type: string
                  type: array
                subnetIds:
                  description: SubnetIDs is an array of subnets in the VPC attached
                    to the load balancer.
                  items:
                    type: string
                  type: array
                tags:
                  description: Tags is a map of tags associated with the load balancer.
                  type: object
              type: object
            securityGroups:
              description: SecurityGroups is a map from the role/kind of the security
                group to its unique name, if any.
              type: object
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: awsmachineproviderspecs.awsprovider.k8s.io
spec:
  group: awsprovider.k8s.io
  names:
    kind: AWSMachineProviderSpec
    plural: awsmachineproviderspecs
  scope: Namespaced
  validation:
    openAPIV3Schema:
      properties:
        additionalSecurityGroups:
          description: AdditionalSecurityGroups is an array of references to security
            groups that should be applied to the instance. These security groups would
            be set in addition to any security groups defined at the cluster level
            or in the actuator.
          items:
            properties:
              arn:
                description: ARN of resource
                type: string
              filters:
                description: 'Filters is a set of key/value pairs used to identify
                  a resource They are applied according to the rules defined by the
                  AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html'
                items:
                  properties:
                    name:
                      description: Name of the filter. Filter names are case-sensitive.
                      type: string
                    values:
                      description: Values includes one or more filter values. Filter
                        values are case-sensitive.
                      items:
                        type: string
                      type: array
                  required:
                  - name
                  - values
                  type: object
                type: array
              id:
                description: ID of resource
                type: string
            type: object
          type: array
        additionalTags:
          description: AdditionalTags is the set of tags to add to an instance, in
            addition to the ones added by default by the actuator. These tags are
            additive. The actuator will ensure these tags are present, but will not
            remove any other tags that may exist on the instance.
          type: object
        ami:
          description: AMI is the reference to the AMI from which to create the machine
            instance.
          properties:
            arn:
              description: ARN of resource
              type: string
            filters:
              description: 'Filters is a set of key/value pairs used to identify a
                resource They are applied according to the rules defined by the AWS
                API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html'
              items:
                properties:
                  name:
                    description: Name of the filter. Filter names are case-sensitive.
                    type: string
                  values:
                    description: Values includes one or more filter values. Filter
                      values are case-sensitive.
                    items:
                      type: string
                    type: array
                required:
                - name
                - values
                type: object
              type: array
            id:
              description: ID of resource
              type: string
          type: object
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        iamInstanceProfile:
          description: IAMInstanceProfile is a name of an IAM instance profile to
            assign to the instance
          type: string
        imageLookupOrg:
          description: ImageLookupOrg is the AWS Organization ID to use for image
            lookup if AMI is not set.
          type: string
        instanceType:
          description: 'InstanceType is the type of instance to create. Example: m4.xlarge'
          type: string
        keyName:
          description: KeyName is the name of the SSH key to install on the instance.
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        kubeadmConfiguration:
          description: KubeadmConfiguration holds the kubeadm configuration options
          properties:
            init:
              description: InitConfiguration is used to customize any kubeadm init
                configuration parameters.
              properties:
                apiVersion:
                  description: 'APIVersion defines the versioned schema of this representation
                    of an object. Servers should convert recognized schemas to the
                    latest internal value, and may reject unrecognized values. More
                    info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
                  type: string
                bootstrapTokens:
                  description: BootstrapTokens is respected at `kubeadm init` time
                    and describes a set of Bootstrap Tokens to create. This information
                    IS NOT uploaded to the kubeadm cluster configmap, partly because
                    of its sensitive nature
                  items:
                    properties:
                      description:
                        description: Description sets a human-friendly message why
                          this token exists and what it's used for, so other administrators
                          can know its purpose.
                        type: string
                      expires:
                        description: Expires specifies the timestamp when this token
                          expires. Defaults to being set dynamically at runtime based
                          on the TTL. Expires and TTL are mutually exclusive.
                        format: date-time
                        type: string
                      groups:
                        description: Groups specifies the extra groups that this token
                          will authenticate as when/if used for authentication
                        items:
                          type: string
                        type: array
                      token:
                        description: Token is used for establishing bidirectional
                          trust between nodes and masters. Used for joining nodes
                          in the cluster.
                        type: object
                      ttl:
                        description: TTL defines the time to live for this token.
                          Defaults to 24h. Expires and TTL are mutually exclusive.
                        type: object
                      usages:
                        description: Usages describes the ways in which this token
                          can be used. Can by default be used for establishing bidirectional
                          trust, but that can be changed here.
                        items:
                          type: string
                        type: array
                    required:
                    - token
                    type: object
                  type: array
                kind:
                  description: 'Kind is a string value representing the REST resource
                    this object represents. Servers may infer this from the endpoint
                    the client submits requests to. Cannot be updated. In CamelCase.
                    More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
                  type: string
                localAPIEndpoint:
                  description: LocalAPIEndpoint represents the endpoint of the API
                    server instance that's deployed on this control plane node In
                    HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint
                    in the sense that ControlPlaneEndpoint is the global endpoint
                    for the cluster, which then loadbalances the requests to each
                    individual API server. This configuration object lets you customize
                    what IP/DNS name and port the local API server advertises it's
                    accessible on. By default, kubeadm tries to auto-detect the IP
                    of the default interface and use that, but in case that process
                    fails you may set the desired value here.
                  properties:
                    advertiseAddress:
                      description: AdvertiseAddress sets the IP address for the API
                        server to advertise.
                      type: string
                    bindPort:
                      description: BindPort sets the secure port for the API Server
                        to bind to. Defaults to 6443.
                      format: int32
                      type: integer
                  required:
                  - advertiseAddress
                  - bindPort
                  type: object
                nodeRegistration:
                  description: NodeRegistration holds fields that relate to registering
                    the new master node to the cluster
                  properties:
                    criSocket:
                      description: CRISocket is used to retrieve container runtime
                        info. This information will be annotated to the Node API object,
                        for later re-use
                      type: string
                    kubeletExtraArgs:
                      description: KubeletExtraArgs passes through extra arguments
                        to the kubelet. The arguments here are passed to the kubelet
                        command line via the environment file kubeadm writes at runtime
                        for the kubelet to source. This overrides the generic base-level
                        configuration in the kubelet-config-1.X ConfigMap Flags have
                        higher priority when parsing. These values are local and specific
                        to the node kubeadm is executing on.
                      type: object
                    name:
                      description: Name is the `.Metadata.Name` field of the Node
                        API object that will be created in this `kubeadm init` or
                        `kubeadm joiń` operation. This field is also used in the CommonName
                        field of the kubelet's client certificate to the API server.
                        Defaults to the hostname of the node if not provided.
                      type: string
                    taints:
                      description: 'Taints specifies the taints the Node API object
                        should be registered with. If this field is unset, i.e. nil,
                        in the `kubeadm init` process it will be defaulted to []v1.Taint{''node-role.kubernetes.io/master=""''}.
                        If you don''t want to taint your master node, set this field
                        to an empty slice, i.e. `taints: {}` in the YAML file. This
                        field is solely used for Node registration.'
                      items:
                        type: object
                      type: array
                  type: object
              type: object
            join:
              description: JoinConfiguration is used to customize any kubeadm join
                configuration parameters.
              properties:
                apiVersion:
                  description: 'APIVersion defines the versioned schema of this representation
                    of an object. Servers should convert recognized schemas to the
                    latest internal value, and may reject unrecognized values. More
                    info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
                  type: string
                caCertPath:
                  description: CACertPath is the path to the SSL certificate authority
                    used to secure comunications between node and master. Defaults
                    to "/etc/kubernetes/pki/ca.crt".
                  type: string
                controlPlane:
                  description: ControlPlane defines the additional control plane instance
                    to be deployed on the joining node. If nil, no additional control
                    plane instance will be deployed.
                  properties:
                    localAPIEndpoint:
                      description: LocalAPIEndpoint represents the endpoint of the
                        API server instance to be deployed on this node.
                      properties:
                        advertiseAddress:
                          description: AdvertiseAddress sets the IP address for the
                            API server to advertise.
                          type: string
                        bindPort:
                          description: BindPort sets the secure port for the API Server
                            to bind to. Defaults to 6443.
                          format: int32
                          type: integer
                      required:
                      - advertiseAddress
                      - bindPort
                      type: object
                  type: object
                discovery:
                  description: Discovery specifies the options for the kubelet to
                    use during the TLS Bootstrap process
                  properties:
                    bootstrapToken:
                      description: BootstrapToken is used to set the options for bootstrap
                        token based discovery BootstrapToken and File are mutually
                        exclusive
                      properties:
                        apiServerEndpoint:
                          description: APIServerEndpoint is an IP or domain name to
                            the API server from which info will be fetched.
                          type: string
                        caCertHashes:
                          description: 'CACertHashes specifies a set of public key
                            pins to verify when token-based discovery is used. The
                            root CA found during discovery must match one of these
                            values. Specifying an empty set disables root CA pinning,
                            which can be unsafe. Each hash is specified as "<type>:<value>",
                            where the only currently supported type is "sha256". This
                            is a hex-encoded SHA-256 hash of the Subject Public Key
                            Info (SPKI) object in DER-encoded ASN.1. These hashes
                            can be calculated using, for example, OpenSSL: openssl
                            x509 -pubkey -in ca.crt openssl rsa -pubin -outform der
                            2>&/dev/null | openssl dgst -sha256 -hex'
                          items:
                            type: string
                          type: array
                        token:
                          description: Token is a token used to validate cluster information
                            fetched from the master.
                          type: string
                        unsafeSkipCAVerification:
                          description: UnsafeSkipCAVerification allows token-based
                            discovery without CA verification via CACertHashes. This
                            can weaken the security of kubeadm since other nodes can
                            impersonate the master.
                          type: boolean
                      required:
                      - token
                      - unsafeSkipCAVerification
                      type: object
                    file:
                      description: File is used to specify a file or URL to a kubeconfig
                        file from which to load cluster information BootstrapToken
                        and File are mutually exclusive
                      properties:
                        kubeConfigPath:
                          description: KubeConfigPath is used to specify the actual
                            file path or URL to the kubeconfig file from which to
                            load cluster information
                          type: string
                      required:
                      - kubeConfigPath
                      type: object
                    timeout:
                      description: Timeout modifies the discovery timeout
                      type: object
                    tlsBootstrapToken:
                      description: TLSBootstrapToken is a token used for TLS bootstrapping.
                        If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token,
                        but can be overridden. If .File is set, this field **must
                        be set** in case the KubeConfigFile does not contain any other
                        authentication information
                      type: string
                  required:
                  - tlsBootstrapToken
                  type: object
                kind:
                  description: 'Kind is a string value representing the REST resource
                    this object represents. Servers may infer this from the endpoint
                    the client submits requests to. Cannot be updated. In CamelCase.
                    More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
                  type: string
                nodeRegistration:
                  description: NodeRegistration holds fields that relate to registering
                    the new master node to the cluster
                  properties:
                    criSocket:
                      description: CRISocket is used to retrieve container runtime
                        info. This information will be annotated to the Node API object,
                        for later re-use
                      type: string
                    kubeletExtraArgs:
                      description: KubeletExtraArgs passes through extra arguments
                        to the kubelet. The arguments here are passed to the kubelet
                        command line via the environment file kubeadm writes at runtime
                        for the kubelet to source. This overrides the generic base-level
                        configuration in the kubelet-config-1.X ConfigMap Flags have
                        higher priority when parsing. These values are local and specific
                        to the node kubeadm is executing on.
                      type: object
                    name:
                      description: Name is the `.Metadata.Name` field of the Node
                        API object that will be created in this `kubeadm init` or
                        `kubeadm joiń` operation. This field is also used in the CommonName
                        field of the kubelet's client certificate to the API server.
                        Defaults to the hostname of the node if not provided.
                      type: string
                    taints:
                      description: 'Taints specifies the taints the Node API object
                        should be registered with. If this field is unset, i.e. nil,
                        in the `kubeadm init` process it will be defaulted to []v1.Taint{''node-role.kubernetes.io/master=""''}.
                        If you don''t want to taint your master node, set this field
                        to an empty slice, i.e. `taints: {}` in the YAML file. This
                        field is solely used for Node registration.'
                      items:
                        type: object
                      type: array
                  type: object
              required:
              - nodeRegistration
              - caCertPath
              - discovery
              type: object
          type: object
        metadata:
          type: object
        publicIP:
          description: 'PublicIP specifies whether the instance should get a public
            IP. Precedence for this setting is as follows: 1. This field if set 2.
            Cluster/flavor setting 3. Subnet default'
          type: boolean
        rootDeviceSize:
          description: RootDeviceSize is the size of the root volume.
          format: int64
          type: integer
        subnet:
          description: Subnet is a reference to the subnet to use for this instance.
            If not specified, the cluster subnet will be used.
          properties:
            arn:
              description: ARN of resource
              type: string
            filters:
              description: 'Filters is a set of key/value pairs used to identify a
                resource They are applied according to the rules defined by the AWS
                API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html'
              items:
                properties:
                  name:
                    description: Name of the filter. Filter names are case-sensitive.
                    type: string
                  values:
                    description: Values includes one or more filter values. Filter
                      values are case-sensitive.
                    items:
                      type: string
                    type: array
                required:
                - name
                - values
                type: object
              type: array
            id:
              description: ID of resource
              type: string
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: awsmachineproviderstatuses.awsprovider.k8s.io
spec:
  group: awsprovider.k8s.io
  names:
    kind: AWSMachineProviderStatus
    plural: awsmachineproviderstatuses
  scope: Namespaced
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        conditions:
          description: Conditions is a set of conditions associated with the Machine
            to indicate errors or other status
          items:
            properties:
              lastProbeTime:
                description: LastProbeTime is the last time we probed the condition.
                format: date-time
                type: string
              lastTransitionTime:
                description: LastTransitionTime is the last time the condition transitioned
                  from one status to another.
                format: date-time
                type: string
              message:
                description: Message is a human-readable message indicating details
                  about last transition.
                type: string
              reason:
                description: Reason is a unique, one-word, CamelCase reason for the
                  condition's last transition.
                type: string
              status:
                description: Status is the status of the condition.
                type: string
              type:
                description: Type is the type of the condition.
                type: string
            required:
            - type
            - status
            - lastProbeTime
            - lastTransitionTime
            - reason
            - message
            type: object
          type: array
        instanceID:
          description: InstanceID is the instance ID of the machine created in AWS
          type: string
        instanceState:
          description: InstanceState is the state of the AWS instance for this machine
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: clusters.cluster.k8s.io
spec:
  group: cluster.k8s.io
  names:
    kind: Cluster
    plural: clusters
  scope: Namespaced
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        spec:
          properties:
            clusterNetwork:
              description: Cluster network configuration
              properties:
                pods:
                  description: The network ranges from which Pod networks are allocated.
                  properties:
                    cidrBlocks:
                      items:
                        type: string
                      type: array
                  required:
                  - cidrBlocks
                  type: object
                serviceDomain:
                  description: Domain name for services.
                  type: string
                services:
                  description: The network ranges from which service VIPs are allocated.
                  properties:
                    cidrBlocks:
                      items:
                        type: string
                      type: array
                  required:
                  - cidrBlocks
                  type: object
              required:
              - services
              - pods
              - serviceDomain
              type: object
            providerSpec:
              description: Provider-specific serialized configuration to use during
                cluster creation. It is recommended that providers maintain their
                own versioned API types that should be serialized/deserialized from
                this field.
              properties:
                value:
                  description: Value is an inlined, serialized representation of the
                    resource configuration. It is recommended that providers maintain
                    their own versioned API types that should be serialized/deserialized
                    from this field, akin to component config.
                  type: object
                valueFrom:
                  description: Source for the provider configuration. Cannot be used
                    if value is not empty.
                  properties:
                    machineClass:
                      description: The machine class from which the provider config
                        should be sourced.
                      properties:
                        provider:
                          description: Provider is the name of the cloud-provider
                            which MachineClass is intended for.
                          type: string
                      type: object
                  type: object
              type: object
          required:
          - clusterNetwork
          type: object
        status:
          properties:
            apiEndpoints:
              description: APIEndpoint represents the endpoint to communicate with
                the IP.
              items:
                properties:
                  host:
                    description: The hostname on which the API server is serving.
                    type: string
                  port:
                    description: The port on which the API server is serving.
                    format: int64
                    type: integer
                required:
                - host
                - port
                type: object
              type: array
            errorMessage:
              description: If set, indicates that there is a problem reconciling the
                state, and will be set to a descriptive error message.
              type: string
            errorReason:
              description: If set, indicates that there is a problem reconciling the
                state, and will be set to a token value suitable for programmatic
                interpretation.
              type: string
            providerStatus:
              description: Provider-specific status. It is recommended that providers
                maintain their own versioned API types that should be serialized/deserialized
                from this field.
              type: object
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: machineclasses.cluster.k8s.io
spec:
  group: cluster.k8s.io
  names:
    kind: MachineClass
    plural: machineclasses
  scope: Namespaced
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        providerSpec:
          description: Provider-specific configuration to use during node creation.
          type: object
      required:
      - providerSpec
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: machinedeployments.cluster.k8s.io
spec:
  group: cluster.k8s.io
  names:
    kind: MachineDeployment
    plural: machinedeployments
  scope: Namespaced
  subresources:
    scale:
      labelSelectorPath: .status.labelSelector
      specReplicasPath: .spec.replicas
      statusReplicasPath: .status.replicas
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        spec:
          properties:
            minReadySeconds:
              description: Minimum number of seconds for which a newly created machine
                should be ready. Defaults to 0 (machine will be considered available
                as soon as it is ready)
              format: int32
              type: integer
            paused:
              description: Indicates that the deployment is paused.
              type: boolean
            progressDeadlineSeconds:
              description: The maximum time in seconds for a deployment to make progress
                before it is considered to be failed. The deployment controller will
                continue to process failed deployments and a condition with a ProgressDeadlineExceeded
                reason will be surfaced in the deployment status. Note that progress
                will not be estimated during the time a deployment is paused. Defaults
                to 600s.
              format: int32
              type: integer
            replicas:
              description: Number of desired machines. Defaults to 1. This is a pointer
                to distinguish between explicit zero and not specified.
              format: int32
              type: integer
            revisionHistoryLimit:
              description: The number of old MachineSets to retain to allow rollback.
                This is a pointer to distinguish between explicit zero and not specified.
                Defaults to 1.
              format: int32
              type: integer
            selector:
              description: Label selector for machines. Existing MachineSets whose
                machines are selected by this will be the ones affected by this deployment.
                It must match the machine template's labels.
              type: object
            strategy:
              description: The deployment strategy to use to replace existing machines
                with new ones.
              properties:
                rollingUpdate:
                  description: Rolling update config params. Present only if MachineDeploymentStrategyType
                    = RollingUpdate.
                  properties:
                    maxSurge:
                      description: 'The maximum number of machines that can be scheduled
                        above the desired number of machines. Value can be an absolute
                        number (ex: 5) or a percentage of desired machines (ex: 10%).
                        This can not be 0 if MaxUnavailable is 0. Absolute number
                        is calculated from percentage by rounding up. Defaults to
                        1. Example: when this is set to 30%, the new MachineSet can
                        be scaled up immediately when the rolling update starts, such
                        that the total number of old and new machines do not exceed
                        130% of desired machines. Once old machines have been killed,
                        new MachineSet can be scaled up further, ensuring that total
                        number of machines running at any time during the update is
                        at most 130% of desired machines.'
                      oneOf:
                      - type: string
                      - type: integer
                    maxUnavailable:
                      description: 'The maximum number of machines that can be unavailable
                        during the update. Value can be an absolute number (ex: 5)
                        or a percentage of desired machines (ex: 10%). Absolute number
                        is calculated from percentage by rounding down. This can not
                        be 0 if MaxSurge is 0. Defaults to 0. Example: when this is
                        set to 30%, the old MachineSet can be scaled down to 70% of
                        desired machines immediately when the rolling update starts.
                        Once new machines are ready, old MachineSet can be scaled
                        down further, followed by scaling up the new MachineSet, ensuring
                        that the total number of machines available at all times during
                        the update is at least 70% of desired machines.'
                      oneOf:
                      - type: string
                      - type: integer
                  type: object
                type:
                  description: Type of deployment. Currently the only supported strategy
                    is "RollingUpdate". Default is RollingUpdate.
                  type: string
              type: object
            template:
              description: Template describes the machines that will be created.
              properties:
                metadata:
                  description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata'
                  type: object
                spec:
                  description: 'Specification of the desired behavior of the machine.
                    More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status'
                  properties:
                    configSource:
                      description: ConfigSource is used to populate in the associated
                        Node for dynamic kubelet config. This field already exists
                        in Node, so any updates to it in the Machine spec will be
                        automatically copied to the linked NodeRef from the status.
                        The rest of dynamic kubelet config support should then work
                        as-is.
                      type: object
                    metadata:
                      description: ObjectMeta will autopopulate the Node created.
                        Use this to indicate what labels, annotations, name prefix,
                        etc., should be used when creating the Node.
                      type: object
                    providerID:
                      description: ProviderID is the identification ID of the machine
                        provided by the provider. This field must match the provider
                        ID as seen on the node object corresponding to this machine.
                        This field is required by higher level consumers of cluster-api.
                        Example use case is cluster autoscaler with cluster-api as
                        provider. Clean-up logic in the autoscaler compares machines
                        to nodes to find out machines at provider which could not
                        get registered as Kubernetes nodes. With cluster-api as a
                        generic out-of-tree provider for autoscaler, this field is
                        required by autoscaler to be able to have a provider view
                        of the list of machines. Another list of nodes is queried
                        from the k8s apiserver and then a comparison is done to find
                        out unregistered machines and are marked for delete. This
                        field will be set by the actuators and consumed by higher
                        level entities like autoscaler that will be interfacing with
                        cluster-api as generic provider.
                      type: string
                    providerSpec:
                      description: ProviderSpec details Provider-specific configuration
                        to use during node creation.
                      properties:
                        value:
                          description: Value is an inlined, serialized representation
                            of the resource configuration. It is recommended that
                            providers maintain their own versioned API types that
                            should be serialized/deserialized from this field, akin
                            to component config.
                          type: object
                        valueFrom:
                          description: Source for the provider configuration. Cannot
                            be used if value is not empty.
                          properties:
                            machineClass:
                              description: The machine class from which the provider
                                config should be sourced.
                              properties:
                                provider:
                                  description: Provider is the name of the cloud-provider
                                    which MachineClass is intended for.
                                  type: string
                              type: object
                          type: object
                      type: object
                    taints:
                      description: The list of the taints to be applied to the corresponding
                        Node in additive manner. This list will not overwrite any
                        other taints added to the Node on an ongoing basis by other
                        entities. These taints should be actively reconciled e.g.
                        if you ask the machine controller to apply a taint and then
                        manually remove the taint the machine controller will put
                        it back) but not have the machine controller remove any taints
                      items:
                        type: object
                      type: array
                    versions:
                      description: Versions of key software to use. This field is
                        optional at cluster creation time, and omitting the field
                        indicates that the cluster installation tool should select
                        defaults for the user. These defaults may differ based on
                        the cluster installer, but the tool should populate the values
                        it uses when persisting Machine objects. A Machine spec missing
                        this field at runtime is invalid.
                      properties:
                        controlPlane:
                          description: ControlPlane is the semantic version of the
                            Kubernetes control plane to run. This should only be populated
                            when the machine is a control plane.
                          type: string
                        kubelet:
                          description: Kubelet is the semantic version of kubelet
                            to run
                          type: string
                      required:
                      - kubelet
                      type: object
                  required:
                  - providerSpec
                  type: object
              type: object
          required:
          - selector
          - template
          type: object
        status:
          properties:
            availableReplicas:
              description: Total number of available machines (ready for at least
                minReadySeconds) targeted by this deployment.
              format: int32
              type: integer
            observedGeneration:
              description: The generation observed by the deployment controller.
              format: int64
              type: integer
            readyReplicas:
              description: Total number of ready machines targeted by this deployment.
              format: int32
              type: integer
            replicas:
              description: Total number of non-terminated machines targeted by this
                deployment (their labels match the selector).
              format: int32
              type: integer
            unavailableReplicas:
              description: Total number of unavailable machines targeted by this deployment.
                This is the total number of machines that are still required for the
                deployment to have 100% available capacity. They may either be machines
                that are running but not yet available or machines that still have
                not been created.
              format: int32
              type: integer
            updatedReplicas:
              description: Total number of non-terminated machines targeted by this
                deployment that have the desired template spec.
              format: int32
              type: integer
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: machines.cluster.k8s.io
spec:
  additionalPrinterColumns:
  - JSONPath: .spec.providerID
    description: Provider ID
    name: ProviderID
    type: string
  - JSONPath: .status.phase
    description: Machine status such as Terminating/Pending/Running/Failed etc
    name: Phase
    type: string
  - JSONPath: .status.nodeRef.name
    description: Node name associated with this machine
    name: NodeName
    priority: 1
    type: string
  group: cluster.k8s.io
  names:
    kind: Machine
    plural: machines
  scope: Namespaced
  subresources:
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        spec:
          properties:
            configSource:
              description: ConfigSource is used to populate in the associated Node
                for dynamic kubelet config. This field already exists in Node, so
                any updates to it in the Machine spec will be automatically copied
                to the linked NodeRef from the status. The rest of dynamic kubelet
                config support should then work as-is.
              type: object
            metadata:
              description: ObjectMeta will autopopulate the Node created. Use this
                to indicate what labels, annotations, name prefix, etc., should be
                used when creating the Node.
              type: object
            providerID:
              description: ProviderID is the identification ID of the machine provided
                by the provider. This field must match the provider ID as seen on
                the node object corresponding to this machine. This field is required
                by higher level consumers of cluster-api. Example use case is cluster
                autoscaler with cluster-api as provider. Clean-up logic in the autoscaler
                compares machines to nodes to find out machines at provider which
                could not get registered as Kubernetes nodes. With cluster-api as
                a generic out-of-tree provider for autoscaler, this field is required
                by autoscaler to be able to have a provider view of the list of machines.
                Another list of nodes is queried from the k8s apiserver and then a
                comparison is done to find out unregistered machines and are marked
                for delete. This field will be set by the actuators and consumed by
                higher level entities like autoscaler that will be interfacing with
                cluster-api as generic provider.
              type: string
            providerSpec:
              description: ProviderSpec details Provider-specific configuration to
                use during node creation.
              properties:
                value:
                  description: Value is an inlined, serialized representation of the
                    resource configuration. It is recommended that providers maintain
                    their own versioned API types that should be serialized/deserialized
                    from this field, akin to component config.
                  type: object
                valueFrom:
                  description: Source for the provider configuration. Cannot be used
                    if value is not empty.
                  properties:
                    machineClass:
                      description: The machine class from which the provider config
                        should be sourced.
                      properties:
                        provider:
                          description: Provider is the name of the cloud-provider
                            which MachineClass is intended for.
                          type: string
                      type: object
                  type: object
              type: object
            taints:
              description: The list of the taints to be applied to the corresponding
                Node in additive manner. This list will not overwrite any other taints
                added to the Node on an ongoing basis by other entities. These taints
                should be actively reconciled e.g. if you ask the machine controller
                to apply a taint and then manually remove the taint the machine controller
                will put it back) but not have the machine controller remove any taints
              items:
                type: object
              type: array
            versions:
              description: Versions of key software to use. This field is optional
                at cluster creation time, and omitting the field indicates that the
                cluster installation tool should select defaults for the user. These
                defaults may differ based on the cluster installer, but the tool should
                populate the values it uses when persisting Machine objects. A Machine
                spec missing this field at runtime is invalid.
              properties:
                controlPlane:
                  description: ControlPlane is the semantic version of the Kubernetes
                    control plane to run. This should only be populated when the machine
                    is a control plane.
                  type: string
                kubelet:
                  description: Kubelet is the semantic version of kubelet to run
                  type: string
              required:
              - kubelet
              type: object
          required:
          - providerSpec
          type: object
        status:
          properties:
            addresses:
              description: Addresses is a list of addresses assigned to the machine.
                Queried from cloud provider, if available.
              items:
                type: object
              type: array
            conditions:
              description: 'Conditions lists the conditions synced from the node conditions
                of the corresponding node-object. Machine-controller is responsible
                for keeping conditions up-to-date. MachineSet controller will be taking
                these conditions as a signal to decide if machine is healthy or needs
                to be replaced. Refer: https://kubernetes.io/docs/concepts/architecture/nodes/#condition'
              items:
                type: object
              type: array
            errorMessage:
              description: ErrorMessage will be set in the event that there is a terminal
                problem reconciling the Machine and will contain a more verbose string
                suitable for logging and human consumption.  This field should not
                be set for transitive errors that a controller faces that are expected
                to be fixed automatically over time (like service outages), but instead
                indicate that something is fundamentally wrong with the Machine's
                spec or the configuration of the controller, and that manual intervention
                is required. Examples of terminal errors would be invalid combinations
                of settings in the spec, values that are unsupported by the controller,
                or the responsible controller itself being critically misconfigured.  Any
                transient errors that occur during the reconciliation of Machines
                can be added as events to the Machine object and/or logged in the
                controller's output.
              type: string
            errorReason:
              description: ErrorReason will be set in the event that there is a terminal
                problem reconciling the Machine and will contain a succinct value
                suitable for machine interpretation.  This field should not be set
                for transitive errors that a controller faces that are expected to
                be fixed automatically over time (like service outages), but instead
                indicate that something is fundamentally wrong with the Machine's
                spec or the configuration of the controller, and that manual intervention
                is required. Examples of terminal errors would be invalid combinations
                of settings in the spec, values that are unsupported by the controller,
                or the responsible controller itself being critically misconfigured.  Any
                transient errors that occur during the reconciliation of Machines
                can be added as events to the Machine object and/or logged in the
                controller's output.
              type: string
            lastOperation:
              description: LastOperation describes the last-operation performed by
                the machine-controller. This API should be useful as a history in
                terms of the latest operation performed on the specific machine. It
                should also convey the state of the latest-operation for example if
                it is still on-going, failed or completed successfully.
              properties:
                description:
                  description: Description is the human-readable description of the
                    last operation.
                  type: string
                lastUpdated:
                  description: LastUpdated is the timestamp at which LastOperation
                    API was last-updated.
                  format: date-time
                  type: string
                state:
                  description: State is the current status of the last performed operation.
                    E.g. Processing, Failed, Successful etc
                  type: string
                type:
                  description: Type is the type of operation which was last performed.
                    E.g. Create, Delete, Update etc
                  type: string
              type: object
            lastUpdated:
              description: LastUpdated identifies when this status was last observed.
              format: date-time
              type: string
            nodeRef:
              description: NodeRef will point to the corresponding Node if it exists.
              type: object
            phase:
              description: Phase represents the current phase of machine actuation.
                E.g. Pending, Running, Terminating, Failed etc.
              type: string
            providerStatus:
              description: ProviderStatus details a Provider-specific status. It is
                recommended that providers maintain their own versioned API types
                that should be serialized/deserialized from this field.
              type: object
            versions:
              description: 'Versions specifies the current versions of software on
                the corresponding Node (if it exists). This is provided for a few
                reasons:  1) It is more convenient than checking the NodeRef, traversing
                it to    the Node, and finding the appropriate field in Node.Status.NodeInfo    (which
                uses different field names and formatting). 2) It removes some of
                the dependency on the structure of the Node,    so that if the structure
                of Node.Status.NodeInfo changes, only    machine controllers need
                to be updated, rather than every client    of the Machines API. 3)
                There is no other simple way to check the control plane    version.
                A client would have to connect directly to the apiserver    running
                on the target node in order to find out its version.'
              properties:
                controlPlane:
                  description: ControlPlane is the semantic version of the Kubernetes
                    control plane to run. This should only be populated when the machine
                    is a control plane.
                  type: string
                kubelet:
                  description: Kubelet is the semantic version of kubelet to run
                  type: string
              required:
              - kubelet
              type: object
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  creationTimestamp: null
  labels:
    controller-tools.k8s.io: "1.0"
  name: machinesets.cluster.k8s.io
spec:
  group: cluster.k8s.io
  names:
    kind: MachineSet
    plural: machinesets
  scope: Namespaced
  subresources:
    scale:
      labelSelectorPath: .status.labelSelector
      specReplicasPath: .spec.replicas
      statusReplicasPath: .status.replicas
    status: {}
  validation:
    openAPIV3Schema:
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        metadata:
          type: object
        spec:
          properties:
            deletePolicy:
              description: DeletePolicy defines the policy used to identify nodes
                to delete when downscaling. Defaults to "Random".  Valid values are
                "Random, "Newest", "Oldest"
              enum:
              - Random
              - Newest
              - Oldest
              type: string
            minReadySeconds:
              description: MinReadySeconds is the minimum number of seconds for which
                a newly created machine should be ready. Defaults to 0 (machine will
                be considered available as soon as it is ready)
              format: int32
              type: integer
            replicas:
              description: Replicas is the number of desired replicas. This is a pointer
                to distinguish between explicit zero and unspecified. Defaults to
                1.
              format: int32
              type: integer
            selector:
              description: 'Selector is a label query over machines that should match
                the replica count. Label keys and values that must match in order
                to be controlled by this MachineSet. It must match the machine template''s
                labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors'
              type: object
            template:
              description: Template is the object that describes the machine that
                will be created if insufficient replicas are detected.
              properties:
                metadata:
                  description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata'
                  type: object
                spec:
                  description: 'Specification of the desired behavior of the machine.
                    More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status'
                  properties:
                    configSource:
                      description: ConfigSource is used to populate in the associated
                        Node for dynamic kubelet config. This field already exists
                        in Node, so any updates to it in the Machine spec will be
                        automatically copied to the linked NodeRef from the status.
                        The rest of dynamic kubelet config support should then work
                        as-is.
                      type: object
                    metadata:
                      description: ObjectMeta will autopopulate the Node created.
                        Use this to indicate what labels, annotations, name prefix,
                        etc., should be used when creating the Node.
                      type: object
                    providerID:
                      description: ProviderID is the identification ID of the machine
                        provided by the provider. This field must match the provider
                        ID as seen on the node object corresponding to this machine.
                        This field is required by higher level consumers of cluster-api.
                        Example use case is cluster autoscaler with cluster-api as
                        provider. Clean-up logic in the autoscaler compares machines
                        to nodes to find out machines at provider which could not
                        get registered as Kubernetes nodes. With cluster-api as a
                        generic out-of-tree provider for autoscaler, this field is
                        required by autoscaler to be able to have a provider view
                        of the list of machines. Another list of nodes is queried
                        from the k8s apiserver and then a comparison is done to find
                        out unregistered machines and are marked for delete. This
                        field will be set by the actuators and consumed by higher
                        level entities like autoscaler that will be interfacing with
                        cluster-api as generic provider.
                      type: string
                    providerSpec:
                      description: ProviderSpec details Provider-specific configuration
                        to use during node creation.
                      properties:
                        value:
                          description: Value is an inlined, serialized representation
                            of the resource configuration. It is recommended that
                            providers maintain their own versioned API types that
                            should be serialized/deserialized from this field, akin
                            to component config.
                          type: object
                        valueFrom:
                          description: Source for the provider configuration. Cannot
                            be used if value is not empty.
                          properties:
                            machineClass:
                              description: The machine class from which the provider
                                config should be sourced.
                              properties:
                                provider:
                                  description: Provider is the name of the cloud-provider
                                    which MachineClass is intended for.
                                  type: string
                              type: object
                          type: object
                      type: object
                    taints:
                      description: The list of the taints to be applied to the corresponding
                        Node in additive manner. This list will not overwrite any
                        other taints added to the Node on an ongoing basis by other
                        entities. These taints should be actively reconciled e.g.
                        if you ask the machine controller to apply a taint and then
                        manually remove the taint the machine controller will put
                        it back) but not have the machine controller remove any taints
                      items:
                        type: object
                      type: array
                    versions:
                      description: Versions of key software to use. This field is
                        optional at cluster creation time, and omitting the field
                        indicates that the cluster installation tool should select
                        defaults for the user. These defaults may differ based on
                        the cluster installer, but the tool should populate the values
                        it uses when persisting Machine objects. A Machine spec missing
                        this field at runtime is invalid.
                      properties:
                        controlPlane:
                          description: ControlPlane is the semantic version of the
                            Kubernetes control plane to run. This should only be populated
                            when the machine is a control plane.
                          type: string
                        kubelet:
                          description: Kubelet is the semantic version of kubelet
                            to run
                          type: string
                      required:
                      - kubelet
                      type: object
                  required:
                  - providerSpec
                  type: object
              type: object
          required:
          - selector
          type: object
        status:
          properties:
            availableReplicas:
              description: The number of available replicas (ready for at least minReadySeconds)
                for this MachineSet.
              format: int32
              type: integer
            errorMessage:
              type: string
            errorReason:
              description: In the event that there is a terminal problem reconciling
                the replicas, both ErrorReason and ErrorMessage will be set. ErrorReason
                will be populated with a succinct value suitable for machine interpretation,
                while ErrorMessage will contain a more verbose string suitable for
                logging and human consumption.  These fields should not be set for
                transitive errors that a controller faces that are expected to be
                fixed automatically over time (like service outages), but instead
                indicate that something is fundamentally wrong with the MachineTemplate's
                spec or the configuration of the machine controller, and that manual
                intervention is required. Examples of terminal errors would be invalid
                combinations of settings in the spec, values that are unsupported
                by the machine controller, or the responsible machine controller itself
                being critically misconfigured.  Any transient errors that occur during
                the reconciliation of Machines can be added as events to the MachineSet
                object and/or logged in the controller's output.
              type: string
            fullyLabeledReplicas:
              description: The number of replicas that have labels matching the labels
                of the machine template of the MachineSet.
              format: int32
              type: integer
            observedGeneration:
              description: ObservedGeneration reflects the generation of the most
                recently observed MachineSet.
              format: int64
              type: integer
            readyReplicas:
              description: The number of ready replicas for this MachineSet. A machine
                is considered ready when the node has been created and is "Ready".
              format: int32
              type: integer
            replicas:
              description: Replicas is the most recently observed number of replicas.
              format: int32
              type: integer
          required:
          - replicas
          type: object
  version: v1alpha1
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  creationTimestamp: null
  name: aws-provider-manager-role
rules:
- apiGroups:
  - awsprovider.k8s.io
  resources:
  - awsclusterproviderconfigs
  - awsclusterproviderstatuses
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - clusters
  - clusters/status
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - awsprovider.k8s.io
  resources:
  - awsmachineproviderconfigs
  - awsmachineproviderstatuses
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - machines
  - machines/status
  - machinedeployments
  - machinedeployments/status
  - machinesets
  - machinesets/status
  - machineclasses
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - clusters
  - clusters/status
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes
  - events
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  creationTimestamp: null
  name: cluster-api-manager-role
rules:
- apiGroups:
  - cluster.k8s.io
  resources:
  - clusters
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - ""
  resources:
  - events
  verbs:
  - get
  - list
  - watch
  - create
- apiGroups:
  - cluster.k8s.io
  resources:
  - machines
  - machines/status
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - machinedeployments
  - machinedeployments/status
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - machinesets
  - machinesets/status
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - machines
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
- apiGroups:
  - cluster.k8s.io
  resources:
  - machines
  verbs:
  - get
  - list
  - watch
  - create
  - update
  - patch
  - delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  creationTimestamp: null
  name: aws-provider-manager-rolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: aws-provider-manager-role
subjects:
- kind: ServiceAccount
  name: default
  namespace: aws-provider-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  creationTimestamp: null
  name: cluster-api-manager-rolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-api-manager-role
subjects:
- kind: ServiceAccount
  name: default
  namespace: cluster-api-system
---
apiVersion: v1
kind: Service
metadata:
  labels:
    control-plane: controller-manager
    controller-tools.k8s.io: "1.0"
  name: aws-provider-controller-manager-service
  namespace: aws-provider-system
spec:
  ports:
  - port: 443
  selector:
    control-plane: controller-manager
    controller-tools.k8s.io: "1.0"
---
apiVersion: v1
kind: Service
metadata:
  labels:
    control-plane: controller-manager
    controller-tools.k8s.io: "1.0"
  name: cluster-api-controller-manager-service
  namespace: cluster-api-system
spec:
  ports:
  - port: 443
  selector:
    control-plane: controller-manager
    controller-tools.k8s.io: "1.0"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    control-plane: controller-manager
    controller-tools.k8s.io: "1.0"
  name: aws-provider-controller-manager
  namespace: aws-provider-system
spec:
  selector:
    matchLabels:
      control-plane: controller-manager
      controller-tools.k8s.io: "1.0"
  serviceName: aws-provider-controller-manager-service
  template:
    metadata:
      labels:
        control-plane: controller-manager
        controller-tools.k8s.io: "1.0"
    spec:
      containers:
      - args:
        - -v=3
        - -logtostderr=true
        - -stderrthreshold=INFO
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        image: gcr.io/cluster-api-provider-aws/cluster-api-aws-controller:v0.3.0
        imagePullPolicy: IfNotPresent
        name: manager
        resources:
          limits:
            cpu: 100m
            memory: 30Mi
          requests:
            cpu: 100m
            memory: 20Mi
        volumeMounts:
        - mountPath: /home/.aws
          name: credentials
      terminationGracePeriodSeconds: 10
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      - key: CriticalAddonsOnly
        operator: Exists
      - effect: NoExecute
        key: node.alpha.kubernetes.io/notReady
        operator: Exists
      - effect: NoExecute
        key: node.alpha.kubernetes.io/unreachable
        operator: Exists
      volumes:
      - name: credentials
        secret:
          secretName: manager-bootstrap-credentials
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    control-plane: controller-manager
    controller-tools.k8s.io: "1.0"
  name: cluster-api-controller-manager
  namespace: cluster-api-system
spec:
  selector:
    matchLabels:
      control-plane: controller-manager
      controller-tools.k8s.io: "1.0"
  serviceName: cluster-api-controller-manager-service
  template:
    metadata:
      labels:
        control-plane: controller-manager
        controller-tools.k8s.io: "1.0"
    spec:
      containers:
      - command:
        - /manager
        image: gcr.io/k8s-cluster-api/cluster-api-controller:0.1.1
        name: manager
        resources:
          limits:
            cpu: 100m
            memory: 30Mi
          requests:
            cpu: 100m
            memory: 20Mi
      terminationGracePeriodSeconds: 10
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      - key: CriticalAddonsOnly
        operator: Exists
      - effect: NoExecute
        key: node.alpha.kubernetes.io/notReady
        operator: Exists
      - effect: NoExecute
        key: node.alpha.kubernetes.io/unreachable
        operator: Exists
---
apiVersion: v1
kind: Secret
metadata:
  name: manager-bootstrap-credentials
  namespace: aws-provider-system
type: Opaque
data:
  credentials:***********************==

not create 5 vpcs.

Environment:

• Cluster-api-provider-aws version:

  guohao@buffer ~/Downloads $ clusterctl version
  Version Info: GitReleaseTag: "v0.3.0", MajorVersion: "0", MinorVersion:"3", GitReleaseCommit:"359991ec850866", GitTreeState:"clean"

• Kubernetes version: (use kubectl version):

  guohao@buffer ~ $ kubectl version
  Client Version: version.Info{Major:"1", Minor:"13", GitVersion:"v1.13.1", GitCommit:"eec55b9ba98609a46fee712359c7b5b365bdd920", GitTreeState:"clean", BuildDate:"2018-12-13T10:39:04Z", GoVersion:"go1.11.2", Compiler:"gc", Platform:"darwin/amd64"}
  Server Version: version.Info{Major:"1", Minor:"14", GitVersion:"v1.14.2", GitCommit:"66049e3b21efe110454d67df4fa62b08ea79a19b", GitTreeState:"clean", BuildDate:"2019-05-17T00:58:35Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}

• OS (e.g. from /etc/os-release):

  guohao@buffer ~ $ uname -a
  Darwin buffer 18.6.0 Darwin Kernel Version 18.6.0: Thu Apr 25 23:16:27 PDT 2019; root:xnu-4903.261.4~2/RELEASE_X86_64 x86_64

[detiber]

@Sn0rt it looks like the AWS account that you are using is missing some required permissions:

E0610 08:12:02.487821       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: cc277ef7-2f64-49e3-9426-bb87acfeee88

Since the error is happening prior to setting the tags on the VPC, it is attempting to create a new VPC.

That said, we should likely attempt to make this condition more visible to the user and avoid continually attempting to recreate the VPC in this case.

[Sn0rt]

@detiber the AWS account TOKEN and KEY has created by clusterawsadm alpha bootstrap create-stack command line.

and the VPC has been created that you can find at the before.

user arn:aws:iam::179516646050:user/bootstrapper.cluster-api-provider-aws.sigs.k8s.io

        {
            "Path": "/",
            "GroupName": "bootstrapper.cluster-api-provider-aws.sigs.k8s.io",
            "GroupId": "AGPASTTAGUKRCB52MWBC4",
            "Arn": "arn:aws:iam::179516646050:group/bootstrapper.cluster-api-provider-aws.sigs.k8s.io",
            "CreateDate": "2019-06-10T06:17:06Z"
        },

group bootstrapper.cluster-api-provider-aws.sigs.k8s.io

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "ec2:AllocateAddress",
                "ec2:AssociateRouteTable",
                "ec2:AttachInternetGateway",
                "ec2:AuthorizeSecurityGroupIngress",
                "ec2:CreateInternetGateway",
                "ec2:CreateNatGateway",
                "ec2:CreateRoute",
                "ec2:CreateRouteTable",
                "ec2:CreateSecurityGroup",
                "ec2:CreateSubnet",
                "ec2:CreateTags",
                "ec2:CreateVpc",
                "ec2:DeleteInternetGateway",
                "ec2:DeleteNatGateway",
                "ec2:DeleteRouteTable",
                "ec2:DeleteSecurityGroup",
                "ec2:DeleteSubnet",
                "ec2:DeleteTags",
                "ec2:DeleteVpc",
                "ec2:DescribeAccountAttributes",
                "ec2:DescribeAddresses",
                "ec2:DescribeAvailabilityZones",
                "ec2:DescribeInstances",
                "ec2:DescribeInternetGateways",
                "ec2:DescribeImages",
                "ec2:DescribeNatGateways",
                "ec2:DescribeNetworkInterfaces",
                "ec2:DescribeRouteTables",
                "ec2:DescribeSecurityGroups",
                "ec2:DescribeSubnets",
                "ec2:DescribeVpcs",
                "ec2:DetachInternetGateway",
                "ec2:DisassociateRouteTable",
                "ec2:ModifyInstanceAttribute",
                "ec2:ModifyNetworkInterfaceAttribute",
                "ec2:ModifySubnetAttribute",
                "ec2:ReleaseAddress",
                "ec2:RevokeSecurityGroupIngress",
                "ec2:RunInstances",
                "ec2:TerminateInstances",
                "elasticloadbalancing:CreateLoadBalancer",
                "elasticloadbalancing:ConfigureHealthCheck",
                "elasticloadbalancing:DeleteLoadBalancer",
                "elasticloadbalancing:DescribeLoadBalancers",
                "elasticloadbalancing:DescribeLoadBalancerAttributes",
                "elasticloadbalancing:ModifyLoadBalancerAttributes",
                "elasticloadbalancing:RegisterInstancesWithLoadBalancer"
            ],
            "Resource": [
                "*"
            ],
            "Effect": "Allow"
        },
        {
            "Condition": {
                "StringLike": {
                    "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
                }
            },
            "Action": [
                "iam:CreateServiceLinkedRole"
            ],
            "Resource": [
                "arn:aws:iam::179516646050:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing"
            ],
            "Effect": "Allow"
        },
        {
            "Action": [
                "iam:PassRole"
            ],
            "Resource": [
                "arn:aws:iam::179516646050:role/*.cluster-api-provider-aws.sigs.k8s.io"
            ],
            "Effect": "Allow"
        }
    ]
}

err info

E0611 02:51:45.017606       1 controller.go:134] Error reconciling cluster object test1; failed to reconcile network for cluster "test1": failed to create new vpc: failed to set vpc attributes: UnauthorizedOperation: You are not authorized to perform this operation.
    status code: 403, request id: 65f5d3bd-27f6-4e67-b2c3-bad159867142