# Wait until leader has been elected and bootstrap consul ACLs
for i in {1..9}; do
# capture stdout and stderr
set +e
sleep 5
if ! OUTPUT=$(consul acl bootstrap 2>&1); then
echo "$${i}: consul acl bootstrap: $${OUTPUT}"
if [[ "$${OUTPUT}" = *"No cluster leader"* ]]; then
echo "consul no cluster leader"
continue
else
echo "consul already bootstrapped"
exit 0
fi
fi
set -e
echo "$${OUTPUT}" | grep -i secretid | awk '{print $2}' > "$${CONSUL_BOOTSTRAP_TOKEN}"
if [[ -s "$${CONSUL_BOOTSTRAP_TOKEN}" ]]; then
echo "consul bootstrapped"
break
fi
done
Here is my /var/log/user-data.log:
ACL bootstrap begin
+ for i in {1..9}
+ set +e
+ sleep 5
++ consul acl bootstrap
+ OUTPUT='Failed ACL bootstrapping: Unexpected response code: 401 (ACL support disabled)'
+ echo '1: consul acl bootstrap: Failed ACL bootstrapping: Unexpected response code: 401 (ACL support disabled)
'
1: consul acl bootstrap: Failed ACL bootstrapping: Unexpected response code: 401 (ACL support disabled)
+ [[ Failed ACL bootstrapping: Unexpected response code: 401 (ACL support disabled) = *\N\o\ \c\l\u\s\t\e\r\ \l
\e\a\d\e\r* ]]
+ echo 'consul already bootstrapped'
consul already bootstrapped
+ exit 0
But when I SSH into this node and run consul acl bootstrap then the command completed without errors:
For the script:
Here is my
/var/log/user-data.log:But when I SSH into this node and run
consul acl bootstrapthen the command completed without errors:The relevant part of
consul.hcl:The same script was working on 1.17.3 but not on 1.18.2.
Relevant logs
Notice: Node ID was changed. I am not sure why. ``` Jun 06 14:28:02 ip-172-31-9-168 systemd[1]: Starting "HashiCorp Consul - A service mesh solution"... Jun 06 14:28:16 ip-172-31-9-168 consul[372]: ==> Starting Consul agent... Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Version: '1.18.2' Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Build Date: '2024-05-16 19:10:00 +0000 UTC' Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Node ID: '30448039-a65e-7f1b-15f1-868d306d00 21' Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Node name: 'ip-172-31-9-168' Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Datacenter: 'dc1' (Segment: '') Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Server: false (Bootstrap: false) Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Client Addr: [127.0.0.1] (HTTP: 8500, HTTPS: -1, gRPC: -1, gRPC-TLS: -1, DNS: 8600) Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Cluster Addr: 172.31.9.168 (LAN: 8301, WAN: 8302) Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Gossip Encryption: false Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Auto-Encrypt-TLS: false Jun 06 14:28:16 ip-172-31-9-168 consul[372]: ACL Enabled: false Jun 06 14:28:16 ip-172-31-9-168 consul[372]: ACL Default Policy: allow Jun 06 14:28:16 ip-172-31-9-168 consul[372]: HTTPS TLS: Verify Incoming: false, Verify Outgoing: false, Min Version: TLSv1_2 Jun 06 14:28:16 ip-172-31-9-168 consul[372]: gRPC TLS: Verify Incoming: false, Min Version: TLSv1_2 Jun 06 14:28:16 ip-172-31-9-168 consul[372]: Internal RPC TLS: Verify Incoming: false, Verify Outgoing: false (Verify Hostname: false), Min Version: TLSv1_2 ... Jun 06 14:28:16 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:16.781Z [INFO] agent.http: Registered res ource endpoint: endpoint=/api/hcp/v2/link Jun 06 14:28:16 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:16.781Z [INFO] agent: Starting server: ad dress=127.0.0.1:8500 network=tcp protocol=http Jun 06 14:28:16 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:16.784Z [INFO] agent: started state synce r Jun 06 14:28:16 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:16.784Z [INFO] agent: Consul agent runnin g! Jun 06 14:28:16 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:16.785Z [WARN] agent.router.manager: No s ervers available Jun 06 14:28:16 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:16.785Z [ERROR] agent.anti_entropy: failed to sync remote state: error="No known Consul servers" Jun 06 14:28:19 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:19.653Z [WARN] agent.router.manager: No s ervers available Jun 06 14:28:19 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:19.653Z [WARN] agent: Syncing node info f ailed.: error="No known Consul servers" Jun 06 14:28:19 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:19.653Z [ERROR] agent: failed to sync chan ges: error="No known Consul servers" Jun 06 14:28:19 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:19.657Z [WARN] agent.router.manager: No s ervers available Jun 06 14:28:19 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:19.658Z [WARN] agent: Syncing node info f ailed.: error="No known Consul servers" Jun 06 14:28:19 ip-172-31-9-168 consul[372]: 2024-06-06T14:28:19.658Z [ERROR] agent: failed to sync chan ges: error="No known Consul servers" ... Jun 06 14:29:28 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:28.490Z [ERROR] agent.anti_entropy: failed to sync remote state: error="No known Consul servers" Jun 06 14:29:32 ip-172-31-9-168 systemd[1]: consul.service: start operation timed out. Terminating. Jun 06 14:29:32 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:32.905Z [INFO] agent: Caught: signal=term inated Jun 06 14:29:32 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:32.905Z [INFO] agent: Gracefully shutting down agent ... Jun 06 14:29:35 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:35.947Z [INFO] agent.dns: Stopping server : protocol=DNS address=127.0.0.1:8600 network=tcp Jun 06 14:29:35 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:35.947Z [INFO] agent.dns: Stopping server : protocol=DNS address=127.0.0.1:8600 network=udp Jun 06 14:29:35 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:35.948Z [INFO] agent: Stopping server: ad dress=127.0.0.1:8500 network=tcp protocol=http Jun 06 14:29:35 ip-172-31-9-168 consul[372]: 2024-06-06T14:29:35.950Z [INFO] agent: Waiting for endpoin ts to shut down ... The above repeats couple of times, then ... Jun 06 14:31:10 ip-172-31-9-168 systemd[1]: consul.service: Scheduled restart job, restart counter is at 2. Jun 06 14:31:10 ip-172-31-9-168 systemd[1]: Stopped "HashiCorp Consul - A service mesh solution". Jun 06 14:31:10 ip-172-31-9-168 systemd[1]: Starting "HashiCorp Consul - A service mesh solution"... Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: skipping file /etc/consul.d/consul.env, extension mu st be .hcl or .json, or config format must be set Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: BootstrapExpect is set to 1; this is the same as Boo tstrap mode. Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: bootstrap = true: do not enable unless necessary Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent.tlsutil: Update: version=1 Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent.tlsutil: OutgoingRPCWrapper: version=1 Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent.tlsutil: OutgoingALPNRPCWrapper: version=1 Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: [core][Channel #1] Channel created Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: [core][Channel #1] original dial target is: "consul: //dc1.83f75e4d-5e09-1e4b-acc6-87c637078b74/server.dc1" Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: [core][Channel #1] parsed dial target is: {Scheme:co nsul Authority:dc1.83f75e4d-5e09-1e4b-acc6-87c637078b74 URL:{Scheme:consul Opaque: User: Host:dc1.83f75e 4d-5e09-1e4b-acc6-87c637078b74 Path:/server.dc1 RawPath: OmitHost:false ForceQuery:false RawQuery: Fragm ent: RawFragment:}} Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: [core][Channel #1] Channel authority set to "server. dc1" Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: [core][Channel #1] Resolver state updated: { "Addresses": null, "ServiceConfig": null, "Attributes": null } () Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent: [core][Channel #1] Channel switches to new LB policy "consul-internal" Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent.grpc.balancer: creating balancer: target=consul://dc1 .83f75e4d-5e09-1e4b-acc6-87c637078b74/server.dc1 Jun 06 14:31:11 ip-172-31-9-168 consul[883]: agent.grpc.balancer: switching server: target=consul://dc1. 83f75e4d-5e09-1e4b-acc6-87c637078b74/server.dc1 from=