mjeffryes
commented
2 days ago Thanks for reporting this @gavindsouza. Could you share the program you used (or at least the remote.Command resource ) so we can attempt to repro?
Open gavindsouza opened 3 days ago
mjeffryes
commented
2 days ago Thanks for reporting this @gavindsouza. Could you share the program you used (or at least the remote.Command resource ) so we can attempt to repro?
gavindsouza
commented
1 day ago The commented delete action in the run_docker_compose is where the failure occurs. Appreciate the help.
"""Script that builds a fresh image and deploys it to an AWS EC2 instance."""
import os
import json
import time
import pulumi
from pulumi import FileAsset
from pulumi_aws import ec2, route53
from pathlib import Path
import pulumi_command as command
from pulumi_docker_build import (
Image,
RegistryArgs,
BuildContextArgs,
DockerfileArgs,
Platform,
)
SERVER_NAME = "application.service"
SERVER_USER = "ubuntu"
NODE_VERSION = "20.18.0"
PYTHON_VERSION = "3.11.9"
RELEASE_VERSION = int(time.time())
# AWS Environment Variables
AWS_KEY_NAME = os.environ["AWS_KEY_NAME"]
AWS_PEM_KEY = os.environ["AWS_PEM_KEY"]
AWS_SUBNET = os.environ["AWS_SUBNET"]
AWS_SECURITY_GROUPS = eval(os.environ["AWS_SECURITY_GROUPS"])
AWS_INSTANCE_TYPE = ec2.InstanceType.T2_X_LARGE
# Load application configuration
apps_data = Path("apps.json").read_text()
apps_data_json = json.loads(apps_data)
main_app_info = apps_data_json.pop(0)
assert main_app_info["name"] == "main_app", "Main app must be the first in the list."
MAIN_APP_COMMIT_SHA = main_app_info["version"]
# Docker image settings
IMAGE_NAME = f"ghcr.io/organization/{SERVER_NAME}:{RELEASE_VERSION}"
for app in apps_data_json:
app["url"] = app["url"].format(PAT=os.environ["GITHUB_PAT"])
Path("_apps.json").write_text(json.dumps(apps_data_json))
# Generate Docker Compose configuration
Path("_compose.yml").write_text(
Path("docker-compose.yml")
.read_text()
.format(
IMAGE_NAME=IMAGE_NAME,
INSTALL_APPS=", ".join(f"--install-app {x['name']}" for x in apps_data_json),
)
)
# Build and push the Docker image
generate_image = Image(
"application-service-image",
tags=[IMAGE_NAME],
context=BuildContextArgs(location="."),
dockerfile=DockerfileArgs(location="Containerfile"),
platforms=[Platform.LINUX_AMD64],
build_args={
"MAIN_APP_VERSION": MAIN_APP_COMMIT_SHA,
"PYTHON_VERSION": PYTHON_VERSION,
"NODE_VERSION": NODE_VERSION,
},
push=True,
registries=[
RegistryArgs(
address="ghcr.io",
username=os.environ["GITHUB_USER"],
password=os.environ["GITHUB_PKG_TOKEN"],
),
RegistryArgs(
address="docker.io",
),
],
)
# EC2 instance configuration
server = ec2.Instance(
SERVER_NAME,
instance_type=AWS_INSTANCE_TYPE,
ami=ec2.get_ami(
most_recent=True,
owners=["amazon"],
filters=[{"name": "name", "values": ["*ubuntu-latest-*"]}],
).id,
key_name=ec2.get_key_pair(key_name=AWS_KEY_NAME).key_name,
vpc_security_group_ids=AWS_SECURITY_GROUPS,
subnet_id=AWS_SUBNET,
opts=pulumi.ResourceOptions(depends_on=[generate_image]),
)
# Set up remote connection for commands
ssh_connection = command.remote.ConnectionArgs(
host=server.public_ip,
user=SERVER_USER,
private_key=AWS_PEM_KEY,
)
# Install Docker on the remote EC2 instance
setup_docker = command.remote.Command(
"install-docker",
connection=ssh_connection,
create="curl -fsSL https://get.docker.com | sh",
opts=pulumi.ResourceOptions(depends_on=[server]),
)
# Upload Docker Compose configuration file to the EC2 instance
upload_compose_file = command.remote.CopyToRemote(
"update-compose-file",
connection=ssh_connection,
source=FileAsset("_compose.yml"),
remote_path=f"/home/{SERVER_USER}/docker-compose.yml",
opts=pulumi.ResourceOptions(depends_on=[server]),
)
# Run Docker Compose on the remote instance to start the application
run_docker_compose = command.remote.Command(
"start-or-update-application",
connection=ssh_connection,
create=(
f"sudo docker login 'ghcr.io' -u '{os.environ['GITHUB_USER']}' -p '{os.environ['GITHUB_PKG_TOKEN']}' && "
"sudo docker compose -p application_service up -d"
),
# delete=(f"sudo docker compose -p application_service down"),
opts=pulumi.ResourceOptions(depends_on=[setup_docker, upload_compose_file]),
)
# Configure DNS record for the application
update_dns = route53.Record(
"update-dns",
name="application",
zone_id=route53.get_zone(name=os.environ["AWS_ROUTE53_ZONE_NAME"]).id,
type="A",
ttl=300,
records=[server.public_ip],
opts=pulumi.ResourceOptions(depends_on=[server]),
)
# Export outputs
pulumi.export("public_ip", server.public_ip)
pulumi.export("public_dns", server.public_dns)
pulumi.export("image_name", IMAGE_NAME)
pulumi.export("public_domain", update_dns.fqdn)
What happened?
I did
pulumi up -s dev -y, everything looked ok. I ranpulumi destroy -s dev -yhoping everything gets dropped. However, there's a EOF failure while executingdeletefor aremote.Command.I couldn't seem to get past it now. Although it seems like the command itself has successfully run on remote. Modifying the file and trying up/delete/destroy didn't work - and it might've been concerning if it did too.
I passed
--continue-on-erroradditionally to fully execute destroy (and drop the created resources). Followed up with apulumi stack rm dev.^ This seems to have worked for me as a workaround.
Example
Output of
pulumi aboutAdditional context
No response
Contributing
Vote on this issue by adding a 👍 reaction. To contribute a fix for this issue, leave a comment (and link to your pull request, if you've opened one already).