`paramiko.ssh_exception.SSHException: Channel is not open` when chaining invoke tasks

[sydney-opcity]

Hi there, I have a set of invoke tasks that are, minimally, as follows:

from fabric import Connection
from invoke import Context, task

from tasks.lib.console import console
from tasks.lib.constants import (AWS_CONTAINER_REGISTRY,
                                 EC2_INSTANCE,
                                 PORT,
                                 REMOTE_IMAGE_NAME)
from tasks.lib.utility import _stop_container, ecr_login
from tasks.ssm import ssm as connect_ssm

@task
def pull(c: Context, host: str = EC2_INSTANCE, tag: str = "latest") -> None:
    """Pulls the latest docker image from ECR to the EC2 instance.
    """
    c = Connection(host)
    ecr_login(c)
    c.run(f"docker pull {AWS_CONTAINER_REGISTRY}/{REMOTE_IMAGE_NAME}:{tag}", pty=True)

@task
def stop(c: Context, host: str = EC2_INSTANCE, name: str = "name") -> None:
    """Stops and removes a running remote development container. Defaults to your remote development EC2.
    """
    c = Connection(host)
    with console.status("[bold green]Stopping and removing the running container..."):
        result = _stop_container(c, name=name)
    console.log(result.stdout)

@task
def run(
    c: Context,
    tag: str = "latest",
    host: str = EC2_INSTANCE,
    port: str = PORT,
    name: str = "name",
) -> None:
    """Starts a remote container if not already running and connects to it.
    """
    run_cmd = " ".join(
        [
            "docker run",
            f"-t -d --name {name}",
            f"-p {port}:{port}",
            f"{AWS_CONTAINER_REGISTRY}/{REMOTE_IMAGE_NAME}:{tag}",
        ]
    )
    cmd = f'if [ ! "$(docker ps -q -f name={name})" ]; then {run_cmd}; else echo "Already running!"; fi'

    connect_ssm(c, host=host, local_port=port, host_port=port, cmd=cmd)

I am trying to chain these events together so that one does not need to invoke stop, pull, and run in sequence. Let's call it update.

def update(
    c: Context,
    tag: str = "latest",
    host: str = EC2_INSTANCE,
    port: str = PORT,
    name: str = "name",
) -> None:
    stop(c, host=host, name=name)
    pull(c, host=host, tag=tag)
    run(c, tag=tag, host=host, port=port, name=name)

I find that, while stop and pull run without issue, I get the following error on run:

paramiko.ssh_exception.SSHException: Channel is not open

I believe that the error occurs during the first check of the connect_ssm piece, which is reproduced minimally below:

@task
def ssm(
    c: Context, host: str, cmd: str | None = None, local_port: str | None = None, host_port: str | None = None
) -> None:
    """
    Use AWS SSM to open an SSH connection with a specific EC2 instance.
    """
    if not _check_known_host(c, host):
        public_key = c.run("cat ~/.ssh/<path_to_key>", hide=True).stdout.rstrip()
        remote_cmd = f"mkdir -p ~/.ssh && echo {public_key} >> /home/ubuntu/.ssh/authorized_keys"
        add_public_key_to_ec2_cmd = (
            f"aws ssm send-command --instance-ids {host} --document-name AWS-RunShellScript "
            f"--parameters commands='{remote_cmd}'"
        )
        c.run(add_public_key_to_ec2_cmd, hide=True)

    cmd = cmd or ""
    if local_port and not host_port:
        return
    if host_port:
        cmd = f"-L {local_port or host_port}:localhost:{host_port} '{cmd}'"

    c.run(f"ssh {host} -t {cmd}", pty=True)

def _check_known_host(c: Context, host: str) -> bool:
    return c.run(f"ssh-keygen -F {host}", warn=True, hide=True).ok

I have tried adding c = Connection(host) to the run task as well as the ssm task, but I get errors of the following sort:

Encountered a bad command exit code!

Command: 'cat ~/.ssh/<path_to_key>'

Exit code: 1

Stdout:

Stderr:

cat: /home/ubuntu/.ssh/<path_to_key>: No such file or directory

Notably though, I can run inv pull, inv stop, and inv run in sequence with no issue.

I'm thinking that the issue has something to do with the Context, but this is where I am getting stuck -- I don't understand why these tasks work sequentially but not when chained together.

I apologize for the length of code involved in understanding this issue, but any help in the right direction would be appreciated.