terravision CLI hangs on last resource

[dnk8n]

CLI output hung, in case template is useful for debugging:

variable "aws_region" {}
variable "instance_type" {}
variable "vpc_cidr" {}
variable "subnets_cidr" {
  type = list(string)
}
variable "availability_zones" {
  type = list(string)
}
variable "cloudflare_api_token" {}
variable "github_token" {}
variable "domain" {}
variable "environments" {
  type = map(
    object(
      {
        subdomain                   = string
        deploy_key_pvt              = string
        deploy_key_pub              = string
        instance_key_pvt            = string
        instance_key_pub            = string
        override                    = string
        traefik_basicauth           = string
        acme_email                  = string
        postgres_password_superuser = string
        postgres_password_app       = string
        postgres_password_app_admin = string
        postgres_user_app_power     = string
        azure_tenant_id             = string
        azure_client_id             = string
      }
    )
  )
}

terraform {
  backend "s3" {}
  required_providers {
    cloudflare = {
      source = "cloudflare/cloudflare"
    }
    github = {
      source = "integrations/github"
    }
  }
}

provider "local" {}

provider "null" {}

provider "aws" {
  region = var.aws_region
}

provider "cloudflare" {
  api_token = var.cloudflare_api_token
}

provider "github" {
  token = var.github_token
}

locals {
  config_dir = "${path.module}/config"
  env        = terraform.workspace
  env_vars   = lookup(var.environments, local.env, null)
}

data "cloudflare_zones" "dtrack_zone" {
  filter {
    name   = var.domain
    status = "active"
    paused = false
  }
}

data "http" "ip" {
  url = "https://ifconfig.me/ip"
}

data "aws_ami" "ubuntu" {
  most_recent = true
  filter {
    name   = "name"
    values = ["ubuntu/images/hvm-ssd/ubuntu-*-22.04-amd64-server-*"]
  }
  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }
  owners = ["099720109477"] # Canonical
}

resource "aws_vpc" "dtrack_vpc" {
  cidr_block = var.vpc_cidr
  tags = {
    Name = "${local.env}-dtrack-vpc"
  }
}

resource "aws_subnet" "dtrack_subnet" {
  count                   = length(var.subnets_cidr)
  vpc_id                  = aws_vpc.dtrack_vpc.id
  cidr_block              = element(var.subnets_cidr, count.index)
  availability_zone       = element(var.availability_zones, count.index)
  map_public_ip_on_launch = true
  tags = {
    Name = "${local.env}-dtrack-subnet-${count.index + 1}"
  }
}

resource "aws_internet_gateway" "dtrack_igw" {
  vpc_id = aws_vpc.dtrack_vpc.id
  tags = {
    Name = "${local.env}-dtrack-igw"
  }
}

resource "aws_route_table" "dtrack_route_table" {
  vpc_id = aws_vpc.dtrack_vpc.id
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.dtrack_igw.id
  }
  tags = {
    Name = "${local.env}-dtrack-route-table"
  }
}

resource "aws_route_table_association" "dtrack_route_table_association" {
  count          = length(var.subnets_cidr)
  subnet_id      = element(aws_subnet.dtrack_subnet.*.id, count.index)
  route_table_id = aws_route_table.dtrack_route_table.id
}

resource "aws_security_group" "dtrack_sg" {
  name        = "${local.env}-dtrack-sg"
  description = "Allow SSH and HTTP from a specific IP"
  vpc_id      = aws_vpc.dtrack_vpc.id
  ingress {
    description = "SSH"
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["${chomp(data.http.ip.response_body)}/32"]
  }
  ingress {
    description = "HTTP"
    from_port   = 80
    to_port     = 80
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  ingress {
    description = "HTTP"
    from_port   = 443
    to_port     = 443
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  egress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
  tags = {
    Name = "${local.env}-dtrack-sg"
  }
}

resource "aws_key_pair" "dtrack_key_pair" {
  key_name   = "${local.env}-dtrack-key-pair"
  public_key = file(local.env_vars.instance_key_pub)
  tags = {
    Name = "${local.env}-dtrack-key-pair"
  }
}

resource "aws_instance" "dtrack_instance" {
  ami                    = data.aws_ami.ubuntu.id
  instance_type          = var.instance_type
  availability_zone      = var.availability_zones.0
  subnet_id              = aws_subnet.dtrack_subnet.0.id
  vpc_security_group_ids = [aws_security_group.dtrack_sg.id]
  key_name               = aws_key_pair.dtrack_key_pair.key_name
  root_block_device {
    volume_type = "gp3"
    volume_size = 30
    tags = {
      Name = "${local.env}-dtrack-instance-root-block-device"
    }
  }
  tags = {
    Name = "${local.env}-dtrack-instance"
  }
  provisioner "local-exec" {
    command = <<EOT
      until nc -z -v -w5 ${aws_instance.dtrack_instance.public_ip} 22; do
        echo "Waiting for SSH to become available..."
        sleep 5
      done
      echo "$(ssh-keyscan ${aws_instance.dtrack_instance.public_ip})" > "${local.config_dir}/ansible/${local.env}.known_hosts"
      sed -i "s/${aws_instance.dtrack_instance.public_ip}/${local.env_vars.subdomain}.${var.domain}/" "${local.config_dir}/ansible/${local.env}.known_hosts"
    EOT
  }
}

resource "cloudflare_record" "dtrack_record" {
  zone_id = data.cloudflare_zones.dtrack_zone.zones[0].id
  name    = local.env_vars.subdomain
  value   = aws_instance.dtrack_instance.public_ip
  type    = "A"
  ttl     = 120
}

resource "cloudflare_record" "dtrack_traefik_record" {
  zone_id = data.cloudflare_zones.dtrack_zone.zones[0].id
  name    = "traefik.${local.env_vars.subdomain}"
  value   = aws_instance.dtrack_instance.public_ip
  type    = "A"
  ttl     = 120
}

resource "github_repository_deploy_key" "dtrack_deploy_key" {
  title      = "${local.env}-dtrack-deploy-key"
  repository = "dtrack"
  key        = file(local.env_vars.deploy_key_pub)
  read_only  = "true"
}

resource "local_file" "ansible_hosts" {
  content = templatefile(
    "${local.config_dir}/ansible/templates/hosts.ini.tpl",
    {
      environments = var.environments
      domain       = var.domain
    }
  )
  filename        = "${local.config_dir}/ansible/hosts.ini"
  file_permission = "0600"
}

terraform.tfvars

aws_region         = "af-south-1"
instance_type      = "t3.medium"
vpc_cidr           = "10.0.0.0/16"
subnets_cidr       = ["10.0.1.0/24"]
availability_zones = ["af-south-1a"]

# Global Secrets
cloudflare_api_token = "dummy"
github_token         = "dummy"
domain               = "dummy"

environments = {
  staging = {
    subdomain        = "dtrack-staging"
    deploy_key_pvt   = "keys/staging/deploy/id"
    deploy_key_pub   = "keys/staging/deploy/id.pub"
    instance_key_pvt = "keys/staging/instance/id"
    instance_key_pub = "keys/staging/instance/id.pub"
    override         = "staging-debug"

    # Environment Secrets
    traefik_basicauth           = "dummy"
    acme_email                  = "dummy"
    postgres_password_superuser = "dummy"
    postgres_password_app       = "dummy"
    postgres_password_app_admin = "dummy"
    postgres_user_app_power     = "Dummy.User@example.com"
    azure_tenant_id             = "dummy"
    azure_client_id             = "dummy"
  }
  prod = {
    subdomain        = "dtrack"
    deploy_key_pvt   = "keys/prod/deploy/id"
    deploy_key_pub   = "keys/prod/deploy/id.pub"
    instance_key_pvt = "keys/prod/instance/id"
    instance_key_pub = "keys/prod/instance/id.pub"
    override         = "prod"

    # Environment Secrets
    traefik_basicauth           = "dummy"
    acme_email                  = "dummy"
    postgres_password_superuser = "dummy"
    postgres_password_app       = "dummy"
    postgres_password_app_admin = "dummy"
    postgres_user_app_power     = "Dummy.User@example.com"
    azure_tenant_id             = "dummy"
    azure_client_id             = "dummy"
  }
}

[dnk8n]

Command I ran:

poetry run terravision draw --source ~/src/dtrack --varfile ~/src/dtrack/terraform.tfvars

My setup/install procedure:

sudo apt update && sudo apt install graphviz
cd /tmp
git clone git@github.com:patrickchugh/terravision.git
cd /tmp/terravision
export PATH=$PATH:$(pwd)
poetry install

[maker2413]

I'm also encountering this. Did you find a way to get around this issue?

[patrickchugh]

@dnk8n @maker2413 Hello, thank you for your feedback and thanks for testing out Terravision and helping to make this a better product. I have recently completed a major revamp to the code which uses the terraform binary to download source files and generate initial relationships, before augmenting it with source code parsing as before. This is a significant departure from the previous approach and whilst slower, ensures a more accurate output and will handle all terraform in built functions now when encountered in the source. Please do a git pull and re-test again with the latest source code. If your problem is resolved, please close the the thread in github issues and mark it as completed. If not, please give as much detail as you can about the errors and include any source files if possible so I can reproduce your issue. Once again, thanks for helping and look forward to hearing from you soon.

P.

[patrickchugh]

can this issue be closed now?

[dnk8n]

I plan to be able to give feedback this week. It’s on my task list. Please understand that terravision is not right at the top. Anyone else able to validate and report their experience?

On Wed, 29 Nov 2023 at 11:31, Patrick Chugh @.***> wrote:

can this issue be closed now?

— Reply to this email directly, view it on GitHub https://github.com/patrickchugh/terravision/issues/66#issuecomment-1831531879, or unsubscribe https://github.com/notifications/unsubscribe-auth/AAZHR2IZOMCRF23UV3SEGE3YG36IJAVCNFSM6AAAAAA6UGB5WKVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMYTQMZRGUZTCOBXHE . You are receiving this because you were mentioned.Message ID: @.***>

[dnk8n]

I had a go with the new code, thanks for your patience... I understand how exciting it is to roll out a new feature and how frustrating it is waiting for feedback!

First, my project used a backend conf for terraform init, so I needed to hardcode that for this exercise - logged an issue here: https://github.com/patrickchugh/terravision/issues/85

Then when reinstalling from scratch, I needed to manually install debugpy before I could continue.

Now that I am properly installed, I could run:

(terravision-py3.10) dean@dean-ThinkPad-X250:/tmp/terravision$ terravision draw --source ~/src/dtrack --varfile ~/src/dtrack/terraform.tfvars

It did all the right things until the end:

Plan: 12 to add, 0 to change, 0 to destroy.

─────────────────────────────────────────────────────────────────────

Saved the plan to: tfplan.bin

To perform exactly these actions, run the following command to apply:
    terraform apply "tfplan.bin"

Analysing plan..

Unprocessed terraform graph dictionary:

{
    "aws_instance.dtrack_instance": [
        "cloudflare_record.dtrack_record",
        "cloudflare_record.dtrack_traefik_record"
    ],
    "aws_internet_gateway.dtrack_igw": [
        "aws_route_table.dtrack_route_table"
    ],
    "aws_key_pair.dtrack_key_pair": [
        "aws_instance.dtrack_instance"
    ],
    "aws_route_table.dtrack_route_table": [
        "aws_route_table_association.dtrack_route_table_association"
    ],
    "aws_route_table_association.dtrack_route_table_association~1": [],
    "aws_security_group.dtrack_sg": [
        "aws_instance.dtrack_instance"
    ],
    "aws_subnet.dtrack_subnet~1": [
        "aws_instance.dtrack_instance",
        "aws_route_table_association.dtrack_route_table_association"
    ],
    "aws_vpc.dtrack_vpc": [
        "aws_internet_gateway.dtrack_igw",
        "aws_security_group.dtrack_sg",
        "aws_subnet.dtrack_subnet",
        "aws_subnet.dtrack_subnet~1"
    ],
    "cloudflare_record.dtrack_record": [],
    "cloudflare_record.dtrack_traefik_record": [],
    "github_repository_deploy_key.dtrack_deploy_key": [],
    "local_file.ansible_hosts": []
}

Parsing Terraform Source Files..
  Added Source Location: /
ERROR: No Terraform .tf files found in current directory or your source location. Use --source parameter to specify location or Github URL of source files

Even changing current directory to where my main.tf lives didn't produce a different result.

[patrickchugh]

@dnk8n Thanks for the feedback and patience. I have just made some bugfixes to this. Can you please try to run with latest code?

Working on the terraform init part separately

[dnk8n]

It gets this far, and then hangs:

Plan: 12 to add, 0 to change, 0 to destroy.

─────────────────────────────────────────────────────────────────────

Saved the plan to: /tmp/tfplan.bin

To perform exactly these actions, run the following command to apply:
    terraform apply "/tmp/tfplan.bin"

Analysing plan..

Unprocessed terraform graph dictionary:

{
    "aws_instance.dtrack_instance": [
        "cloudflare_record.dtrack_record",
        "cloudflare_record.dtrack_traefik_record"
    ],
    "aws_internet_gateway.dtrack_igw": [
        "aws_route_table.dtrack_route_table"
    ],
    "aws_key_pair.dtrack_key_pair": [
        "aws_instance.dtrack_instance"
    ],
    "aws_route_table.dtrack_route_table": [
        "aws_route_table_association.dtrack_route_table_association"
    ],
    "aws_route_table_association.dtrack_route_table_association~1": [],
    "aws_security_group.dtrack_sg": [
        "aws_instance.dtrack_instance"
    ],
    "aws_subnet.dtrack_subnet~1": [
        "aws_instance.dtrack_instance",
        "aws_route_table_association.dtrack_route_table_association"
    ],
    "aws_vpc.dtrack_vpc": [
        "aws_internet_gateway.dtrack_igw",
        "aws_security_group.dtrack_sg",
        "aws_subnet.dtrack_subnet",
        "aws_subnet.dtrack_subnet~1"
    ],
    "cloudflare_record.dtrack_record": [],
    "cloudflare_record.dtrack_traefik_record": [],
    "github_repository_deploy_key.dtrack_deploy_key": [],
    "local_file.ansible_hosts": []
}

Parsing Terraform Source Files..
  Added Source Location: /home/dean/src/dtrack
  Parsing /home/dean/src/dtrack/terraform.tfvars.tpl
  Parsing /home/dean/src/dtrack/terraform.tfvars
  Parsing /home/dean/src/dtrack/main.tf
    Found 9 variable stanza(s)
    Found 1 locals stanza(s)
    Found 12 resource stanza(s)
    Found 3 data stanza(s)
  Will use auto variables from file : /home/dean/src/dtrack/terraform.tfvars.tpl 

  Will use auto variables from file : /home/dean/src/dtrack/terraform.tfvars 

Processing variables..

Processing resources..

Not sure how it found Parsing /home/dean/src/dtrack/terraform.tfvars.tpl

[dnk8n]

Deleting the .tpl file and rerunning doesn't change the result.

[dnk8n]

How long is the process expected to take. Unfortunately in South Africa we have electricity load shedding, and my laptop battery is about to die. Will try again when back online.

[dnk8n]

Seems to just hang. 37min and still going (using 100% of one thread) ...

[patrickchugh]

@dnk8n Would you have time to git pull the latest code and try again?

[dnk8n]

Looks much better! I think enough to close this issue.

Maybe you know better than me @patrickchugh if it is showing all the available AWS resources (compare the dictionaries to the image). Seems to be missing detail?

FYI:

Saved the plan to: /tmp/tfplan.bin

To perform exactly these actions, run the following command to apply:
    terraform apply "/tmp/tfplan.bin"

Analysing plan..

Unprocessed terraform graph dictionary:

{
    "aws_instance.dtrack_instance": [
        "cloudflare_record.dtrack_record",
        "cloudflare_record.dtrack_traefik_record"
    ],
    "aws_internet_gateway.dtrack_igw": [
        "aws_route_table.dtrack_route_table"
    ],
    "aws_key_pair.dtrack_key_pair": [
        "aws_instance.dtrack_instance"
    ],
    "aws_route_table.dtrack_route_table": [
        "aws_route_table_association.dtrack_route_table_association~1"
    ],
    "aws_route_table_association.dtrack_route_table_association~1": [],
    "aws_security_group.dtrack_sg": [
        "aws_instance.dtrack_instance"
    ],
    "aws_subnet.dtrack_subnet~1": [
        "aws_instance.dtrack_instance",
        "aws_route_table_association.dtrack_route_table_association~1"
    ],
    "aws_vpc.dtrack_vpc": [
        "aws_internet_gateway.dtrack_igw",
        "aws_security_group.dtrack_sg",
        "aws_subnet.dtrack_subnet~1",
        "aws_subnet.dtrack_subnet~1"
    ],
    "cloudflare_record.dtrack_record": [],
    "cloudflare_record.dtrack_traefik_record": [],
    "github_repository_deploy_key.dtrack_deploy_key": [],
    "local_file.ansible_hosts": []
}

Parsing Terraform Source Files..
  Added Source Location: /home/dean/src/dtrack
  Parsing /home/dean/src/dtrack/terraform.tfvars
  Parsing /home/dean/src/dtrack/main.tf
    Found 9 variable stanza(s)
    Found 1 locals stanza(s)
    Found 12 resource stanza(s)
    Found 3 data stanza(s)
  Will use auto variables from file : /home/dean/src/dtrack/terraform.tfvars 

Processing variables..

Processing resources..
   aws_vpc.dtrack_vpc
   aws_subnet.dtrack_subnet
   aws_internet_gateway.dtrack_igw
   aws_route_table.dtrack_route_table
   aws_route_table_association.dtrack_route_table_association
   aws_security_group.dtrack_sg
   aws_key_pair.dtrack_key_pair
   aws_instance.dtrack_instance
   cloudflare_record.dtrack_record
   cloudflare_record.dtrack_traefik_record
   github_repository_deploy_key.dtrack_deploy_key
   local_file.ansible_hosts

Checking for additional links between 12 resources..
   aws_instance.dtrack_instance --> aws_instance.dtrack_instance
   aws_vpc.dtrack_vpc --> aws_route_table.dtrack_route_table

Enriched graphviz dictionary:

{
    "aws_az.af~south~1a": [
        "aws_subnet.dtrack_subnet~1"
    ],
    "aws_instance.dtrack_instance": [
        "aws_instance.dtrack_instance"
    ],
    "aws_internet_gateway.igw": [
        "aws_route_table.dtrack_route_table",
        "tv_aws_internet.internet"
    ],
    "aws_key_pair.dtrack_key_pair": [
        "aws_instance.dtrack_instance"
    ],
    "aws_route_table.dtrack_route_table": [
        "aws_route_table_association.dtrack_route_table_association~1"
    ],
    "aws_route_table_association.dtrack_route_table_association~1": [],
    "aws_security_group.dtrack_sg": [
        "aws_instance.dtrack_instance"
    ],
    "aws_subnet.dtrack_subnet~1": [
        "aws_route_table_association.dtrack_route_table_association~1",
        "aws_security_group.dtrack_sg"
    ],
    "aws_vpc.dtrack_vpc": [
        "aws_az.af~south~1a",
        "aws_internet_gateway.igw",
        "aws_route_table.dtrack_route_table",
        "aws_security_group.dtrack_sg",
        "aws_subnet.dtrack_subnet~1"
    ],
    "cloudflare_record.dtrack_record": [
        "aws_instance.dtrack_instance"
    ],
    "cloudflare_record.dtrack_traefik_record": [
        "aws_instance.dtrack_instance"
    ],
    "github_repository_deploy_key.dtrack_deploy_key": [],
    "local_file.ansible_hosts": [],
    "tv_aws_internet.internet": []
}

Rendering Architecture Image...
  Output file: /home/dean/src/dtrack/architecture.dot.png
  Completed!

[patrickchugh]

@dnk8n I found there are a few glitches in the diagram you got. If you can open a new issue regarding this bug handling security groups and share your code I can make it output the correct image similar to this one attached: