search

hashicorp / terraform-provider-aws

The AWS Provider enables Terraform to manage AWS resources.
https://registry.terraform.io/providers/hashicorp/aws
Mozilla Public License 2.0
9.74k stars 9.1k forks source link

Intermittent connection reset by peer error when Terraform apply / init #23614

Open praveenprem opened 2 years ago

praveenprem commented 2 years ago

Community Note

Terraform CLI and Terraform AWS Provider Version

Terraform v1.1.6
on darwin_amd64
+ provider registry.terraform.io/hashicorp/aws v4.4.0
+ provider registry.terraform.io/hashicorp/local v2.1.0
+ provider registry.terraform.io/hashicorp/tls v3.1.0

Affected Resource(s)

Terraform Configuration Files

Please include all Terraform configurations required to reproduce the bug. Bug reports without a functional reproduction may be closed without investigation.

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "4.4.0"
    }
    tls = {
      source  = "hashicorp/tls"
      version = "3.1.0"
    }
  }
}

variable "amp_cidr" { default = "10.120.10.0/24" }
variable "service_cidr" { default = "10.120.11.0/24" }

provider "tls" {
}

provider "aws" {
  region = "eu-west-1"
}

data "aws_region" "current" {}

#=============== Monitoring Tier ===============

locals {
  amp_tags_common = {
    Name      = "AMP POC"
    Reference = "3347",
    Creator   = "Prav"
    Trier     = "Monitoring"
  }
  amp_tags_private = {
    Name      = "AMP POC Private"
  }
}

resource "aws_prometheus_workspace" "amp" {
  alias = "amp-poc"

  tags = merge(local.amp_tags_common, local.amp_tags_private)
}

resource "aws_vpc" "amp_vpc" {
  cidr_block           = var.amp_cidr
  enable_dns_hostnames = true

  tags = local.amp_tags_common
}

resource "aws_subnet" "amp_subnet" {
  vpc_id     = aws_vpc.amp_vpc.id
  cidr_block = cidrsubnet(aws_vpc.amp_vpc.cidr_block, 2, 0)

  tags = local.amp_tags_common
}

resource "aws_subnet" "amp_subnet_public" {
  vpc_id     = aws_vpc.amp_vpc.id
  cidr_block = cidrsubnet(aws_vpc.amp_vpc.cidr_block, 2, 2)

  tags = merge(local.amp_tags_common, local.amp_tags_private)
}

resource "aws_internet_gateway" "amp_igw" {
  vpc_id = aws_vpc.amp_vpc.id

  tags = local.amp_tags_common
}

resource "aws_eip" "amp_nat" {
  vpc = true

  tags = local.amp_tags_common
}

resource "aws_nat_gateway" "amp_nat" {
  subnet_id     = aws_subnet.amp_subnet.id
  allocation_id = aws_eip.amp_nat.allocation_id

  tags = local.amp_tags_common
}

resource "aws_route_table" "amp_rtb" {
  vpc_id = aws_vpc.amp_vpc.id

  route {
    cidr_block     = "0.0.0.0/0"
    nat_gateway_id = aws_nat_gateway.amp_nat.id
  }

  tags = local.amp_tags_common
}

resource "aws_route_table_association" "amp_rtb_assoc" {
  route_table_id = aws_route_table.amp_rtb.id
  subnet_id      = aws_subnet.amp_subnet.id
}

resource "aws_network_acl" "amp_nacl" {
  vpc_id = aws_vpc.amp_vpc.id

  egress {
    action     = "allow"
    from_port  = 0
    protocol   = -1
    rule_no    = 100
    to_port    = 0
    cidr_block = "0.0.0.0/0"
  }

  ingress {
    action     = "allow"
    from_port  = 0
    protocol   = -1
    rule_no    = 100
    to_port    = 0
    cidr_block = aws_vpc.service_vpc.cidr_block
  }

  tags = local.amp_tags_common
}

resource "aws_network_acl_association" "amp_nacl_assoc" {
  network_acl_id = aws_network_acl.amp_nacl.id
  subnet_id      = aws_subnet.amp_subnet.id
}

resource "aws_security_group" "amp_sg" {
  name   = local.amp_tags_common.Name
  vpc_id = aws_vpc.amp_vpc.id

  ingress {
    from_port   = 443
    protocol    = "tcp"
    to_port     = 443
    cidr_blocks = [aws_vpc.service_vpc.cidr_block]
  }

  egress {
    from_port   = 0
    protocol    = -1
    to_port     = 0
    cidr_blocks = [aws_vpc.service_vpc.cidr_block]
  }

  tags = local.service_tags
}

resource "aws_vpc_endpoint" "amp_endpoint" {
  service_name        = "com.amazonaws.${data.aws_region.current.name}.aps-workspaces"
  vpc_id              = aws_vpc.amp_vpc.id
  vpc_endpoint_type   = "Interface"
  private_dns_enabled = true
  security_group_ids  = [aws_security_group.amp_sg.id]

  tags = local.amp_tags_common
}

#=============== Service Tier ===============

locals {
  service_tags = {
    Name      = "AMP POC - Service"
    Reference = "3347",
    Creator   = "Prav"
    Trier     = "Service"
  }
}

resource "aws_vpc" "service_vpc" {
  cidr_block           = var.service_cidr
  enable_dns_hostnames = true

  tags = local.service_tags
}

resource "aws_subnet" "service_subnet" {
  vpc_id            = aws_vpc.service_vpc.id
  cidr_block        = aws_vpc.service_vpc.cidr_block
  availability_zone = "${data.aws_region.current.name}a"

  tags = local.service_tags
}

resource "aws_internet_gateway" "service_igw" {
  vpc_id = aws_vpc.service_vpc.id

  tags = local.service_tags
}

resource "aws_route_table" "service_rtb" {
  vpc_id = aws_vpc.service_vpc.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.service_igw.id
  }

  tags = local.service_tags
}

resource "aws_route_table_association" "service_rtb_assoc" {
  route_table_id = aws_route_table.service_rtb.id
  subnet_id      = aws_subnet.service_subnet.id
}

resource "aws_network_acl" "service_nacl" {
  vpc_id = aws_vpc.service_vpc.id

  ingress {
    action     = "allow"
    from_port  = 0
    protocol   = -1
    to_port    = 0
    rule_no    = 100
    cidr_block = "0.0.0.0/0"
  }

  ingress {
    action     = "allow"
    from_port  = 0
    protocol   = -1
    to_port    = 0
    rule_no    = 200
    cidr_block = aws_vpc.amp_vpc.cidr_block
  }

  egress {
    action     = "allow"
    from_port  = 0
    protocol   = -1
    rule_no    = 100
    to_port    = 0
    cidr_block = "0.0.0.0/0"
  }

  tags = local.service_tags
}

resource "aws_network_acl_association" "service_nacl_assoc" {
  network_acl_id = aws_network_acl.service_nacl.id
  subnet_id      = aws_subnet.service_subnet.id
}

resource "aws_security_group" "service_sg" {
  name   = local.service_tags.Name
  vpc_id = aws_vpc.service_vpc.id

  ingress {
    from_port        = 22
    protocol         = "tcp"
    to_port          = 22
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }

  ingress {
    from_port   = 0
    protocol    = -1
    to_port     = 0
    cidr_blocks = [aws_vpc.service_vpc.cidr_block]
  }

  ingress {
    from_port   = 0
    protocol    = -1
    to_port     = 0
    cidr_blocks = [aws_vpc.amp_vpc.cidr_block]
  }

  egress {
    from_port        = 0
    protocol         = -1
    to_port          = 0
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }

  tags = local.service_tags
}

data "aws_iam_policy_document" "assume_policy" {
  statement {
    sid = "AllowEC2ToAccessAMP"

    actions = ["sts:AssumeRole"]

    principals {
      identifiers = ["ec2.amazonaws.com"]
      type        = "Service"
    }
    effect = "Allow"
  }
}

resource "aws_iam_role" "amp_access_role" {
  name                = replace(local.service_tags.Name, " ", "_")
  assume_role_policy  = data.aws_iam_policy_document.assume_policy.json
  managed_policy_arns = ["arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess"]

  tags = local.service_tags
}

resource "aws_iam_instance_profile" "service_instance_policy" {
  name = replace(local.service_tags.Name, " ", "_")
  role = aws_iam_role.amp_access_role.name

  tags = local.service_tags
}

resource "tls_private_key" "service_key" {
  algorithm = "RSA"
  rsa_bits  = 4096
}

resource "aws_key_pair" "amp_poc_key" {
  key_name   = replace(local.service_tags.Name, " ", "-")
  public_key = tls_private_key.service_key.public_key_openssh
}

data "aws_ami" "ubuntu" {
  most_recent = true

  filter {
    name   = "name"
    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
  }

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }

  owners = ["099720109477"] # Canonical
}

resource "aws_network_interface" "service_instance" {
  subnet_id       = aws_subnet.service_subnet.id
  private_ip      = cidrhost(aws_vpc.service_vpc.cidr_block, 10)
  security_groups = [aws_security_group.service_sg.id]

  tags = local.service_tags
}

resource "aws_eip" "service_instance" {
  vpc               = true
  network_interface = aws_network_interface.service_instance.id

  tags = local.service_tags
}

resource "aws_instance" "service_instance" {
  ami               = data.aws_ami.ubuntu.id
  instance_type     = "t3.micro"
  key_name          = aws_key_pair.amp_poc_key.key_name
  availability_zone = "${data.aws_region.current.name}a"

  iam_instance_profile = aws_iam_instance_profile.service_instance_policy.name

  network_interface {
    device_index         = 0
    network_interface_id = aws_network_interface.service_instance.id
  }

  root_block_device {
    volume_size = 30
    volume_type = "gp3"

    tags = local.service_tags
  }

  user_data = local.user_data

  tags = local.service_tags
}

#=============== VCP Peering ===============

resource "aws_vpc_peering_connection" "service_to_amp" {
  peer_vpc_id = aws_vpc.amp_vpc.id
  vpc_id      = aws_vpc.service_vpc.id
  auto_accept = true

  tags = local.service_tags
}

resource "aws_route" "service_to_amp" {
  route_table_id            = aws_route_table.service_rtb.id
  destination_cidr_block    = aws_vpc.amp_vpc.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.service_to_amp.id
}

resource "aws_route" "amp_to_service" {
  route_table_id            = aws_route_table.amp_rtb.id
  destination_cidr_block    = aws_vpc.service_vpc.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.service_to_amp.id
}

#============ Outputs ===============
resource "local_file" "foo" {
  content  = tls_private_key.service_key.private_key_pem
  filename = "${path.module}/id_rsa.pem"
}

output "eip" {
  value = aws_eip.service_instance.public_ip
}

Debug Output

I'm happy to provide the trace logs, do you have a method of securing the data? Such as the use of GPG encryption.

Panic Output

There's no panic output.

Expected Behavior

Terraform should prompt for the confirmation of terraform apply. terraform plan however does not experience this issue.

Actual Behavior

RequestError: send request failed occurs during state refresh.

sandbox:():eu-west-1 ➜  (10/03 12:43) issues/3347 git:(master) ✗ TF_LOG=TRACE TF_LOG_PATH=terraform.log terraform apply
tls_private_key.service_key: Refreshing state... [id=61254b58a3d21d40475dfc9de24d063dcc5da93b]
local_file.foo: Refreshing state... [id=d74c63c2847c8c02514a7f0bf9f9f7184dc354cc]
aws_grafana_workspace.grafana: Refreshing state... [id=g-bf5302095f]
aws_vpc.service_vpc: Refreshing state... [id=vpc-04a26adfd71dee33b]
aws_vpc.amp_vpc: Refreshing state... [id=vpc-062fad650632b08a8]
aws_prometheus_workspace.amp: Refreshing state... [id=ws-b9dc386b-5ddc-4d88-ad47-fa35692defce]
aws_key_pair.amp_poc_key: Refreshing state... [id=AMP-POC---Service]
aws_eip.amp_nat: Refreshing state... [id=eipalloc-024ab182785e36efa]
aws_iam_role.grafana_access_role: Refreshing state... [id=AMP_POC]
aws_iam_role.amp_access_role: Refreshing state... [id=AMP_POC_-_Service]
aws_internet_gateway.amp_igw: Refreshing state... [id=igw-08e2ed73d8601cae9]
aws_subnet.amp_subnet: Refreshing state... [id=subnet-0b3ebc3a94a2ea2af]
aws_nat_gateway.amp_nat: Refreshing state... [id=nat-03c338bc70030dee8]
aws_route_table.amp_rtb: Refreshing state... [id=rtb-056c1f29f66bad6e7]
aws_route_table_association.amp_rtb_assoc: Refreshing state... [id=rtbassoc-01745394c110054f4]
aws_iam_instance_profile.service_instance_policy: Refreshing state... [id=AMP_POC_-_Service]
╷
│ Error: error reading EC2 VPC (vpc-04a26adfd71dee33b) ClassicLinkEnabled: RequestError: send request failed
│ caused by: Post "https://ec2.eu-west-1.amazonaws.com/": read tcp 192.168.1.162:50134->54.239.35.17:443: read: connection reset by peer
│
│   with aws_vpc.service_vpc,
│   on amp-poc.tf line 176, in resource "aws_vpc" "service_vpc":
│  176: resource "aws_vpc" "service_vpc" {
│
╵

Other instances

sandbox:():eu-west-1 ➜  (10/03 12:30) issues/3347 git:(master) ✗ terraform apply
tls_private_key.service_key: Refreshing state... [id=61254b58a3d21d40475dfc9de24d063dcc5da93b]
local_file.foo: Refreshing state... [id=d74c63c2847c8c02514a7f0bf9f9f7184dc354cc]
╷
│ Error: Request cancelled
│
│   with provider["registry.terraform.io/hashicorp/aws"],
│   on amp-poc.tf line 20, in provider "aws":
│   20: provider "aws" {
│
│ The plugin.(*GRPCProvider).ValidateProviderConfig request was cancelled.
╵
sandbox:():eu-west-1 ➜  (10/03 12:33) issues/3347 git:(master) ✗ terraform apply
tls_private_key.service_key: Refreshing state... [id=61254b58a3d21d40475dfc9de24d063dcc5da93b]
local_file.foo: Refreshing state... [id=d74c63c2847c8c02514a7f0bf9f9f7184dc354cc]
aws_grafana_workspace.grafana: Refreshing state... [id=g-bf5302095f]
aws_vpc.amp_vpc: Refreshing state... [id=vpc-062fad650632b08a8]
aws_eip.amp_nat: Refreshing state... [id=eipalloc-024ab182785e36efa]
aws_prometheus_workspace.amp: Refreshing state... [id=ws-b9dc386b-5ddc-4d88-ad47-fa35692defce]
aws_key_pair.amp_poc_key: Refreshing state... [id=AMP-POC---Service]
aws_vpc.service_vpc: Refreshing state... [id=vpc-04a26adfd71dee33b]
aws_iam_role.grafana_access_role: Refreshing state... [id=AMP_POC]
aws_iam_role.amp_access_role: Refreshing state... [id=AMP_POC_-_Service]
aws_internet_gateway.service_igw: Refreshing state... [id=igw-019e8b76a2d313f54]
aws_subnet.service_subnet: Refreshing state... [id=subnet-0991aefb2c6bea785]
aws_route_table.service_rtb: Refreshing state... [id=rtb-0c6ce1a790c267016]
aws_route_table_association.service_rtb_assoc: Refreshing state... [id=rtbassoc-06d75781666989dbb]
aws_iam_instance_profile.service_instance_policy: Refreshing state... [id=AMP_POC_-_Service]
╷
│ Error: error reading EC2 VPC (vpc-062fad650632b08a8) ClassicLinkEnabled: RequestError: send request failed
│ caused by: Post "https://ec2.eu-west-1.amazonaws.com/": read tcp 192.168.1.162:49880->54.239.39.130:443: read: connection reset by peer
│
│   with aws_vpc.amp_vpc,
│   on amp-poc.tf line 46, in resource "aws_vpc" "amp_vpc":
│   46: resource "aws_vpc" "amp_vpc" {
│
╵
╷
│ Error: RequestError: send request failed
│ caused by: Post "https://ec2.eu-west-1.amazonaws.com/": read tcp 192.168.1.162:49883->54.239.39.130:443: read: connection reset by peer
│
│   with data.aws_ami.ubuntu,
│   on amp-poc.tf line 327, in data "aws_ami" "ubuntu":
│  327: data "aws_ami" "ubuntu" {
│
╵

Steps to Reproduce

  1. terraform init
  2. terraform apply

Important Factoids

References

praveenprem commented 2 years ago

Update

I got a colleague to run this exact same thing on x86 (Intel i5) and it worked the first time.

justinretzolk commented 2 years ago

Hey @praveenprem 👋 Thank you for taking the time to raise this, and for the additional follow up that your colleague didn't run into the same issues. In this case, I think the debug logs would be quite helpful. As far as your question around securely sharing those logs, the best bet would be to use our public GPG key to encrypt them. The key may be found on keybase at: https://keybase.io/hashicorp

praveenprem commented 2 years ago

Hey @justinretzolk, Thanks for the reply.

Here are the trace logs for the issue I had yesterday. terraform.log.gpg.zip

I'm having much better luck today but still, get errors from time to time.

praveenprem commented 2 years ago

Update

Hi @justinretzolk, thought this might interest you. We've discovered a weird situation where the same colleague also got similar errors today on an x86 machine and the common factor was that we both get these errors on BT ISP (UK). Whereas last time my colleague tried it on a different ISP (Three) where it worked for him.

He got the same connection reset by peer error on the AWS Provider and I believe that this is caused by the AWS side, although it's not isolated to a single BT connection as I get this on our office network and my home network where they both provided by BT.

matanr-orca commented 2 years ago

is there a way to do retries? it happens to me and screwing with automation

vineetsharma883 commented 1 year ago

do we know the reason for it ? I am facing same issue at some terraform init runs(not in every case)

praveenprem commented 1 year ago

@vineetsharma883 My guess would be that AWS has changed their API rate limits which cause this issue. So there's nothing we can do other than run Terraform within an EC2 instance that might bypass the public API rate limits.

@matanr-orca This could be something you can explore with automation.

ronjouch commented 1 year ago

Same trouble today with a terraform init -reconfigure -backend=true : I 100% reliably Error refreshing state: Failed to read remote state: read tcp 192.168.1.154:59696->52.216.99.53:443: read: connection reset by peer

Running with TF_LOG=debug, I see that:

  1. Several sts.amazonaws.com commands (sts/GetCallerIdentity) succeed!
  2. Several s3.amazonaws.com commands (s3/ListObjects, s3/ListObjects) succeed!
  3. ... but something barfs up later down the line: the init hangs ... then after a couple minutes and without much context I get the above-mentioned error message (with no idea of what AWS command caused it! It just pops up immediately after a 2023-05-15T05:35:16.771-0400 [DEBUG] [aws-sdk-go] line, with no details of what aws-sdk-go was trying to do 😕)

Extra info confirming above-reported inconsistency:

  1. I had no problem running the same command last week
  2. A colleague (same country, different city/ISP) has no problem running the command today

A nit, finally: @vineetsharma883 can you update this issue title to end with when Terraform apply / init ? (Adding / init)

ronjouch commented 1 year ago

Fixed my case! Mine was quite dumb, and although I can't confirm it's the reason for all the upvoters of https://github.com/hashicorp/terraform-provider-aws/issues/23614 and https://github.com/hashicorp/terraform-provider-aws/issues/14163 , given the number of upvotes they have, I'm pretty sure at least a couple had the same problem than I had. Hold your beer, it's pretty dumb. Are you ready?

My wifi was dog slow (<1kbps), causing reproducible failure/timeout to download $work's large (2MB) terraform.tfstate hosted on S3 :facepalm: . Most small REST calls to AWS worked (which was confusing!), but downloading this large file was too slow and broke my tf command. After fixing my wifi, no more errors.

→ @justinretzolk again, zarroo guarantee/clue that it's the single root for this issue. That being said, given the critical-ness of being able to download the tfstate, what about adding a lil' "quality of life" / "helping users not shooting themselves in the foot" feature that:

  1. Either detects slow downloads, and warns if tfstate cannot be downloaded reasonably fast ...
  2. ... or at least wraps the tfstate download in a try/catch (or golang equivalent, pardon my non-golangness), leaving the current net/http "connection reset by peer" behavior as-is ... but instead of aborting with a cryptic connection reset by peer error (when doing what? calling what URL), printing some kind of user-helpful error like Failed to download state file <uri_to_tfstate>, aborting. Maybe you, or your ISP, or your hosting provider, has network quality/speed trouble.

Thanks for terraform and tf-provider-aws 🙂.

justinretzolk commented 1 year ago

Hey @ronjouch 👋 Thank you for following up! What an interesting situation! 😅 For that sort of a request, you'd want to file an issue in the Terraform Core repository, since downloading the state is part of Core rather than the AWS provider (and given the situation, I'd imagine this could impact other backends as well).

64kramsystem commented 1 year ago

In my case, this is due to rate limiting at some level (not sure if AWS, internet provider or other).

My solution has been to limit the Terraform operations parallelism; 10 is the default, and 8 (-parallelism=8) worked for me.

Hope it helps somebody 😄

ravimalvia10 commented 4 months ago

In my case, this is due to rate limiting at some level (not sure if AWS, internet provider or other).

My solution has been to limit the Terraform operations parallelism; 10 is the default, and 8 (-parallelism=8) worked for me.

Hope it helps somebody 😄

@64kramsystem thank you for the solution

64kramsystem commented 4 months ago

In my case, this is due to rate limiting at some level (not sure if AWS, internet provider or other). My solution has been to limit the Terraform operations parallelism; 10 is the default, and 8 (-parallelism=8) worked for me. Hope it helps somebody 😄

@64kramsystem thank you for the solution

Some time after, I've upgraded the AWS provider, and didn't need to limit the parallelism anymore :smile:

mayank0202 commented 4 months ago

I am using aws provider 4.67 and yesterday only i start getting this connection reset by peer and tfstate is not able to refresh properly. Any solution for this? Is it an issue from aws side ?

praveenprem commented 4 months ago

I am using aws provider 4.67 and yesterday only i start getting this connection reset by peer and tfstate is not able to refresh properly. Any solution for this? Is it an issue from aws side ?

@mayank0202 I suspect that there are too many resources in the TF config, I've been working on smaller TF modules and haven't seen this issue lately. It's only those with 100+ resources getting this issue.

mayank0202 commented 4 months ago

I am using aws provider 4.67 and yesterday only i start getting this connection reset by peer and tfstate is not able to refresh properly. Any solution for this? Is it an issue from aws side ?

@mayank0202 I suspect that there are too many resources in the TF config, I've been working on smaller TF modules and haven't seen this issue lately. It's only those with 100+ resources getting this issue.

@praveenprem I faced this issue where my module has to deploy only 9-10 resources and my team also faced the same issue so thats why i thought if there maybe an issue from aws side

praveenprem commented 3 months ago

@mayank0202 Does it happen on an ISP that only you're calling the AWS API as well? If you're getting the error at work where other people are also running Terraform, AWS is probably throttling the work networks' public IP.

If you've access to a VPN provider I'd try that at work to disassociate from the work public IP to test the theory. That worked for me when I had issues with larger scripts I had problems with.

jayanthankj commented 3 months ago

We get the following error while running terraform apply in a github action during terraform apply

Error saving state: failed to upload state: RequestError: send request failed
caused by: Put “https://s3-state-file-location”:
tcp read: connection reset by peer

I don't think its related to ISP in my case.

Our Configuration: Terraform - 1.5 AWS provider version - 5.31

Any solution for this?

praveenprem commented 3 months ago

@jayanthankj Unfortunately, we don't know exactly why it's happening. Rate limit is just a theory that's most likely to happen. The only solution to this is to run Terraform from an EC2 instance as that seems to not have this problem.

You can try running a private worker in an EC2 and testing it.

AndriyDmytrenko commented 1 month ago

I have a very similar behaviour in Azure (Azure VM as a build agent, azurerm is a provider). I mean, it's probably not a provider level, but terraform engine level issue.