pulumi / pulumi-aws-native

AWS Native Provider for Pulumi
Apache License 2.0
91 stars 17 forks source link

EC2 SecurityGroup{,Ingress,Egress} are flaky #1592

Open ixti opened 1 week ago

ixti commented 1 week ago

What happened?

Results are unpredictable, and may break the configuration with each run.

After running Pulumi application it may create some security group rules, may create some that are not even defined (egress to any IPv4), or delete some that were previously created - while thinking those resources are still in place.

Example

import * as aws from "@pulumi/aws-native";

// const provider = new aws.Provider("...", ...);
// const vpc      = new aws.ec2.Vpc("...", ...);

const albSecurityGroup = new awsNative.ec2.SecurityGroup("alb-sg", {
  groupDescription: "Managed by Pulumi",
  groupName:        "alb-sg",
  vpcId:            vpc.id
}, {
  provider: provider
});

const webSecurityGroup = new awsNative.ec2.SecurityGroup("web-sg", {
  groupDescription: "Managed by Pulumi",
  groupName:        "web-sg",
  vpcId:            vpc.id
}, {
  provider: provider
});

new awsNative.ec2.SecurityGroupIngress("alb-sg-http-ingress", {
  description: "Allow HTTP traffic from anywhere",
  groupId:     albSecurityGroup.id,
  fromPort:    80,
  toPort:      80,
  ipProtocol:  "tcp",
  cidrIp:      "0.0.0.0/0"
}, {
  provider: provider
});

new awsNative.ec2.SecurityGroupIngress("alb-sg-https-ingress", {
  description: "Allow HTTPS traffic from anywhere",
  groupId:     albSecurityGroup.id,
  fromPort:    443,
  toPort:      443,
  ipProtocol:  "tcp",
  cidrIp:      "0.0.0.0/0"
}, {
  provider: provider
});

new awsNative.ec2.SecurityGroupEgress("alb-sg-web-egress", {
  description:                "Allow initiate traffic to web-sg",
  groupId:                    albSecurityGroup.id,
  fromPort:                   8080,
  toPort:                     8080,
  ipProtocol:                 "tcp",
  destinationSecurityGroupId: webSecurityGroup.id
}, {
  provider: provider
});

new awsNative.ec2.SecurityGroupIngress("web-sg-alb-ingress", {
  description:           "Allow traffic from alb-sg",
  groupId:               webSecurityGroup.id,
  fromPort:              8080,
  toPort:                8080,
  ipProtocol:            "tcp",
  sourceSecurityGroupId: albSecurityGroup.id
}, {
  provider: provider
});

Output of pulumi about

CLI          
Version      3.119.0
Go Version   go1.22.3
Go Compiler  gc

Plugins
KIND      NAME        VERSION
resource  aws         6.41.0
resource  aws-native  0.108.4
language  nodejs      unknown

Host     
OS       gentoo
Version  2.15
Arch     x86_64

This project is written in nodejs: executable='/run/user/1000/fnm_multishells/31420_1719185505246/bin/node' version='v20.10.0'

Current Stack: organization/xxx/staging

TYPE                                        URN
pulumi:pulumi:Stack                         urn:pulumi:staging::xxx::pulumi:pulumi:Stack::xxx-staging
pulumi:providers:aws-native                 urn:pulumi:staging::xxx::pulumi:providers:aws-native::aws-native-us-west-2-provider
aws-native:ec2:InternetGateway              urn:pulumi:staging::xxx::aws-native:ec2:InternetGateway::igw
aws-native:ec2:Vpc                          urn:pulumi:staging::xxx::aws-native:ec2:Vpc::vpc
aws-native:ec2:Eip                          urn:pulumi:staging::xxx::aws-native:ec2:Eip::eip-2
aws-native:ec2:Eip                          urn:pulumi:staging::xxx::aws-native:ec2:Eip::eip-1
aws-native:ec2:Subnet                       urn:pulumi:staging::xxx::aws-native:ec2:Subnet::public-subnet-2
aws-native:ec2:Subnet                       urn:pulumi:staging::xxx::aws-native:ec2:Subnet::public-subnet-1
aws-native:ec2:VpcGatewayAttachment         urn:pulumi:staging::xxx::aws-native:ec2:VpcGatewayAttachment::igw-attachment
aws-native:ec2:Subnet                       urn:pulumi:staging::xxx::aws-native:ec2:Subnet::private-subnet-1
aws-native:ec2:Subnet                       urn:pulumi:staging::xxx::aws-native:ec2:Subnet::private-subnet-2
aws-native:ec2:SecurityGroup                urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup::svc-sg
aws-native:ec2:NatGateway                   urn:pulumi:staging::xxx::aws-native:ec2:NatGateway::ngw-1
aws-native:ec2:RouteTable                   urn:pulumi:staging::xxx::aws-native:ec2:RouteTable::public-rt-1
aws-native:ec2:NatGateway                   urn:pulumi:staging::xxx::aws-native:ec2:NatGateway::ngw-2
aws-native:ec2:RouteTable                   urn:pulumi:staging::xxx::aws-native:ec2:RouteTable::public-rt-2
aws-native:ec2:SecurityGroup                urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup::web-sg
aws-native:ec2:RouteTable                   urn:pulumi:staging::xxx::aws-native:ec2:RouteTable::private-rt-1
aws-native:ec2:SecurityGroup                urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup::redis-sg
aws-native:ec2:Route                        urn:pulumi:staging::xxx::aws-native:ec2:Route::public-route-1
aws-native:ec2:SubnetRouteTableAssociation  urn:pulumi:staging::xxx::aws-native:ec2:RouteTable$aws-native:ec2:SubnetRouteTableAssociation::public-rt-1-assoc
aws-native:ec2:RouteTable                   urn:pulumi:staging::xxx::aws-native:ec2:RouteTable::private-rt-2
aws-native:ec2:SecurityGroupEgress          urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupEgress::svc-sg-any-egress
aws-native:ec2:SubnetRouteTableAssociation  urn:pulumi:staging::xxx::aws-native:ec2:RouteTable$aws-native:ec2:SubnetRouteTableAssociation::public-rt-2-assoc
aws-native:ec2:Route                        urn:pulumi:staging::xxx::aws-native:ec2:Route::public-route-2
aws-native:ec2:SecurityGroup                urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup::postgres-sg
aws-native:ec2:Route                        urn:pulumi:staging::xxx::aws-native:ec2:Route::private-route-1
aws-native:ec2:SecurityGroupEgress          urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupEgress::web-sg-any-egress
aws-native:ec2:SubnetRouteTableAssociation  urn:pulumi:staging::xxx::aws-native:ec2:RouteTable$aws-native:ec2:SubnetRouteTableAssociation::private-rt-1-assoc
aws-native:ec2:SecurityGroup                urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup::alb-sg
aws-native:ec2:SubnetRouteTableAssociation  urn:pulumi:staging::xxx::aws-native:ec2:RouteTable$aws-native:ec2:SubnetRouteTableAssociation::private-rt-2-assoc
aws-native:ec2:Route                        urn:pulumi:staging::xxx::aws-native:ec2:Route::private-route-2
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::redis-sg-dmz-ingress
aws-native:ec2:SecurityGroupEgress          urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupEgress::redis-sg-dmz-egress
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::redis-sg-svc-ingress
aws-native:ec2:SecurityGroupEgress          urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupEgress::postgres-sg-dmz-egress
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::postgres-sg-svc-ingress
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::postgres-sg-dmz-ingress
aws-native:ec2:SecurityGroupEgress          urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupEgress::alb-sg-web-egress
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::alb-sg-https-ingress
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::alb-sg-http-ingress
aws-native:ec2:SecurityGroupIngress         urn:pulumi:staging::xxx::aws-native:ec2:Vpc$aws-native:ec2:SecurityGroup$aws-native:ec2:SecurityGroupIngress::web-sg-alb-ingress

Found no pending operations associated with staging

Backend        
Name           evil-eurasier
URL            s3://***
User           ixti
Organizations  
Token type     personal

Dependencies:
NAME                VERSION
@pulumi/aws-native  0.108.4
@pulumi/aws         6.41.0
@pulumi/pulumi      3.121.0
@types/node         20.14.8

Pulumi locates its logs in /tmp by default

Additional context

No response

Contributing

Vote on this issue by adding a 👍 reaction. To contribute a fix for this issue, leave a comment (and link to your pull request, if you've opened one already).

corymhall commented 6 days ago

@ixti thanks for creating this issue! After looking into this I think you may be running up against some default behavior of CloudFormation/CCAPI (which aws-native uses under the hood). By default when you create a security group via API a default egress-all rule will be created for you. For aws-native the only way to prevent that from happening is to provide a different rule instead. As long as one rule is provided, the default egress-all rule will not be created.

Also, the securityGroupEgress property on the SecurityGroup resource does not work well with the separate SecurityGroupEgress resource. Using both together can lead to unexpected rule deletion.

Below is an example of using securityGroupEgress with a dummy rule (doesn't allow any traffic, but prevents the egress-alll rule from being created) along with ignoreChanges. The ignoreChanges is necessary in order to work with the SecurityGroupEgress resource.

const albSecurityGroup = new awsNative.ec2.SecurityGroup(
  'alb-sg',
  {
    groupDescription: 'Managed by Pulumi',
    groupName: 'alb-sg',
    vpcId: vpc.id,
    securityGroupEgress: [
      {
        cidrIp: '255.255.255.255/32',
        ipProtocol: 'icmp',
        fromPort: 252,
        toPort: 86,
        description: 'Disallow all traffic',
      },
    ],
  },
  {
    provider: provider,
    ignoreChanges: ['securityGroupEgress', 'securityGroupIngress'],
  },
);

We should definitely document this, so I'll leave this issue open to track that progress.

ixti commented 2 days ago

@corymhall the problem is that I need to define some sort of DMZ where peers of the same SG can talk to each other freely, and additionally allow ingress from different SG.

I have:

const postgresSecurityGroup = new awsNative.ec2.SecurityGroup(name, {
  groupDescription:     "Managed by Pulumi",
  groupName:            "postgres-sg",
  securityGroupEgress:  [],
  securityGroupIngress: [],
  vpcId:                vpc.id
}, {
  parent:   vpc,
  provider: provider
});

new awsNative.ec2.SecurityGroupIngress("postgres-sg-svc-ingress", {
  description:           "Allow Postgres traffic from workers",
  groupId:               postgresSecurityGroup.id,
  fromPort:              5432,
  toPort:                5432,
  ipProtocol:            "tcp",
  sourceSecurityGroupId: workerSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup, svcSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider
});

new awsNative.ec2.SecurityGroupIngress("postgres-sg-dmz-ingress", {
  description:           "Allow any traffic within the security group perimeter",
  groupId:               postgresSecurityGroup.id,
  fromPort:              0,
  toPort:                0,
  ipProtocol:            "-1",
  sourceSecurityGroupId: postgresSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider
});

new awsNative.ec2.SecurityGroupEgress("postgres-sg-dmz-egress", {
  description:                "Allow any traffic within the security group perimeter",
  groupId:                    postgresSecurityGroup.id,
  fromPort:                   0,
  toPort:                     0,
  ipProtocol:                 "-1",
  destinationSecurityGroupId: postgresSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider
});

And it randomly deletes rules :((

ixti commented 2 days ago

Probably, the problem that I'm having in Pulumi.yaml:

options:
  refresh: always
ixti commented 2 days ago

Frankly, pulumi refresh on aws-native is really weird - constantly lots of warnings :((

ixti commented 2 days ago

I was able to make it stable by changing code to look like:

const postgresSecurityGroup = new awsNative.ec2.SecurityGroup(name, {
  groupDescription:     "Managed by Pulumi",
  groupName:            "postgres-sg",
  securityGroupEgress:  [],
  securityGroupIngress: [],
  vpcId:                vpc.id
}, {
  ignoreChanges: ["securityGroupIngress", "securityGroupEgress"],
  parent:        vpc,
  provider:      provider
});

new awsNative.ec2.SecurityGroupIngress("postgres-sg-svc-ingress", {
  description:           "Allow Postgres traffic from workers",
  groupId:               postgresSecurityGroup.id,
  fromPort:              5432,
  toPort:                5432,
  ipProtocol:            "tcp",
  sourceSecurityGroupId: workerSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup, svcSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider,
  replaceOnChanges: ["*"]
});

new awsNative.ec2.SecurityGroupIngress("postgres-sg-dmz-ingress", {
  description:           "Allow any traffic within the security group perimeter",
  groupId:               postgresSecurityGroup.id,
  fromPort:              0,
  toPort:                0,
  ipProtocol:            "-1",
  sourceSecurityGroupId: postgresSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider,
  replaceOnChanges: ["*"]
});

new awsNative.ec2.SecurityGroupEgress("postgres-sg-dmz-egress", {
  description:                "Allow any traffic within the security group perimeter",
  groupId:                    postgresSecurityGroup.id,
  fromPort:                   0,
  toPort:                     0,
  ipProtocol:                 "-1",
  destinationSecurityGroupId: postgresSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider,
  replaceOnChanges: ["*"]
});

UPDATE: But that creates default allow anywhere egress :((

corymhall commented 4 hours ago

UPDATE: But that creates default allow anywhere egress :((

@ixti Unfortunately the default allow-all egress rule is something AWS does by default (even if you use the CLI aws ec2 create-security-group). You only have two options for preventing that.

  1. After you create the security group revoke the egress-all rule (i.e. aws ec2 revoke-security-group-egress)
  2. When you create the security group rule, include a dummy egress rule. The key here is that it has to be done as part of the security group creation in the securityGroupEgress property. You can later add explicit egress rules via the SecurityGroupEgress resource.

So for your use case it would look like this

const postgresSecurityGroup = new awsNative.ec2.SecurityGroup(name, {
  groupDescription:     "Managed by Pulumi",
  groupName:            "postgres-sg",
  securityGroupEgress:  [
      {
        cidrIp: '255.255.255.255/32',
        ipProtocol: 'icmp',
        fromPort: 252,
        toPort: 86,
        description: 'Disallow all traffic',
      },
    ],,
  securityGroupIngress: [],
  vpcId:                vpc.id
}, {
  ignoreChanges: ["securityGroupIngress", "securityGroupEgress"],
  parent:        vpc,
  provider:      provider
});

new awsNative.ec2.SecurityGroupIngress("postgres-sg-svc-ingress", {
  description:           "Allow Postgres traffic from workers",
  groupId:               postgresSecurityGroup.id,
  fromPort:              5432,
  toPort:                5432,
  ipProtocol:            "tcp",
  sourceSecurityGroupId: workerSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup, svcSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider,
  replaceOnChanges: ["*"]
});

new awsNative.ec2.SecurityGroupIngress("postgres-sg-dmz-ingress", {
  description:           "Allow any traffic within the security group perimeter",
  groupId:               postgresSecurityGroup.id,
  fromPort:              0,
  toPort:                0,
  ipProtocol:            "-1",
  sourceSecurityGroupId: postgresSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider,
  replaceOnChanges: ["*"]
});

new awsNative.ec2.SecurityGroupEgress("postgres-sg-dmz-egress", {
  description:                "Allow any traffic within the security group perimeter",
  groupId:                    postgresSecurityGroup.id,
  fromPort:                   0,
  toPort:                     0,
  ipProtocol:                 "-1",
  destinationSecurityGroupId: postgresSecurityGroup.id
}, {
  dependsOn:        [postgresSecurityGroup],
  parent:           postgresSecurityGroup,
  provider:         provider,
  replaceOnChanges: ["*"]
});