aws-samples / data-lineage-for-data-lake-example

MIT No Attribution
24 stars 24 forks source link

Error: error creating Neptune Cluster: DBClusterParameterGroupNotFound: DBClusterParam #5

Open bluecrayon52 opened 1 year ago

bluecrayon52 commented 1 year ago

Using Terraform version 1.3.5, I receive the following error when attempting to run terraform apply:

Screen Shot 2022-11-23 at 11 16 02 AM
KeyonQu commented 1 year ago

I get the same error

k-kur-gss commented 1 year ago

I solved it by adding two new objects to terraform for storage, for DBCluster param group and DB param group since both seem to be required for neptune now.

mmarinm commented 1 year ago

Same issue. Adding neptune_cluster_parameter_group_name and neptune_parameter_group_name in storage.tf is what fixed it for me, code example bellow

################################################################
# S3 buckets for data lake
################################################################

module "s3_bucket_raw" {
  source = "./modules/data_lake_s3_bucket"
  name   = "data-lake-raw-${local.post_fix}"
}

resource "aws_s3_bucket_object" "employee" {
  bucket = module.s3_bucket_raw.bucket.bucket
  key    = "employee/employee.csv"
  source = "asset/sample_data/employee.csv"
  etag   = filemd5("asset/sample_data/employee.csv")
}

module "s3_bucket_curated" {
  source = "./modules/data_lake_s3_bucket"
  name   = "data-lake-curated-${local.post_fix}"
}

module "s3_bucket_aggregated" {
  source = "./modules/data_lake_s3_bucket"
  name   = "data-lake-aggregated-${local.post_fix}"
}

################################################################
# Neptune database
################################################################

locals {
  neptune_db_port = 8182
  parameter_group_name = "default.neptune1.2"
}

resource "aws_neptune_subnet_group" "default" {
  name       = "data-lineage-neptune-subnet-group"
  subnet_ids = "${aws_subnet.subnet.*.id}"
}

resource aws_security_group "sg_neptune_db" {
  name = "sg_lineage_neptune_db"
  ingress {
    from_port  = local.neptune_db_port
    to_port = local.neptune_db_port
    protocol = "tcp"
    cidr_blocks = [aws_vpc.main.cidr_block]
  }
  egress {
    from_port = 0
    protocol  = "-1"
    to_port   = 0
    cidr_blocks = ["0.0.0.0/0"]
  }
  vpc_id = aws_vpc.main.id
}

resource "aws_neptune_cluster" "default" {
  cluster_identifier                  = "data-lineage-neptune-cluster"
  skip_final_snapshot                 = true
  iam_database_authentication_enabled = false
  apply_immediately                   = true
  neptune_subnet_group_name = aws_neptune_subnet_group.default.name
  vpc_security_group_ids = [aws_security_group.sg_neptune_db.id]
  port = local.neptune_db_port
  neptune_cluster_parameter_group_name = local.parameter_group_name
}

resource "aws_neptune_cluster_instance" "data_lineage_instance" {
  identifier = "data-lineage-neptune-instance"
  cluster_identifier  = aws_neptune_cluster.default.id
  apply_immediately   = true
  instance_class      = "db.t3.medium"
  neptune_subnet_group_name = aws_neptune_subnet_group.default.name
  port = local.neptune_db_port
  neptune_parameter_group_name = local.parameter_group_name
}