google_bigquery_table destroyed and recretead every time if created in dataset with default key and key not set on table resource object

[MikeMoore63]

Community Note

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
• Please do not leave +1 or me too comments, they generate extra noise for issue followers and do not help prioritize the request.
• If you are interested in working on this issue or have submitted a pull request, please leave a comment.
• If an issue is assigned to the modular-magician user, it is either in the process of being autogenerated, or is planned to be autogenerated soon. If an issue is assigned to a user, that user is claiming responsibility for the issue. If an issue is assigned to hashibot, a community member has claimed the issue already.

Terraform Version

Terraform v0.15.4 on darwin_amd64

• provider registry.terraform.io/hashicorp/google v3.90.1
• provider registry.terraform.io/hashicorp/google-beta v3.90.1
• provider registry.terraform.io/hashicorp/local v1.4.0

Affected Resource(s)

• google_bigquery_table

Terraform Configuration Files

variable "project" {
  description = "The project to deploy with"
  default     = ""
  type        = string
}
variable "assume_role" {
  description = "The service account to impersonate if reuired"
  default     = ""
  type        = string
}
variable "kmskey" {
  description = "the cmek key to use"
  default     = ""
  type        = string
}
locals {
  eimtablelist             = [
    {
      table_id          = "table1",
      schema            = jsonencode([
        {
          name = "field1",
          type = "STRING",
          mode = "NULLABLE"
        }
      ]),
      description       = null
      time_partitioning = {
        type                     = "DAY",
        field                    = null,
        require_partition_filter = false,
        expiration_ms            = 94608000000,
      },
      expiration_time   = null
      clustering        = [
      ],
      labels            = {
      }
      encryption_configuration = {
        kms_key_name =  local.bqkeyconfidentiatlitylow
      }
    },
    {
      table_id          = "table2",
      schema            = jsonencode([
        {
          name = "field1",
          type = "STRING",
          mode = "NULLABLE"
        }
      ]),
      description       = null
      time_partitioning = {
        type                     = "DAY",
        field                    = null,
        require_partition_filter = false,
        expiration_ms            = 94608000000,
      },
      expiration_time   = null
      clustering        = [
      ],
      labels            = {
      }
      encryption_configuration = {
        kms_key_name =  local.bqkeyconfidentiatlitylow
      }
    },
    {
      table_id          = "table3",
      schema            = jsonencode([
        {
          name = "field1",
          type = "STRING",
          mode = "NULLABLE"
        }
      ]),
      description       = null
      time_partitioning = {
        type                     = "DAY",
        field                    = null,
        require_partition_filter = false,
        expiration_ms            = 94608000000,
      },
      expiration_time   = null
      clustering        = [
      ],
      labels            = {
      }
      encryption_configuration = null
    }
  ]
  eimproject               = {
    region1 = var.project
  }
  eimtables                = {for table in local.eimtablelist : table["table_id"] => table}
  bqkeyconfidentiatlitylow = var.kmskey
}

terraform {
  required_providers {
    google      = {
      source  = "hashicorp/google"
      version = "~> 3.90.0"
    }
    google-beta = {
      source  = "hashicorp/google-beta"
      version = "~> 3.90.0"
    }
    local       = {
      source  = "hashicorp/local"
      version = "~> 1.4.0"
    }
  }
}

provider "google" {
  project = var.project
  scopes  = var.assume_role == "" ? [
    "https://www.googleapis.com/auth/bigquery",
    "https://www.googleapis.com/auth/pubsub",
    "https://www.googleapis.com/auth/cloud-platform.read-only",
    "https://www.googleapis.com/auth/source.read_only",
    "https://www.googleapis.com/auth/devstorage.full_control",
    "https://www.googleapis.com/auth/compute",
    "https://www.googleapis.com/auth/cloudkms"
  ] : [
    "https://www.googleapis.com/auth/cloud-platform.read-only"
  ]
}

provider "google-beta" {
  project = var.project
  scopes  = var.assume_role == "" ? [
    "https://www.googleapis.com/auth/cloud-platform"
  ] : [
    "https://www.googleapis.com/auth/cloud-platform.read-only"
  ]
}

data "google_client_config" "default" {

}

data "google_service_account_access_token" "assume_role" {
  count                  = var.assume_role == "" ? 0 : 1
  provider               = google
  target_service_account = var.assume_role
  scopes                 = [
    "https://www.googleapis.com/auth/bigquery",
    "https://www.googleapis.com/auth/pubsub",
    "https://www.googleapis.com/auth/cloud-platform",
    "https://www.googleapis.com/auth/source.read_only",
    "https://www.googleapis.com/auth/devstorage.full_control",
    "https://www.googleapis.com/auth/compute",
    "https://www.googleapis.com/auth/cloudkms"
  ]
  lifetime               = "2500s"
}

provider "google" {
  alias        = "deploy_provider"
  project      = var.project
  access_token = var.assume_role == "" ? data.google_client_config.default.access_token : data.google_service_account_access_token.assume_role[0].access_token
}

provider "google-beta" {
  alias        = "beta_deploy_provider"
  project      = var.project
  access_token = var.assume_role == "" ? data.google_client_config.default.access_token : data.google_service_account_access_token.assume_role[0].access_token
}

resource "google_bigquery_dataset" "eim_dataset" {
  provider   = google.deploy_provider
  dataset_id = "example_dataset"
  location   = "EU"
  default_encryption_configuration {
    kms_key_name = local.bqkeyconfidentiatlitylow
  }

}

resource "google_bigquery_table" "eimtables" {
  provider        = google.deploy_provider
  for_each        = local.eimtables
  dataset_id      = google_bigquery_dataset.eim_dataset.dataset_id
  friendly_name   = each.key
  table_id        = each.key
  labels          = each.value["labels"]
  schema          = each.value["schema"]
  clustering      = each.value["clustering"]
  expiration_time = each.value["expiration_time"]
  project         = local.eimproject["region1"]
  description     = each.value["description"]

  dynamic "time_partitioning" {
    for_each = each.value["time_partitioning"] != null ? [
      each.value["time_partitioning"]
    ] : []
    content {
      type                     = time_partitioning.value["type"]
      expiration_ms            = time_partitioning.value["expiration_ms"]
      field                    = time_partitioning.value["field"]
      require_partition_filter = time_partitioning.value["require_partition_filter"]
    }
  }
  dynamic "encryption_configuration" {
    for_each = each.value["encryption_configuration"] != null ?  [
      each.value["encryption_configuration"]
    ] : []
    content {
      kms_key_name = encryption_configuration.value["kms_key_name"]
    }
  }
}

Debug Output

Panic Output

Expected Behavior

table3 is created on first run and left in tact on second run

Actual Behavior

table3 is created on first run on 2nd run wants to destroy table3

Steps to Reproduce

1. Create tfvars file with a projectid and kms key that is setup for bigquery cmek usage
2. terraform apply -var-file=pathtovar.tfvars
3. terraform apply -var-file=pathtovar.tfvars

Important Factoids

References

• 0000

b/301412330

[diegosucariaG]

+1, facing the exact same issue here