shadycuz
commented
1 year ago Thanks for reporting this issue, I will look into it soon.
Closed cloudgprabhu closed 1 year ago
shadycuz
commented
1 year ago Thanks for reporting this issue, I will look into it soon.
thoroc
commented
1 year ago I am encountering the same issue with a custom stack (2311 lines). Could it be the size of the template causing this issue?
cf2tf, version 0.5.0
shadycuz
commented
1 year ago @thoroc It could be, I use a recursive function to get to the very last key in the dictionary and then work my way back up the dict. For really large templates it could be enough to hit the default python stack depth.
shadycuz
commented
1 year ago @thoroc Do you get the same results with the previous version of cf2tf?
EDIT: Ignore this @thoroc
shadycuz
commented
1 year ago So I did some digging into this and found where the issue is happening.
For whatever reason valid_attributes is coming back as empty I think. So when we go to iterate to pull the first item out of the list:
first_attr = next(iter(valid_attributes)) We get the StopIteration exception. I wonder why I didn't use valid_attributes[0]? It would have still caused the same crash.
I think there are two things we want to do.
valid_attributes is a list with items in it, if it doesn't we should just comment out this line and continue converting. Would like to work on this soon as fixing this would really help make most templates to "finish" converting, even if that conversion was very poor.
thoroc
commented
1 year ago Thank you for looking at this @shadycuz. For what is worth here is the stack trace I have retrieved on that last run:
// Converting cf-bip-onboarding-api-st-main.yaml to Terraform!
Traceback (most recent call last):
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/bin/cf2tf", line 8, in <module>
sys.exit(cli())
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/click/core.py", line 1130, in __call__
return self.main(*args, **kwargs)
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/app.py", line 44, in cli
config = TemplateConverter(tmpl_path.stem, cf_template, search_manger).convert()
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/convert.py", line 92, in convert
tf_resources = self.convert_to_tf(self.manifest)
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/convert.py", line 139, in convert_to_tf
tf_resources.extend(converter(resources))
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/convert.py", line 316, in convert_resources
resolved_values = self.resolve_values(
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/convert.py", line 172, in resolve_values
data[key] = self.resolve_values(
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/convert.py", line 167, in resolve_values
return functions.ref(self, value)
File "/Users/thoroc/Library/Caches/pypoetry/virtualenvs/test-cf2tf-Sw9GAXey-py3.10/lib/python3.10/site-packages/cf2tf/conversion/expressions.py", line 834, in ref
first_attr = next(iter(valid_attributes))
StopIteration
cloudgprabhu
commented
1 year ago Is there any work around to fix this bug?
shadycuz
commented
1 year ago @cloudgprabhu This has been fixed in #192 and will be released soon, for now here is your template:
// Existing Terraform src code found at /tmp/terraform_src.
data "aws_caller_identity" "current" {}
data "aws_region" "current" {}
locals {
mappings = {
SourceCode = {
General = {
S3Bucket = "%%BUCKET_NAME%%"
KeyPrefix = "%%SOLUTION_NAME%%/%%VERSION%%"
Version = "%%VERSION%%"
}
}
Send = {
AnonymousUsage = {
Data = "yes"
}
}
}
}
variable cross_account_role_name {
description = "(Required) The CrossAccount Role Name that exists in all of the Member Accounts (ex: OrganizationAccountAccessRole)"
type = string
}
variable language {
description = "AWS Trusted Advisor Explorer currently supports English ('en') only."
type = string
default = "en"
}
variable report_schedule {
description = "(Required) Schedule for gathering trusted advisor recommendations and/or tag data, ex: cron(0 9 1 * ? *) see Link:https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#CronExpressions"
type = string
default = "cron(0 9 1 * ? *)"
}
variable glue_crawler_schedule {
description = "(Required) Schedule for updating the trusted advisor recommendations data lake with new data; Please set this to 2 hours post the ReportSchedule, ex: cron(0 11 1 * ? *) see Link:https://docs.aws.amazon.com/glue/latest/dg/monitor-data-warehouse-schedule.html"
type = string
default = "cron(0 11 1 * ? *)"
}
variable interested_tag_keys {
description = "(Optional) Tags that you would like to extract Ex: env,costcenter,asset_id,etc .."
type = string
}
variable sns_email {
description = "(Required) The email address to alert when Trusted Advisor Data is refreshed."
type = string
}
variable log_level {
description = "Log Level on Lambda Functions. Use INFO for Troubleshooting purposes only."
type = string
default = "ERROR"
}
variable mask_account_information {
description = "Setting this to true will mask Account Id, Account Name & Email saved to Logs"
type = string
default = "true"
}
resource "aws_lambda_function" "solution_helper" {
handler = "solution-helper.lambda_handler"
role = aws_iam_role.solution_helper_lambda_execution_role.arn
description = "This function creates a CloudFormation custom lambda resource that creates custom lambda functions by finding and replacing specific values from existing lambda function code."
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "solution-helper.zip"])
}
runtime = "python3.8"
timeout = 300
}
resource "aws_iam_role" "solution_helper_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "solution_helper_lambda_execution_policy" {
name = "AWSTrustedAdEx-SolutionHelperLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.solution_helper_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs: CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs: CreateLogStream",
"logs: PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.solution_helper.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.solution_helper.arn])
]
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_mskconnect_custom_plugin" "uuid_generator" {
// CF Property(ServiceToken) = aws_lambda_function.solution_helper.arn
// CF Property(Region) = data.aws_region.current.name
}
resource "aws_s3_bucket" "aws_trusted_advisor_explorer_bucket_access_logging" {
bucket = {
ServerSideEncryptionConfiguration = [
{
ServerSideEncryptionByDefault = {
SSEAlgorithm = "AES256"
}
}
]
}
// CF Property(PublicAccessBlockConfiguration) = {
// BlockPublicAcls = True
// BlockPublicPolicy = True
// IgnorePublicAcls = True
// RestrictPublicBuckets = True
// }
acl = "log-delivery-write"
tags = {
ProjectName = "AWS Trusted Advisor Explorer"
}
}
resource "aws_s3_bucket" "s3_bucket" {
bucket = {
ServerSideEncryptionConfiguration = [
{
ServerSideEncryptionByDefault = {
SSEAlgorithm = "AES256"
}
}
]
}
logging = {
DestinationBucketName = aws_s3_bucket.aws_trusted_advisor_explorer_bucket_access_logging.id
}
// CF Property(PublicAccessBlockConfiguration) = {
// BlockPublicAcls = True
// BlockPublicPolicy = True
// IgnorePublicAcls = True
// RestrictPublicBuckets = True
// }
acl = "private"
tags = {
ProjectName = "AWS Trusted Advisor Explorer"
}
}
resource "aws_lambda_function" "extract_ta_data" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "extract-ta-data-lambda.zip"])
}
role = aws_iam_role.extract_ta_data_lambda_execution_role.arn
environment {
variables = {
S3BucketName = aws_s3_bucket.s3_bucket.id
IAMRoleName = var.cross_account_role_name
Header_1e93e4c0b5 = "Status,Zone,Instance Type,Platform,Instance Count,Current Monthly Cost,Estimated Monthly Savings,Expiration Date,Reserved Instance Id,Reason"
Header_51fC20e7I2 = "Status,Hosted Zone Name,Hosted Zone Id,Resource Record Set Name,Resource Record Set Type"
Header_DAvU99Dc4C = "Status,Region,Volume Id,Volume Name,Volume Type,Volume Size,Monthly Storage Cost,Snapshot Id,Snapshot Name,Snapshot Age"
Header_G31sQ1E9U = "Status,Region,Cluster,Instance Type,Reason,Estimated Monthly Savings"
Header_Qch7DwouX1 = "Status,Region,AZ,Instance Id,Instance Name,Instance Type,Estimated Monthly Savings,Day1,Day2,Day3,Day4,Day5,Day6,Day7,Day8,Day9,Day10,Day11,Day12,Day13,Day14 Latest Day,14-Day Average CPU Utilization,14-Day Average Network I/O,Number of Days Low Utilization"
Header_Ti39halfu8 = "Status,Region,DB Instance Name,Multi-AZ,Instance Type,Storage Provisioned GB,Days Since Last Connection,Estimated Monthly Savings On Demand"
Header_Z4AUBRNSmz = "Status,Region,IP Address"
Header_cX3c2R1chu = "Status,Region,Instance Type,Platform,Recommended Number of RIs to Purchase,Expected Average RI Utilization,Estimated Savings with Recommendation Monthly,Upfront Cost of RIs,Estimated cost of RIs Monthly,Estimated On-Demand Cost Post Recommended RI Purchase Monthly,Estimated Break Even Months,Lookback Period Days,Term Years"
Header_hjLMh88uM8 = "Status,Region,Load Balancer Name,Reason,Estimated Monthly Savings"
Schema_1e93e4c0b5 = "0,1,2,3,4,5,6,7,8,9"
Schema_51fC20e7I2 = "status,0,1,2,3"
Schema_DAvU99Dc4C = "status,0,1,2,3,4,5,6,7,8"
Schema_G31sQ1E9U = "0,1,2,3,4,5"
Schema_Qch7DwouX1 = "status,region,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21"
Schema_Ti39halfu8 = "status,0,1,2,3,4,5,6"
Schema_Z4AUBRNSmz = "status,0,1"
Schema_cX3c2R1chu = "status,0,1,2,3,4,5,6,7,8,9,10,11"
Schema_hjLMh88uM8 = "status,0,1,2,3"
LOG_LEVEL = var.log_level
Header_Summary = "CheckId,Status,ResourcesProcessed,ResourcesFlagged,ResourcesIgnored,ResourcesSuppressed,EstimatedMonthlySavings,EstimatedPercentMonthlySavings"
MASK_PII = var.mask_account_information
}
}
timeout = 300
handler = "extract-ta-data-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 256
}
resource "aws_iam_role" "extract_ta_data_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "extract_ta_data_lambda_execution_policy" {
name = "ExtractTADataLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.extract_ta_data_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.extract_ta_data.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.extract_ta_data.arn])
]
},
{
Effect = "Allow"
Action = [
"s3:PutObject",
"s3:PutObjectAcl"
]
Resource = join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"])
},
{
Effect = "Allow"
Action = "sts:AssumeRole"
Resource = join("", ["arn:aws:iam::*:role/", var.cross_account_role_name])
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_lambda_function" "verify_ta_check_status_lambda" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "verify-ta-check-status-lambda.zip"])
}
role = aws_iam_role.verify_ta_check_status_lambda_execution_role.arn
environment {
variables = {
IAMRoleName = var.cross_account_role_name
LOG_LEVEL = var.log_level
MASK_PII = var.mask_account_information
}
}
timeout = 60
handler = "verify-ta-check-status-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "verify_ta_check_status_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "verify_ta_check_status_lambda_execution_policy" {
name = "VerifyTACheckStatusLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.verify_ta_check_status_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.verify_ta_check_status_lambda.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.verify_ta_check_status_lambda.arn])
]
},
{
Effect = "Allow"
Action = "sts:AssumeRole"
Resource = join("", ["arn:aws:iam::*:role/", var.cross_account_role_name])
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_lambda_function" "refresh_ta_check_lambda" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "refresh-ta-check-lambda.zip"])
}
role = aws_iam_role.ref_ta_check_lambda_execution_role.arn
environment {
variables = {
IAMRoleName = var.cross_account_role_name
LOG_LEVEL = var.log_level
S3BucketName = aws_s3_bucket.s3_bucket.id
MASK_PII = var.mask_account_information
}
}
timeout = 60
handler = "refresh-ta-check-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "ref_ta_check_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "ref_ta_check_lambda_execution_policy" {
name = "RefTACheckLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.ref_ta_check_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.refresh_ta_check_lambda.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.refresh_ta_check_lambda.arn])
]
},
{
Effect = "Allow"
Action = "sts:AssumeRole"
Resource = join("", ["arn:aws:iam::*:role/", var.cross_account_role_name])
},
{
Effect = "Allow"
Action = [
"s3:PutObject",
"s3:PutObjectAcl"
]
Resource = join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"])
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_waf_sql_injection_match_set" "map_ta_check_step_function" {
name = "MapTACheck"
// CF Property(DefinitionString) = join("
// ", ["{", " "StartAt": "MapTACheck",", " "States": {", " "MapTACheck": {", " "Type": "Map",", " "Iterator": {", " "StartAt": "TARefresh",", " "States": {", " "TARefresh": {", " "Type": "Task",", join("", [" "Resource": "", aws_lambda_function.refresh_ta_check_lambda.arn, "","]), " "Next": "VerifyTACheckStatus",", " "Retry": [", " {", " "ErrorEquals": [ "Lambda.TooManyRequestsException"],", " "IntervalSeconds": 2,", " "MaxAttempts": 6,", " "BackoffRate": 2", " },", " {", " "ErrorEquals": ["States.ALL"],", " "IntervalSeconds": 5,", " "MaxAttempts": 2,", " "BackoffRate": 2", " }", " ]", " },", " "wait_X_seconds": {", " "Type": "Wait",", " "SecondsPath": "$.WaitTimeInSec",", " "Next": "VerifyTACheckStatus"", " },", " "VerifyTACheckStatus": {", " "Type": "Task",", join("", [" "Resource": "", aws_lambda_function.verify_ta_check_status_lambda.arn, "","]), " "Next": "Refresh Complete?",", " "Retry": [", " {", " "ErrorEquals": [", " "Lambda.TooManyRequestsException"", " ],", " "IntervalSeconds": 2,", " "MaxAttempts": 6,", " "BackoffRate": 2", " },", " {", " "ErrorEquals": ["States.ALL"],", " "IntervalSeconds": 2,", " "MaxAttempts": 2,", " "BackoffRate": 2", " }", " ]", " },", " "Refresh Complete?": {", " "Type": "Choice",", " "Choices": [", " {", " "Variable": "$.RefreshStatus",", " "StringEquals": "enqueued",", " "Next": "wait_X_seconds"", " },", " {", " "Variable": "$.RefreshStatus",", " "StringEquals": "processing",", " "Next": "wait_X_seconds"", " }", " ],", " "Default": "TACheck"", " },", " "TACheck": {", " "Type": "Task",", join("", [" "Resource": "", aws_lambda_function.extract_ta_data.arn, "","]), " "End": true,", " "Retry": [", " {", " "ErrorEquals": [", " "Lambda.TooManyRequestsException"", " ],", " "IntervalSeconds": 2,", " "MaxAttempts": 6,", " "BackoffRate": 2", " },", " {", " "ErrorEquals": ["States.ALL"],", " "IntervalSeconds": 2,", " "MaxAttempts": 2,", " "BackoffRate": 2", " }", " ]", " }", " }", " },", " "End": true", " }", " }", "}"])
// CF Property(RoleArn) = aws_iam_role.map_ta_check_execution_role.arn
// CF Property(tags) = {
// ProjectName = "AWS Trusted Advisor Explorer"
// }
}
resource "aws_iam_role" "map_ta_check_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"states.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "map_ta_check_execution_policy" {
name = "StepMapTACheckExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.map_ta_check_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"lambda:InvokeFunction"
]
Resource = [
aws_lambda_function.extract_ta_data.arn,
aws_lambda_function.verify_ta_check_status_lambda.arn,
aws_lambda_function.refresh_ta_check_lambda.arn
]
}
]
}
}
resource "aws_lambda_function" "get_ta_checks" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "get-ta-checks-lambda.zip"])
}
role = aws_iam_role.get_ta_checks_execution_role.arn
environment {
variables = {
LANGUAGE = var.language
EXTRACT_TA_DATA_PER_CHECK_SFN_ARN = aws_waf_sql_injection_match_set.map_ta_check_step_function.id
Category = "cost_optimizing"
MASK_PII = var.mask_account_information
LOG_LEVEL = var.log_level
SupportedChecks = "Qch7DwouX1,hjLMh88uM8,DAvU99Dc4C,Z4AUBRNSmz,Ti39halfu8,51fC20e7I2,G31sQ1E9U,1e93e4c0b5"
}
}
timeout = 5
handler = "get-ta-checks-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "get_ta_checks_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "get_ta_checks_execution_policy" {
name = "GetTAChecksExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.get_ta_checks_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "states:StartExecution"
Resource = aws_waf_sql_injection_match_set.map_ta_check_step_function.id
},
{
Effect = "Allow"
Action = [
"support:*"
]
Resource = "*"
},
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.get_ta_checks.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.get_ta_checks.arn])
]
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_waf_sql_injection_match_set" "map_organizations_step_function" {
name = "MapOrganizations"
// CF Property(DefinitionString) = join("
// ", ["{", " "StartAt": "MapOrganizations",", " "States": {", " "MapOrganizations": {", " "Type": "Map",", " "Iterator": {", " "StartAt": "GetTAChecks",", " "States": {", " "GetTAChecks": {", " "Type": "Task",", join("", [" "Resource": "", aws_lambda_function.get_ta_checks.arn, "","]), " "End": true,", " "Retry": [", " {", " "ErrorEquals": [ "Lambda.TooManyRequestsException"],", " "IntervalSeconds": 2,", " "MaxAttempts": 6,", " "BackoffRate": 2", " },", " {", " "ErrorEquals": ["States.ALL"],", " "IntervalSeconds": 5,", " "MaxAttempts": 2,", " "BackoffRate": 2", " }", " ]", " }", " }", " },", " "End": true", " }", "}", "}"])
// CF Property(RoleArn) = aws_iam_role.map_organizations_execution_role.arn
// CF Property(tags) = {
// ProjectName = "AWS Trusted Advisor Explorer"
// }
}
resource "aws_iam_role" "map_organizations_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"states.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "map_organizations_execution_policy" {
name = "StepMapOrganizationsExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.map_organizations_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"lambda:InvokeFunction"
]
Resource = [
aws_lambda_function.get_ta_checks.arn
]
}
]
}
}
resource "aws_lambda_function" "tag_extractor_lambda" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "extract-tag-data-lambda.zip"])
}
role = aws_iam_role.tag_extractor_lambda_execution_role.arn
environment {
variables = {
S3BucketName = aws_s3_bucket.s3_bucket.id
IAMRoleName = var.cross_account_role_name
LOG_LEVEL = var.log_level
CustomerKeys = var.interested_tag_keys
MASK_PII = var.mask_account_information
}
}
timeout = 300
handler = "extract-tag-data-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 256
}
resource "aws_iam_role" "tag_extractor_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "tag_extractor_lambda_execution_policy" {
name = "TagExtractorLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.tag_extractor_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.tag_extractor_lambda.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.tag_extractor_lambda.arn])
]
},
{
Effect = "Allow"
Action = [
"s3:PutObject",
"s3:PutObjectAcl"
]
Resource = join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"])
},
{
Effect = "Allow"
Action = "sts:AssumeRole"
Resource = join("", ["arn:aws:iam::*:role/", var.cross_account_role_name])
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_waf_sql_injection_match_set" "tag_extractor_step_function" {
name = "ExtractTags"
// CF Property(DefinitionString) = join("
// ", ["{", " "StartAt": "MapTags",", " "States": {", " "MapTags": {", " "Type": "Map",", " "Iterator": {", " "StartAt": "ExtractTags",", " "States": {", " "ExtractTags": {", " "Type": "Task",", join("", [" "Resource": "", aws_lambda_function.tag_extractor_lambda.arn, "","]), " "End": true,", " "Retry": [", " {", " "ErrorEquals": [", " "Lambda.TooManyRequestsException"", " ],", " "IntervalSeconds": 2,", " "MaxAttempts": 6,", " "BackoffRate": 2", " },", " {", " "ErrorEquals": ["States.ALL"],", " "IntervalSeconds": 2,", " "MaxAttempts": 2,", " "BackoffRate": 2", " }", " ]", " }", " }", " },", " "End": true", " }", " }", "}"])
// CF Property(RoleArn) = aws_iam_role.extract_tags_step_execution_role.arn
// CF Property(tags) = {
// ProjectName = "AWS Trusted Advisor Explorer"
// }
}
resource "aws_iam_role" "extract_tags_step_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"states.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "extract_tags_step_execution_policy" {
name = "StepExtractTagsExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.extract_tags_step_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"lambda:InvokeFunction"
]
Resource = [
aws_lambda_function.tag_extractor_lambda.arn
]
}
]
}
}
resource "aws_lambda_function" "get_tags" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "get-tags-lambda.zip"])
}
role = aws_iam_role.get_tags_execution_role.arn
environment {
variables = {
LOG_LEVEL = var.log_level
TAG_DATA_EXTRACT_SFN_ARN = aws_waf_sql_injection_match_set.tag_extractor_step_function.id
ResourceTypes = "rds:db,ec2:instance,ec2:volume,elasticloadbalancing:loadbalancer,route53:hostedzone,redshift:dbname"
MASK_PII = var.mask_account_information
}
}
timeout = 15
handler = "get-tags-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "get_tags_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "get_tags_execution_policy" {
name = "GetTagsExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.get_tags_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "states:StartExecution"
Resource = aws_waf_sql_injection_match_set.tag_extractor_step_function.id
},
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.get_tags.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.get_tags.arn])
]
},
{
Action = [
"ec2:DescribeRegions"
]
Effect = "Allow"
Resource = "*"
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_waf_sql_injection_match_set" "tag_map_organizations_step_function" {
name = "TagMapOrganizations"
// CF Property(DefinitionString) = join("
// ", ["{", " "StartAt": "MapOrganizations",", " "States": {", " "MapOrganizations": {", " "Type": "Map",", " "Iterator": {", " "StartAt": "GetTags",", " "States": {", " "GetTags": {", " "Type": "Task",", join("", [" "Resource": "", aws_lambda_function.get_tags.arn, "","]), " "End": true,", " "Retry": [", " {", " "ErrorEquals": [", " "Lambda.TooManyRequestsException"", " ],", " "IntervalSeconds": 2,", " "MaxAttempts": 6,", " "BackoffRate": 2", " },", " {", " "ErrorEquals": ["States.ALL"],", " "IntervalSeconds": 2,", " "MaxAttempts": 2,", " "BackoffRate": 2", " }", " ]", " }", " }", " },", " "End": true", " } ", "}", "}"])
// CF Property(RoleArn) = aws_iam_role.tag_map_organizations_execution_role.arn
// CF Property(tags) = {
// ProjectName = "AWS Trusted Advisor Explorer"
// }
}
resource "aws_iam_role" "tag_map_organizations_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"states.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "tag_map_organizations_execution_policy" {
name = "StepTagMapOrganizationsExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.tag_map_organizations_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"lambda:InvokeFunction"
]
Resource = [
aws_lambda_function.get_tags.arn
]
}
]
}
}
resource "aws_lambda_function" "get_accounts_lambda" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "get-accounts-info-lambda.zip"])
}
role = aws_iam_role.get_accounts_lambda_execution_role.arn
environment {
variables = {
BUCKET_NAME = aws_s3_bucket.s3_bucket.id
FILE_OVERRIDE = "false"
LOG_LEVEL = var.log_level
OBJECT_NAME = "folder/accounts.csv"
EXTRACT_TA_DATA_SFN_ARN = aws_waf_sql_injection_match_set.map_organizations_step_function.id
TAG_DATA_EXTRACT_SFN_ARN = aws_waf_sql_injection_match_set.tag_map_organizations_step_function.id
Tags = var.interested_tag_keys
MASK_PII = var.mask_account_information
AnonymousUsage = local.mappings["Send"]["AnonymousUsage"]["Data"]
UUID = // Unable to resolve Fn::GetAtt([
// "UUIDGenerator",
// "UUID"
// ])
Version = local.mappings["SourceCode"]["General"]["Version"]
}
}
timeout = 60
handler = "get-accounts-info-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "get_accounts_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "get_accounts_execution_policy" {
name = "awstrustedadvisorex-GetAccountsExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.get_accounts_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "states:StartExecution"
Resource = [
aws_waf_sql_injection_match_set.map_organizations_step_function.id,
aws_waf_sql_injection_match_set.tag_map_organizations_step_function.id
]
},
{
Effect = "Allow"
Action = [
"s3:Get*",
"s3:List*"
]
Resource = [
join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"]),
join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id])
]
},
{
Effect = "Allow"
Action = [
"organizations:Describe*",
"organizations:List*"
]
Resource = "*"
},
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.get_accounts_lambda.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.get_accounts_lambda.arn])
]
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_cloudwatch_event_rule" "cron_rule" {
description = "TrustedAdvisorExplorer Event Rule"
schedule_expression = var.report_schedule
// CF Property(State) = "ENABLED"
// CF Property(Targets) = [
// {
// Arn = aws_lambda_function.get_accounts_lambda.arn
// Id = "AWSTrustedAdExReportScheduler"
// }
// ]
}
resource "aws_lambda_permission" "invoke_lambda_permission_get_accounts" {
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.get_accounts_lambda.arn
principal = "events.amazonaws.com"
source_arn = aws_cloudwatch_event_rule.cron_rule.arn
}
resource "aws_glue_catalog_database" "aws_trusted_adv_ex_database" {
catalog_id = data.aws_caller_identity.current.account_id
// CF Property(DatabaseInput) = {
// Description = "AWS Trusted Advisor Explorer Database"
// Name = "aws_trusted_advisor_explorer_db"
// }
}
resource "aws_iam_role" "aws_trusted_adv_ex_crawler_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"glue.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
managed_policy_arns = [
"arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
]
}
resource "aws_iam_policy" "aws_trusted_adv_ex_crawler_execution_policy" {
name = "AWSTrustedAdvEx-CrawlerExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.aws_trusted_adv_ex_crawler_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"glue:UpdateDatabase",
"glue:UpdatePartition",
"glue:CreateTable",
"glue:UpdateTable",
"glue:ImportCatalogToGlue"
]
Resource = [
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":table/", aws_glue_catalog_database.aws_trusted_adv_ex_database.arn, "/*"]),
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":userDefinedFunction/", aws_glue_catalog_database.aws_trusted_adv_ex_database.arn, "/*"]),
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":database/", aws_glue_catalog_database.aws_trusted_adv_ex_database.arn]),
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":catalog"])
]
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:PutObject"
]
Resource = join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"])
}
]
}
}
resource "aws_glue_crawler" "aws_trusted_adv_ex_crawler" {
database_name = aws_glue_catalog_database.aws_trusted_adv_ex_database.arn
description = "This crawler that keeps your AWS Trusted Advisor Explorer table in Athena up-to-date"
name = "AWSTrustedAdvisorExplorer_Crawler"
role = aws_iam_role.aws_trusted_adv_ex_crawler_role.arn
delta_target {
// CF Property(S3Targets) = [
// {
// Path = join("", [aws_s3_bucket.s3_bucket.id, "/TA-Reports/cost_optimizing"])
// }
// ]
}
schema_change_policy {
update_behavior = "UPDATE_IN_DATABASE"
delete_behavior = "DELETE_FROM_DATABASE"
}
}
resource "aws_glue_crawler" "aws_trusted_adv_ex_tag_crawler" {
database_name = aws_glue_catalog_database.aws_trusted_adv_ex_database.arn
description = "Recurring crawler that keeps your AWS Trusted Advisor Explorer table in Athena up-to-date"
name = "AWSTrustedAdvisorExplorer_Tags_Crawler"
role = aws_iam_role.aws_trusted_adv_ex_crawler_role.arn
delta_target {
// CF Property(S3Targets) = [
// {
// Path = join("", [aws_s3_bucket.s3_bucket.id, "/Tags"])
// }
// ]
}
schema_change_policy {
update_behavior = "UPDATE_IN_DATABASE"
delete_behavior = "DELETE_FROM_DATABASE"
}
schedule = {
ScheduleExpression = var.glue_crawler_schedule
}
}
resource "aws_sns_topic" "my_sns_topic" {
name = "AWSTrustedAdvisorExplorer-DataRefresh"
// CF Property(Subscription) = [
// {
// Endpoint = var.sns_email
// Protocol = "email"
// }
// ]
}
resource "aws_sns_topic_policy" "sn_stopicpolicy" {
policy = {
Version = "2012-10-17"
Statement = [
{
Sid = "AWSTrustedAdvisorExplorerSNSpolicy"
Effect = "Allow"
Principal = {
Service = "events.amazonaws.com"
}
Action = "sns:Publish"
Resource = aws_sns_topic.my_sns_topic.id
}
]
}
// CF Property(Topics) = [
// aws_sns_topic.my_sns_topic.id
// ]
}
resource "aws_cloudwatch_event_rule" "event_rule" {
description = "Event Rule for successful Glue Crawler State"
event_pattern = {
source = [
"aws.glue"
]
detail-type = [
"Glue Crawler State Change"
]
detail = {
state = [
"Succeeded"
]
crawlerName = [
aws_glue_crawler.aws_trusted_adv_ex_crawler.id
]
}
}
// CF Property(State) = "ENABLED"
// CF Property(Targets) = [
// {
// Arn = aws_sns_topic.my_sns_topic.id
// Id = "MySNSTopic"
// InputTransformer = {
// InputPathsMap = {
// crawlerName = "$.detail.crawlerName"
// completionDate = "$.detail.completionDate"
// cloudWatchLogLink = "$.detail.cloudWatchLogLink"
// warningMessage = "$.detail.warningMessage"
// tablesCreated = "$.detail.tablesCreated"
// tablesUpdated = "$.detail.tablesUpdated"
// tablesDeleted = "$.detail.tablesDeleted"
// partitionsCreated = "$.detail.partitionsCreated"
// partitionsUpdated = "$.detail.partitionsUpdated"
// }
// InputTemplate = ""Glue Crawler <crawlerName> has successfully refreshed new Trusted Advisor Data"
//
// "Event Details:-"
// "completionDate: <completionDate>"
// "cloudWatchLogLink: <cloudWatchLogLink>"
// "warningMessage: <warningMessage>"
// "tablesCreated: <tablesCreated>"
// "tablesUpdated: <tablesUpdated>"
// "tablesDeleted: <tablesDeleted>"
// "partitionsCreated: <partitionsCreated>"
// "partitionsUpdated: <partitionsUpdated>""
// }
// },
// {
// Arn = aws_lambda_function.create_athena_view_lambda.arn
// Id = "CreateAthenaViewLambda"
// }
// ]
}
resource "aws_athena_workgroup" "my_athena_work_group" {
name = "AWSTrustedAvisorExplorer-AthenaWorkGroup"
// CF Property(RecursiveDeleteOption) = True
description = "This is a custom workgroup created as part of AWS Trusted Advisor Explorer Solution"
state = "ENABLED"
configuration {
enforce_workgroup_configuration = True
publish_cloudwatch_metrics_enabled = True
result_configuration {
encryption_configuration {
encryption_option = "SSE_S3"
}
output_location = join("", ["s3://", aws_s3_bucket.s3_bucket.id, "/AthenaOutputs/"])
}
}
}
resource "aws_lambda_function" "create_athena_view_lambda" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "create-athena-views-lambda.zip"])
}
role = aws_iam_role.create_athena_view_lambda_execution_role.arn
environment {
variables = {
AthenaOutput = aws_s3_bucket.s3_bucket.id
LOG_LEVEL = var.log_level
Tags = var.interested_tag_keys
AthenaDb = aws_glue_catalog_database.aws_trusted_adv_ex_database.arn
MASK_PII = var.mask_account_information
AthenaWorkGroup = aws_athena_workgroup.my_athena_work_group.arn
}
}
timeout = 10
handler = "create-athena-views-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "create_athena_view_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "create_athena_view_lambda_execution_policy" {
name = "CreateAthenaViewLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.create_athena_view_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.create_athena_view_lambda.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.create_athena_view_lambda.arn])
]
},
{
Effect = "Allow"
Action = "athena:StartQueryExecution"
Resource = join("", ["arn:aws:athena:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":workgroup/", aws_athena_workgroup.my_athena_work_group.arn])
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetBucketLocation",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:AbortMultipartUpload",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:CreateBucket",
"s3:GetObjectAcl",
"s3:PutObject",
"s3:PutObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id, "/*"]),
join("", ["arn:aws:s3:::", aws_s3_bucket.s3_bucket.id])
]
},
{
Effect = "Allow"
Action = [
"glue:CreateDatabase",
"glue:DeleteDatabase",
"glue:GetDatabase",
"glue:GetDatabases",
"glue:UpdateDatabase",
"glue:CreateTable",
"glue:DeleteTable",
"glue:BatchDeleteTable",
"glue:UpdateTable",
"glue:GetTable",
"glue:GetTables",
"glue:BatchCreatePartition",
"glue:CreatePartition",
"glue:DeletePartition",
"glue:BatchDeletePartition",
"glue:UpdatePartition",
"glue:GetPartition",
"glue:GetPartitions",
"glue:BatchGetPartition"
]
Resource = [
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":table/", aws_glue_catalog_database.aws_trusted_adv_ex_database.arn, "/*"]),
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":userDefinedFunction/", aws_glue_catalog_database.aws_trusted_adv_ex_database.arn, "/*"]),
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":database/", aws_glue_catalog_database.aws_trusted_adv_ex_database.arn]),
join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":catalog"])
]
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_lambda_permission" "permission_for_events_to_invoke_lambda" {
function_name = aws_lambda_function.create_athena_view_lambda.arn
action = "lambda:InvokeFunction"
principal = "events.amazonaws.com"
source_arn = aws_cloudwatch_event_rule.event_rule.arn
}
resource "aws_cloudwatch_event_rule" "event_rule_ta_crawler" {
description = "Event Rule to Trigger the TA Data Crawler"
event_pattern = {
source = [
"aws.glue"
]
detail-type = [
"Glue Crawler State Change"
]
detail = {
state = [
"Succeeded"
]
crawlerName = [
aws_glue_crawler.aws_trusted_adv_ex_tag_crawler.id
]
}
}
// CF Property(State) = "ENABLED"
// CF Property(Targets) = [
// {
// Arn = aws_lambda_function.start_glue_crawler_lambda.arn
// Id = "TriggerGlueCrawler"
// }
// ]
}
resource "aws_lambda_function" "start_glue_crawler_lambda" {
code_signing_config_arn = {
S3Bucket = join("-", [local.mappings["SourceCode"]["General"]["S3Bucket"], data.aws_region.current.name])
S3Key = join("/", [local.mappings["SourceCode"]["General"]["KeyPrefix"], "start-crawler-lambda.zip"])
}
role = aws_iam_role.start_glue_crawler_lambda_execution_role.arn
environment {
variables = {
LOG_LEVEL = var.log_level
MASK_PII = var.mask_account_information
CrawlerName = aws_glue_crawler.aws_trusted_adv_ex_crawler.id
}
}
timeout = 60
handler = "start-crawler-lambda.lambda_handler"
runtime = "python3.8"
memory_size = 128
}
resource "aws_iam_role" "start_glue_crawler_lambda_execution_role" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
}
resource "aws_iam_policy" "start_glue_crawler_lambda_execution_policy" {
name = "AWSTrustedAdEx-StartGlueCrawlerLambdaExecutionPolicy"
// CF Property(Roles) = [
// aws_iam_role.start_glue_crawler_lambda_execution_role.arn
// ]
policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = "logs:CreateLogGroup"
Resource = join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":*"])
},
{
Effect = "Allow"
Action = [
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = [
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.start_glue_crawler_lambda.arn, ":*"]),
join("", ["arn:aws:logs:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":log-group:/aws/lambda/", aws_lambda_function.start_glue_crawler_lambda.arn])
]
},
{
Effect = "Allow"
Action = "glue:StartCrawler"
Resource = join("", ["arn:aws:glue:", data.aws_region.current.name, ":", data.aws_caller_identity.current.account_id, ":crawler/", aws_glue_crawler.aws_trusted_adv_ex_crawler.id])
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:GetObjectTagging",
"s3:ListBucket",
"s3:GetObjectAcl"
]
Resource = [
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name, "/*"]),
join("", ["arn:aws:s3:::", local.mappings["SourceCode"]["General"]["S3Bucket"], "-", data.aws_region.current.name])
]
}
]
}
}
resource "aws_lambda_permission" "permission_for_events_to_invoke_glue_lambda" {
function_name = aws_lambda_function.start_glue_crawler_lambda.arn
action = "lambda:InvokeFunction"
principal = "events.amazonaws.com"
source_arn = aws_cloudwatch_event_rule.event_rule_ta_crawler.arn
}
output "raw_ta_data_bucket_name" {
description = "The name of the bucket in which the raw Trusted Advisor check data & Tag information will be stored"
value = aws_s3_bucket.s3_bucket.id
}
output "sns_topic" {
description = "The name of the SNS topic that will be notified after every data refresh."
value = aws_sns_topic.my_sns_topic.id
}
output "athena_database" {
description = "The name of the Athena database."
value = aws_glue_catalog_database.aws_trusted_adv_ex_database.arn
}
output "uuid" {
description = "Newly created random UUID."
// Unable to resolve Fn::GetAtt([
// "UUIDGenerator",
// "UUID"
// ])
}It will likely need to have things manually fixed, feel free to open new issues.
Thanks
Hi, I'm trying to convert the template file that doesn't work well. The template is located here : https://github.com/awslabs/aws-trusted-advisor-explorer/blob/master/deployment/aws-trusted-advisor-explorer.template
Error :
// Converting aws-trusted-advisor-explorer.template to Terraform! Traceback (most recent call last): File "/home/lrt0560/.local/bin/cf2tf", line 8, in <module> sys.exit(cli()) File "/home/lrt0560/.local/lib/python3.10/site-packages/click/core.py", line 1130, in __call__ return self.main(*args, **kwargs) File "/home/lrt0560/.local/lib/python3.10/site-packages/click/core.py", line 1055, in main rv = self.invoke(ctx) File "/home/lrt0560/.local/lib/python3.10/site-packages/click/core.py", line 1404, in invoke return ctx.invoke(self.callback, **ctx.params) File "/home/lrt0560/.local/lib/python3.10/site-packages/click/core.py", line 760, in invoke return __callback(*args, **kwargs) File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/app.py", line 44, in cli config = TemplateConverter(tmpl_path.stem, cf_template, search_manger).convert() File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 94, in convert tf_resources = self.convert_to_tf(self.manifest) File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 146, in convert_to_tf tf_resources.extend(converter(resources)) File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 331, in convert_resources resolved_values = self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 181, in resolve_values data[key] = self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 181, in resolve_values data[key] = self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 197, in resolve_values resolved_list_values = [ File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 198, in <listcomp> self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 181, in resolve_values data[key] = self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 197, in resolve_values resolved_list_values = [ File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 198, in <listcomp> self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 189, in resolve_values value = self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 197, in resolve_values resolved_list_values = [ File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 198, in <listcomp> self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 197, in resolve_values resolved_list_values = [ File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 198, in <listcomp> self.resolve_values( File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/convert.py", line 176, in resolve_values return functions.ref(self, value) File "/home/lrt0560/.local/lib/python3.10/site-packages/cf2tf/conversion/expressions.py", line 813, in ref first_attr = next(iter(valid_attributes)) StopIteration