Closed Almenon closed 1 year ago
Almenon
commented
1 year ago Note that you can delete everything after line 116 (everything after the AWSCURCrawler block) and you will get the same problem. If you delete everything after line 90 (deleting AWSCURCrawler and everything below) the problem does not appear.
shadycuz
commented
1 year ago Thanks for reporting this, will take a look.
shadycuz
commented
1 year ago I have this fixed locally and it will be released in a few days, for now here is your converted template.
$ cf2tf crawler-cfn.yml
// Converting crawler-cfn.yml to Terraform!
// Existing Terraform src code found at /tmp/terraform_src.
data "aws_partition" "current" {}
data "aws_caller_identity" "current" {}
resource "aws_glue_catalog_database" "awscur_database" {
// CF Property(DatabaseInput) = {
// Name = "athenacurcfn_report_name"
// }
catalog_id = data.aws_caller_identity.current.account_id
}
resource "aws_iam_role" "awscur_crawler_component_function" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"glue.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
managed_policy_arns = [
"arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AWSGlueServiceRole"
]
force_detach_policies = [
{
PolicyName = "AWSCURCrawlerComponentFunction"
PolicyDocument = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = "arn:${data.aws_partition.current.partition}:logs:*:*:*"
},
{
Effect = "Allow"
Action = [
"glue:UpdateDatabase",
"glue:UpdatePartition",
"glue:CreateTable",
"glue:UpdateTable",
"glue:ImportCatalogToGlue"
]
Resource = "*"
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:PutObject"
]
Resource = "arn:${data.aws_partition.current.partition}:s3:::report-bucket/cur/report-name/report-name*"
}
]
}
},
{
PolicyName = "AWSCURKMSDecryption"
PolicyDocument = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"kms:Decrypt"
]
Resource = "*"
}
]
}
}
]
}
resource "aws_iam_role" "awscur_crawler_lambda_executor" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
force_detach_policies = [
{
PolicyName = "AWSCURCrawlerLambdaExecutor"
PolicyDocument = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = "arn:${data.aws_partition.current.partition}:logs:*:*:*"
},
{
Effect = "Allow"
Action = [
"glue:StartCrawler"
]
Resource = "*"
}
]
}
}
]
}
resource "aws_glue_crawler" "awscur_crawler" {
name = "AWSCURCrawler-report-name"
description = "A recurring crawler that keeps your CUR table in Athena up-to-date."
role = aws_iam_role.awscur_crawler_component_function.arn
database_name = aws_glue_catalog_database.awscur_database.arn
delta_target {
// CF Property(S3Targets) = [
// {
// Path = "s3://report-bucket/cur/report-name/report-name"
// Exclusions = [
// "**.json",
// "**.yml",
// "**.sql",
// "**.csv",
// "**.gz",
// "**.zip"
// ]
// }
// ]
}
schema_change_policy {
update_behavior = "UPDATE_IN_DATABASE"
delete_behavior = "DELETE_FROM_DATABASE"
}
}
resource "aws_lambda_function" "awscur_initializer" {
code_signing_config_arn = {
ZipFile = "const AWS = require('aws-sdk'); const response = require('./cfn-response'); exports.handler = function(event, context, callback) {
if (event.RequestType === 'Delete') {
response.send(event, context, response.SUCCESS);
} else {
const glue = new AWS.Glue();
glue.startCrawler({ Name: 'AWSCURCrawler-report-name' }, function(err, data) {
if (err) {
const responseData = JSON.parse(this.httpResponse.body);
if (responseData['__type'] == 'CrawlerRunningException') {
callback(null, responseData.Message);
} else {
const responseString = JSON.stringify(responseData);
if (event.ResponseURL) {
response.send(event, context, response.FAILED,{ msg: responseString });
} else {
callback(responseString);
}
}
}
else {
if (event.ResponseURL) {
response.send(event, context, response.SUCCESS);
} else {
callback(null, response.SUCCESS);
}
}
});
}
};
"
}
handler = "index.handler"
timeout = 30
runtime = "nodejs16.x"
reserved_concurrent_executions = 1
role = aws_iam_role.awscur_crawler_lambda_executor.arn
}
resource "aws_ce_cost_category" "aws_start_cur_crawler" {
// CF Property(ServiceToken) = aws_lambda_function.awscur_initializer.arn
}
resource "aws_lambda_permission" "awss3_cur_event_lambda_permission" {
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.awscur_initializer.arn
principal = "s3.amazonaws.com"
source_account = data.aws_caller_identity.current.account_id
source_arn = "arn:${data.aws_partition.current.partition}:s3:::report-bucket"
}
resource "aws_iam_role" "awss3_cur_lambda_executor" {
assume_role_policy = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Principal = {
Service = [
"lambda.amazonaws.com"
]
}
Action = [
"sts:AssumeRole"
]
}
]
}
path = "/"
force_detach_policies = [
{
PolicyName = "AWSS3CURLambdaExecutor"
PolicyDocument = {
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
]
Resource = "arn:${data.aws_partition.current.partition}:logs:*:*:*"
},
{
Effect = "Allow"
Action = [
"s3:PutBucketNotification"
]
Resource = "arn:${data.aws_partition.current.partition}:s3:::report-bucket"
}
]
}
}
]
}
resource "aws_lambda_function" "awss3_cur_notification" {
code_signing_config_arn = {
ZipFile = "const AWS = require('aws-sdk'); const response = require('./cfn-response'); exports.handler = function(event, context, callback) {
const s3 = new AWS.S3();
const putConfigRequest = function(notificationConfiguration) {
return new Promise(function(resolve, reject) {
s3.putBucketNotificationConfiguration({
Bucket: event.ResourceProperties.BucketName,
NotificationConfiguration: notificationConfiguration
}, function(err, data) {
if (err) reject({ msg: this.httpResponse.body.toString(), error: err, data: data });
else resolve(data);
});
});
};
const newNotificationConfig = {};
if (event.RequestType !== 'Delete') {
newNotificationConfig.LambdaFunctionConfigurations = [{
Events: [ 's3:ObjectCreated:*' ],
LambdaFunctionArn: event.ResourceProperties.TargetLambdaArn || 'missing arn',
Filter: { Key: { FilterRules: [ { Name: 'prefix', Value: event.ResourceProperties.ReportKey } ] } }
}];
}
putConfigRequest(newNotificationConfig).then(function(result) {
response.send(event, context, response.SUCCESS, result);
callback(null, result);
}).catch(function(error) {
response.send(event, context, response.FAILED, error);
console.log(error);
callback(error);
});
};
"
}
handler = "index.handler"
timeout = 30
runtime = "nodejs16.x"
reserved_concurrent_executions = 1
role = aws_iam_role.awss3_cur_lambda_executor.arn
}
resource "aws_s3_bucket_notification" "aws_put_s3_cur_notification" {
// CF Property(ServiceToken) = aws_lambda_function.awss3_cur_notification.arn
// CF Property(TargetLambdaArn) = aws_lambda_function.awscur_initializer.arn
bucket = "report-bucket"
// CF Property(ReportKey) = "cur/report-name/report-name"
}
resource "aws_route_table" "awscur_report_status_table" {
// CF Property(DatabaseName) = "athenacurcfn_report_name"
vpc_id = data.aws_caller_identity.current.account_id
// CF Property(TableInput) = {
// Name = "cost_and_usage_data_status"
// TableType = "EXTERNAL_TABLE"
// StorageDescriptor = {
// Columns = [
// {
// Name = "status"
// Type = "string"
// }
// ]
// InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"
// OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"
// SerdeInfo = {
// SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
// }
// Location = "s3://report-bucket/cur/report-name/cost_and_usage_data_status/"
// }
// }
}It will likely have lots of things that need manually fixed, feel free to raise new issues. Thanks
Almenon
commented
1 year ago Thanks!
tatarevick
commented
1 year ago I encountered the same problem with my code after downloading the updated version of cf2tf. I would like to use it, but even after trying different YAML files, I keep receiving the same error.
debug: Converted type from AWS::IAM::Role to aws_iam_role debug: Unable to find section Attributes Reference in /tmp/terraform_src/website/docs/r/iam_role.html.markdown Traceback (most recent call last): File "/home/linuxbrew/.linuxbrew/opt/python@3.11/lib/python3.11/site-packages/cf2tf/terraform/doc_file.py", line 19, in parse_attributes attributes = parse_section("Attributes Reference", file) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/linuxbrew/.linuxbrew/opt/python@3.11/lib/python3.11/site-packages/cf2tf/terraform/doc_file.py", line 37, in parse_section section_location = find_section(name, file) ^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/linuxbrew/.linuxbrew/opt/python@3.11/lib/python3.11/site-packages/cf2tf/terraform/doc_file.py", line 117, in find_section raise Exception() Exception
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/linuxbrew/.linuxbrew/bin/cf2tf", line 8, in
shadycuz
commented
1 year ago @tatarevick Can you post a new issue and provide a template that doesn't work?
tatarevick
commented
1 year ago Yep! Thanks :)
In https://docs.aws.amazon.com/cur/latest/userguide/use-athena-cf.html amazon provides you a cloudformation template to apply to sync cost and usage reports to AWS Athena. Howecer, cf2tf gets an error when running against the template.
The template:
The error: