convert deploy bash script to node

[fivethreeo]

Like this, tested and works:

/*
    npm install archiver --save
    npm install async --save
    npm install string-format --save
*/

const real_exec = require('child_process').exec,
  async = require('async'),
  format = require('string-format'),
  path = require('path'),
  os = require('os'),
  fs = require('fs'),
  archiver = require('archiver');

function exec(cmd, options, cb){
  return real_exec(cmd, options, function(_error, _stdout, _stderr) {
    console.log(cmd);
    console.log(_stderr);
    return cb(_error, _stdout, _stderr);
  });
}

const aws_cmds = {

  'get-user': 'aws iam get-user \
--output json \
--region "{REGION}"',

  'create-role': 'aws iam create-role \
--role-name "{FULL_NAME}" \
--assume-role-policy-document "{TRUST_ARG}"',

  'update-assume-role-policy': 'aws iam update-assume-role-policy \
--role-name "{FULL_NAME}" \
--policy-document "{TRUST_ARG}"',

  'put-role-policy': 'aws iam put-role-policy \
--role-name "{FULL_NAME}" \
--policy-name "{FULL_NAME}_policy" \
--policy-document "{POLICY_ARG}"',

  'lambda create-function': 'aws --region "{REGION}" lambda create-function \
--function-name "{FULL_NAME}" \
--runtime nodejs \
--role "{ROLE_ARN}" \
--handler \'index.handler\' \
--description "{LAMBDA_DESC}" \
--timeout 300 \
--memory-size 128 \
--zip-file "{ZIP_ARG}"',

  'lambda update-function-configuration': 'aws --region "{REGION}" lambda update-function-configuration \
--function-name "{FULL_NAME}" \
--role "{ROLE_ARN}" \
--handler \'index.handler\' \
--description "{LAMBDA_DESC}" \
--timeout 300 \
--memory-size 128',

  'lambda update-function-code': 'aws --region "{REGION}" lambda update-function-code \
--function-name "{FULL_NAME}" \
--zip-file "{ZIP_ARG}"'
};

var command_opts = {},
  default_region = 'us-east-1',
  regions =  ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-northeast-1'],
  resource_dir = path.join(__dirname, '..', '..'); 
  resource_info = require(path.join(__dirname, '..', '..', 'package.json'));

resource_info['version'] = resource_info['version'].replace(/\./g, '-');

var FULL_NAME = format("{name}-{version}", resource_info);
var ZIP_LOCATION = path.join(os.tmpdir(), FULL_NAME + '.zip');

var command_vars = {
  'ZIP_LOCATION': ZIP_LOCATION,
  'ZIP_ARG': 'fileb://' + ZIP_LOCATION, 
  'FULL_NAME': FULL_NAME,
  'POLICY_ARG': 'file://' + path.join(resource_dir, 'execution-policy.json'),
  'TRUST_ARG': 'file://' + path.join(__dirname, 'lib', 'lambda.trust.json'),
  'REGION': default_region,
  'LAMBDA_DESC': format('CloudFormation Custom Resource service for Custom::{name}', resource_info)
};

var output = fs.createWriteStream(ZIP_LOCATION);
var archive = archiver('zip', {store: true});

output.on('close', start_deploy);

archive.on('error', function(err){
    throw err;
});

console.log(format('Zipping Lambda bundle to {}...', ZIP_LOCATION));

archive.directory(resource_dir, '');

archive.pipe(output);

archive.finalize();

console.log('~~~~ Deploying Lambda to all regions (' + regions.join(' ') + '). You may see CREATE errors ~~~~');
console.log('~~~~ This is fine, it simply means that the deployment script will run UPDATEs ~~~~');

function start_deploy() {
  exec(format(aws_cmds['get-user'], command_vars), command_opts, handle_roles);
}

function handle_roles(error, stdout, stderr) {

  var user_info = JSON.parse(stdout);
  var account_re = /arn:aws:iam::(.*?):user.*/

  command_vars['ACCOUNT_NUMBER'] = user_info['User']['Arn'].replace(account_re, '$1');

  command_vars['ROLE_ARN'] = format('arn:aws:iam::{ACCOUNT_NUMBER}:role/{FULL_NAME}', command_vars);

  async.waterfall([
      function(callback) {
        exec(format(aws_cmds['create-role'], command_vars), command_opts, function(_error, _stdout, _stderr) {
            callback(null, _error, _stdout, _stderr);
        })
      },
      function(error, stdout, stderr, callback) {
        exec(format(aws_cmds['update-assume-role-policy'], command_vars), command_opts, function(_error, _stdout, _stderr) {
            callback(null, _error, _stdout, _stderr);
            console.log('Upserted Role!');
        })
      },
      function(error, stdout, stderr, callback) {
        exec(format(aws_cmds['put-role-policy'], command_vars), command_opts, function(_error, _stdout, _stderr) {
          console.log('Added Policy!');
          console.log("Sleeping 5 seconds for policy to propagate.");
          setTimeout(function() {
            callback(null, _error, _stdout, _stderr);
          }, 5000);
        });
      }],
    handle_regions); // waterfall 1
  }

function handle_regions(err, error, stdout, stderr) {
  console.log('Beginning deploy of Lambdas to Regions: ' + regions.join(' '));

  async.eachSeries(regions, handle_region, function () {
    console.log('~~~~ All done! Lambdas are deployed globally and ready for use by CloudFormation. ~~~~');
    console.log('~~~~                They are accessible to your CloudFormation at:                ~~~~');
    console.log(format('aws:arn:<region>:{ACCOUNT_NUMBER}:function:{FULL_NAME}', command_vars));
  });
}

function handle_region(region, region_callback) { 

  console.log('Deploying Lambda to: ' + region);

  command_vars['REGION'] = region;

  async.waterfall([
      function(callback) {
          exec(format(aws_cmds['lambda create-function'], command_vars), command_opts, function(_error, _stdout, _stderr) {
            callback(null, _error, _stdout, _stderr);
          });
      },
      function(error, stdout, stderr, callback) {
          exec(format(aws_cmds['lambda update-function-configuration'], command_vars), command_opts, function(_error, _stdout, _stderr) {
            callback(null, _error, _stdout, _stderr);
        })
      },
      function(error, stdout, stderr, callback) {
        exec(format(aws_cmds['lambda update-function-code'], command_vars), command_opts, function(_error, _stdout, _stderr) {
          callback(null, _error, _stdout, _stderr);
        });
      }
    ],
    function (err, error, stdout, stderr) {
      console.log('Upserted lambda');
      region_callback();
    }
  );
}    

[andrew-templeton]

There is no advantage to this approach over the current one, really - I will be making this upgrade with entirely JS / no native extensions + fully async. Using child_process is something I want to avoid, and we can make this fully JS without much additional effort.

+1 for ticket title, thanks for the code but I would want to go 100% JS

[fivethreeo]

I develop on windows, so bash is a no go. But full js is ofcourse best :)

[fivethreeo]

Do you have time to do this or should I give all js a go?

[andrew-templeton]

You can give it a go, I'll accept the PR if it works. I will have time for it this weekend, but if you need it NOW give it a try :)

[fivethreeo]

Need to study the node aws api a bit, got a working deploy already here, if you have experience with the node aws it might be better you do it.

[fivethreeo]

Change js to use aws-sdk

/*
    npm install aws-sdk --save
    npm install archiver --save
    npm install base64-stream --save
    npm install string-format --save
    npm install async --save
*/

var async = require('async'),
  format = require('string-format'),
  path = require('path'),
  fs = require('fs'),
  archiver = require('archiver'),
  base64 = require('base64-stream'),
  stream = require('stream'),

  DEFAULT_REGION = 'us-east-1',
  REGIONS =  ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-northeast-1'],
  RESOURCE_DIR = path.join(__dirname, '..', '..'),
  CFN_LAMBDA_DIR = path.join(__dirname, 'lib'),
  RESOURCE_INFO = require(path.join(RESOURCE_DIR, 'package.json')),
  FULL_NAME = format("{}-{}", RESOURCE_INFO.name, RESOURCE_INFO.version.replace(/\./g, '-')),
  POLICY = fs.readFileSync(path.join(RESOURCE_DIR, 'execution-policy.json')).toString(),
  TRUST = fs.readFileSync(path.join(CFN_LAMBDA_DIR, 'lambda.trust.json')).toString(),
  LAMBDA_DESC = format('CloudFormation Custom Resource service for Custom::{name}', RESOURCE_INFO)
  ACCOUNT_RE = /arn:aws:iam::(.*?):user.*/;

var zip_parts = []

var archive = archiver('zip');

var converter = new stream.Writable({
  write: function (chunk, encoding, next) {
    zip_parts.push(chunk);
    next()
}});

converter.on('finish', start_deploy)

console.log('Zipping Lambda bundle to buffer...');

archive.directory(RESOURCE_DIR, '');

archive.pipe(converter);

archive.finalize();

var AWS = require('aws-sdk');
AWS.config.region = 'us-east-1';

var iam = new AWS.IAM();

function start_deploy() { // Will be emitted when the input stream has ended, ie. no more data will be provided
  console.log('~~~~ Deploying Lambda to all regions (' + REGIONS.join(' ') + '). ~~~~');
  var base64_zip = Buffer.concat(zip_parts); // Create a buffer from all the received chunks
  iam.getUser({}, function (err, user_data) {
    if (err) { throw err; }
    handle_roles(err, user_data, base64_zip);
  })
}

function handle_roles(err, user_data, base64_zip) {

  var ROLE_ARN = format('arn:aws:iam::{}:role/{}',
    user_data.User.Arn.replace(ACCOUNT_RE, '$1'), FULL_NAME);

  console.log(ROLE_ARN);

  async.waterfall([
      function(callback) {
        iam.createRole({AssumeRolePolicyDocument: TRUST, RoleName: FULL_NAME}, function(err, data) {
          if (err !== null) {
            console.log('Created Role!');
            callback(null, true);
          }
          else {
            callback(null, false);
          }
        });
      },
      function(skip, callback) {
        if (skip) {
          callback(null);
        }
        else {
          iam.updateAssumeRolePolicy({PolicyDocument: TRUST, RoleName: FULL_NAME}, function(err, data) {
            if (err) { throw err; }
            else { callback(null); }
          });
        }
      },
      function(callback) {
        iam.putRolePolicy({PolicyDocument: POLICY, PolicyName: FULL_NAME + '_policy', RoleName: FULL_NAME},
          function(err, data) {
            if (err) { throw err; }
            console.log('Added Policy!');
            console.log("Sleeping 5 seconds for policy to propagate.");
            setTimeout(function() {
              callback(user_data, base64_zip, ROLE_ARN);
            }, 
            10000);
        });
      }],
    handle_regions); // waterfall 1
  }

function handle_regions(user_data, base64_zip, ROLE_ARN) {
  console.log('Beginning deploy of Lambdas to Regions: ' + REGIONS.join(' '));
  async.each(REGIONS,
    function (region, region_callback) {
      handle_region(region, region_callback, base64_zip, ROLE_ARN);
    },
    function () {
      console.log('~~~~ All done! Lambdas are deployed globally and ready for use by CloudFormation. ~~~~');
      console.log('~~~~                They are accessible to your CloudFormation at:                ~~~~');
      console.log(format('aws:arn:<region>:{}:function:{}', user_data.User.Arn.replace(ACCOUNT_RE, '$1'), FULL_NAME));
  });
}

function handle_region(region, region_callback, base64_zip, ROLE_ARN) { 

  var RegionAWS = require('aws-sdk');
  RegionAWS.config.region = region;

  var lambda = new RegionAWS.Lambda();

  console.log('Deploying Lambda to: ' + region);

  async.waterfall([
      function(callback) {
          lambda.createFunction({
            Code: {ZipFile: base64_zip},
            FunctionName: FULL_NAME,
            Description: LAMBDA_DESC,
            Role: ROLE_ARN,
            Handler: 'index.handler',
            Runtime: 'nodejs',
            Timeout: 300,
            MemorySize: 128
          },
          function(err, data) {
            if (err !== null) {
              callback(null, false);
            }
            else {
              console.log(format('Created Function "{}" on {}!', FULL_NAME, region));
              callback(null, true);
            }
        });
      },
      function(skip, callback) {
        if (skip) {
          callback(null, true);
        }
        else {
          lambda.updateFunctionConfiguration({
            FunctionName: FULL_NAME,
            Description: LAMBDA_DESC,
            Role: ROLE_ARN,
            Handler: 'index.handler',
            Timeout: 300,
            MemorySize: 128
          },
          function(err, data) {
            if (err !== null) {
              throw err;
            }
            else {
              console.log(format('Updated Function Configuration for "{}" on {}!', FULL_NAME, region));
              callback(null, false);
            }
          });
        }          
      },
      function(skip, callback) {
        if (skip) {
          callback();
        }
        else {
          lambda.updateFunctionCode({FunctionName: FULL_NAME, ZipFile: base64_zip},
            function(err, data) {
              if (err !== null) {
                throw err;
              }
              else {
                console.log(format('Updated Function Code for "{}" on {}!', FULL_NAME, region));
                callback();
              }
          });
        }       
      }
    ],
    function () {
      console.log(format('Upserted lambda "{}" on {}!', FULL_NAME, region));
      region_callback();
    }
  );
}  

[andrew-templeton]

Wow! Can you make a PR for this?

[fivethreeo]

Will do :)