Merge pull request #61 from developmentseed/v2.2.0

V2.2.0 release

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/node:4
+      - image: circleci/node:6
 
 
     working_directory: ~/kes
@@ -39,11 +39,12 @@ jobs:
             ./bin/cli.js cf validate --kes-folder examples/full
             ./bin/cli.js cf validate --kes-folder examples/nested
 
-            ./bin/cli.js cf validate --kes-folder examples/app_using_template --template examples/template
-
-
             npm run test
 
+      - run:
+          name: Integration Tests
+          command: npm run integration-test
+
       - add_ssh_keys:
           fingerprints:
              - "8f:58:d1:7d:1f:b0:ed:cc:fd:c9:46:cb:ee:6b:40:86"

CHANGELOG

@@ -1,3 +1,10 @@
+## v2.2.0
+
+- show proper logs when the template merging fails due to a yaml 
+- a new flag `--showOutputs` will show the output of a CloudFormation template
+- add cloudformation delete functionality
+- support `system_bucket` as the variable for for holding the bucket name used in deployment
+
 ## v2.1.2
 
 - fix a bug where determineKesClass failed to return the default class

bin/cli.js

@@ -21,7 +21,7 @@ program.version(pckg.version);
 function extractCommanderOptions(program) {
   const options = {};
   Object.keys(program).forEach(property => {
-    if (typeof program[property] === 'string') {
+    if (typeof program[property] === 'string' || typeof program[property] === 'boolean') {
       options[property] = program[property];
     }
   });
@@ -74,7 +74,7 @@ const init = function () {
       config.default.buckets = {};
     }
 
-    config.default.buckets.internal = result.bucket;
+    config.default.system_bucket = result.bucket;
     fs.writeFileSync(path.join(kesFolder, 'config.yml'), yaml.safeDump(config));
 
     fs.createReadStream(
@@ -105,20 +105,23 @@ program
   .option('-k, --kes-folder <kesFolder>', 'Path to config folder')
   .option('-r, --region <region>', 'AWS region', null)
   .option('--stack <stack>', 'stack name, defaults to the config value')
+  .option('--showOutputs', 'Show the list of a CloudFormation template outputs')
+  .option('--yes', 'Skip all confirmation prompts')
   .option('-d, --deployment <deployment>', 'Deployment name, default to default');
 
 program
-  .command('cf [create|update|upsert|deploy|validate|compile]')
+  .command('cf [create|update|upsert|deploy|validate|compile|delete]')
   .description(`CloudFormation Operations:
   create    Creates the CF stack (deprecated, start using deploy)
   update    Updates the CF stack (deprecated, start using deploy)
-  upsert    Creates the CF stack and Update if already exists (deprected, start using deploy)
+  upsert    Creates the CF stack and Update if already exists (deprecated, start using deploy)
   deploy    Creates the CF stack and Update if already exists
+  delete    Delete the CF stack
   validate  Validates the CF stack
   compile   Compiles the CF stack`)
   .action((cmd, o) => {
     const options = extractCommanderOptions(program);
-    kes.buildCf(options ,cmd);
+    kes.buildCf(options, cmd).then(r => kes.utils.success(r)).catch(e => kes.utils.failure(e));
   });
 
 program

bin/readme.js

@@ -28,15 +28,18 @@
  *  -k, --kes-folder <kesFolder>  Path to config folder
  *  -r, --region <region>         AWS region
  *  --stack <stack>               stack name, defaults to the config value
+ *  --showOutputs                  Show the list of a CloudFormation template outputs
+ *  --yes                          Skip all confirmation prompts
  *  -d, --deployment <deployment>  Deployment name, default to default
  *  -h, --help                    output usage information
  *
  *  Commands:
  *  cf [deploy|validate|compile]  CloudFormation Operations:
  *    create    Creates the CF stack (deprecated, start using deploy)
  *    update    Updates the CF stack (deprecated, start using deploy)
- *    upsert    Creates the CF stack and Update if already exists (deprected, start using deploy)
+ *    upsert    Creates the CF stack and Update if already exists (deprecated, start using deploy)
  *    deploy    Creates the CF stack and Update if already exists
+ *    delete    Delete the CF stack
  *    validate  Validates the CF stack
  *    compile   Compiles the CF stack
  *    lambda <lambdaName>                         uploads a given lambda function to Lambda service
@@ -65,7 +68,7 @@
  * the `default` section of the `config.yml` is parsed and used in `cloudformation.template.yml`. If
  * another deployment is specified in the `config.yml` the values of that deployment overrides the
  * values of `default`
- * file which is sent to AWS CloudFormation to create and udpate the stack.
+ * file which is sent to AWS CloudFormation to create and update the stack.
  *
  * ### CF Stack Name
  * The Cloudformation stack name is the same as `stackName` in `config.yml`.
@@ -247,7 +250,14 @@
  *  kes cf deploy
  * ```
  *
- * ### Differenet deployment configurations
+ * ### Delete an existing stack
+ *
+ * To delete an existing stack:
+ * ```bash
+ *   kes cf delete
+ * ```
+ *
+ * ### Different deployment configurations
  *
  * You can configure different values for different deployments. For example you might want to configure your test deployment
  * differently from your staging and production deployments. Here is how to achieve it:
@@ -279,7 +289,7 @@
  * **Note:** You still need an aws user with AssumeRole permission for this to work
  *
  * ```bash
- * kes cf deploy --profile myUser --role arn:aws:iam::00000000000:role/myDeplymentRole
+ * kes cf deploy --profile myUser --role arn:aws:iam::00000000000:role/myDeploymentRole
  * ```
  *
  * ### Updating One Lambda Function

docs/API.md

@@ -10,14 +10,17 @@
     -   [compileCF](#compilecf)
     -   [uploadToS3](#uploadtos3)
     -   [uploadCF](#uploadcf)
+    -   [waitFor](#waitfor)
     -   [cloudFormation](#cloudformation)
     -   [validateTemplate](#validatetemplate)
     -   [describeCF](#describecf)
+    -   [deleteCF](#deletecf)
     -   [opsStack](#opsstack)
     -   [upsertStack](#upsertstack)
     -   [deployStack](#deploystack)
     -   [createStack](#createstack)
     -   [updateStack](#updatestack)
+    -   [deleteStack](#deletestack)
 -   [Lambda](#lambda)
     -   [buildS3Path](#builds3path)
     -   [getHash](#gethash)
@@ -34,6 +37,8 @@
 -   [mergeYamls](#mergeyamls)
 -   [determineKesClass](#determinekesclass)
 -   [failure](#failure)
+-   [success](#success)
+-   [getSystemBucket](#getsystembucket)
 
 ## Config
 
@@ -87,7 +92,7 @@ Returns **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/G
 
 The main Kes class. This class is used in the command module to create
 the CLI interface for kes. This class can be extended in order to override
-and modify the behaviour of kes cli.
+and modify the behavior of kes cli.
 
 **Parameters**
 
@@ -151,6 +156,16 @@ Uploads the Cloud Formation template to a given S3 location
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** returns the promise of an AWS response object
 
+### waitFor
+
+Wait for the current stack and log the current outcome
+
+**Parameters**
+
+-   `wait`  
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** undefined
+
 ### cloudFormation
 
 Calls CloudFormation's update-stack or create-stack methods
@@ -169,6 +184,12 @@ Describes the cloudformation stack deployed
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** returns the promise of an AWS response object
 
+### deleteCF
+
+Deletes the current stack
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** undefined
+
 ### opsStack
 
 Generic create/update  method for CloudFormation
@@ -201,6 +222,12 @@ Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** returns the promise of an AWS response object
 
+### deleteStack
+
+Deletes the main stack
+
+Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** returns the promise of an AWS response object
+
 ## Lambda
 
 Copy, zip and upload lambda functions to S3
@@ -398,3 +425,17 @@ In case of error logs the error and exit with error 1
 **Parameters**
 
 -   `e` **[Error](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error)** error object
+
+## success
+
+Exists the process when called
+
+## getSystemBucket
+
+Discover and returns the system bucket used for deployment
+
+**Parameters**
+
+-   `config` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** cumulus config object
+
+Returns **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** name of the bucket

docs/README.md

@@ -12,7 +12,8 @@
         -   [Lambda Functions](#lambda-functions)
         -   [Handlebar Helpers](#handlebar-helpers)
     -   [Deployment](#deployment)
-        -   [Differenet deployment configurations](#differenet-deployment-configurations)
+        -   [Delete an existing stack](#delete-an-existing-stack)
+        -   [Different deployment configurations](#different-deployment-configurations)
     -   [Deployment Using IAM Role](#deployment-using-iam-role)
         -   [Updating One Lambda Function](#updating-one-lambda-function)
     -   [Use Templates](#use-templates)
@@ -50,15 +51,18 @@ It makes it much easier to deploy lambda functions and create API gateway resour
  -k, --kes-folder <kesFolder>  Path to config folder
  -r, --region <region>         AWS region
  --stack <stack>               stack name, defaults to the config value
+ --showOutputs                  Show the list of a CloudFormation template outputs
+ --yes                          Skip all confirmation prompts
  -d, --deployment <deployment>  Deployment name, default to default
  -h, --help                    output usage information
 
  Commands:
  cf [deploy|validate|compile]  CloudFormation Operations:
    create    Creates the CF stack (deprecated, start using deploy)
    update    Updates the CF stack (deprecated, start using deploy)
-   upsert    Creates the CF stack and Update if already exists (deprected, start using deploy)
+   upsert    Creates the CF stack and Update if already exists (deprecated, start using deploy)
    deploy    Creates the CF stack and Update if already exists
+   delete    Delete the CF stack
    validate  Validates the CF stack
    compile   Compiles the CF stack
    lambda <lambdaName>                         uploads a given lambda function to Lambda service
@@ -86,7 +90,7 @@ The variables in `config.yml` are parsed and used to generate the `cloudformatio
 the `default` section of the `config.yml` is parsed and used in `cloudformation.template.yml`. If
 another deployment is specified in the `config.yml` the values of that deployment overrides the
 values of `default`
-file which is sent to AWS CloudFormation to create and udpate the stack.
+file which is sent to AWS CloudFormation to create and update the stack.
 
 ### CF Stack Name
 
@@ -273,7 +277,15 @@ To create a CF stack or update and existing one run
  kes cf deploy
 ```
 
-### Differenet deployment configurations
+### Delete an existing stack
+
+To delete an existing stack:
+
+```bash
+  kes cf delete
+```
+
+### Different deployment configurations
 
 You can configure different values for different deployments. For example you might want to configure your test deployment
 differently from your staging and production deployments. Here is how to achieve it:
@@ -304,7 +316,7 @@ You can specify an IAM role for the deployment using `--role` option or by setti
 **Note:** You still need an aws user with AssumeRole permission for this to work
 
 ```bash
-kes cf deploy --profile myUser --role arn:aws:iam::00000000000:role/myDeplymentRole
+kes cf deploy --profile myUser --role arn:aws:iam::00000000000:role/myDeploymentRole
 ```
 
 ### Updating One Lambda Function

examples/app_using_template/cloudformation.template.yml

@@ -1,6 +1,9 @@
 Outputs:
 # this is not in the original template. we are adding it here to override
-{{#each apis}}
-  {{Name}}Endpoint:
-    Value: !Ref {{name}}RestApi
+{{#each lambdas}}
+  {{@key}}LambdaFunction:
+    Value: 
+      Fn::GetAtt:
+      - {{@key}}LambdaFunction 
+      - Arn
 {{/each}}

examples/app_using_template/config.yml

@@ -1,7 +1,13 @@
-myDeployment:
-  stackName: myDeployment-using-template
+kesTestDeployment:
+  stackName: kes-test-using-template
   noDash: myDeployment
+  system_bucket: devseed-kes-deployment 
 
   buckets:
-    internal: devseed-kes-deployment
-    tests: myTestArtifacts
+    tests: myTestArtifacts
+
+  lambdas:
+    func3: # function name must match the lambda folder name in build directory
+      handler: index.handler
+      timeout: 300
+      source: 'examples/code/lambda1/'

examples/full/config.yml

@@ -2,10 +2,10 @@ default:
   stackName: kes-full-example
   noDash: KesFullExample
   retryValue: 10
+  system_bucket: devseed-kes-deployment
 
   buckets:
-    internal: devseed-kes-deployment
-    tests: myTestArtifacts
+    test: myTestArtifacts
 
   capabilities:
     - CAPABILITY_NAMED_IAM
@@ -67,7 +67,7 @@ default:
       memory: 256
       source: 'examples/code/api/'
       envs:
-        internal: '{{buckets.internal}}'
+        internal: '{{buckets.test}}'
       apiGateway:
         - path: list
           method: get

examples/lambdas/cloudformation.template.yml

@@ -43,27 +43,32 @@ Resources:
   # Lambda config BEGIN
   #################################################
 {{#each lambdas}}
-  {{name}}LambdaFunction:
+  {{@key}}LambdaFunction:
     Type: AWS::Lambda::Function
     Properties:
       Code:
-        S3Bucket: {{bucket}}
-        S3Key: {{remote}}
-      FunctionName: {{../stackName}}-{{name}}
+      {{#if this.s3Source}}
+        S3Bucket: {{this.s3Source.bucket}}
+        S3Key: {{this.s3Source.key}}
+      {{else}}
+        S3Bucket: {{this.bucket}}
+        S3Key: {{this.remote}}
+      {{/if}}
+      FunctionName: {{../stackName}}-{{@key}}
       Environment:
         Variables:
           stackName: {{../stackName}}
-        {{#each envs}}
+        {{#each this.envs}}
           {{@key}}: {{this}}
         {{/each}}
-      Handler: {{handler}}
-      MemorySize: {{memory}}
+      Handler: {{this.handler}}
+      MemorySize: {{this.memory}}
       Role:
         Fn::GetAtt:
           - LambdaRole
           - Arn
-      Runtime: {{# if runtime}}{{runtime}}{{else}}nodejs6.10{{/if}}
-      Timeout: {{# if timeout}}{{timeout}}{{else}}300{{/if}}
+      Runtime: {{# if this.runtime}}{{this.runtime}}{{else}}nodejs6.10{{/if}}
+      Timeout: {{# if this.timeout}}{{this.timeout}}{{else}}300{{/if}}
 {{/each}}
   #################################################
   # Lambda config END