my-observability-stack up and running

.gitignore

@@ -1,10 +1,8 @@
 # Logs
 logs
 *.log
-npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
-lerna-debug.log*
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
@@ -69,8 +67,7 @@ typings/
 .yarn-integrity
 
 # dotenv environment variables file
-.env
-.env.test
+.env.local
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache
@@ -102,3 +99,5 @@ dist
 
 # TernJS port file
 .tern-port
+
+.DS_Store

stack/.env

@@ -0,0 +1,5 @@
+# AWS
+AWS_DEFAULT_REGION=eu-west-1
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_VPC_ID=

stack/README.md

@@ -0,0 +1,12 @@
+# observability
+
+Deploys an event-driven (a.k.a. distributed) stack on AWS, evaluates observability tooling available in the AWS 🤝 Serverless space.
+
+## Design
+
+## Deploy
+
+- `yarn`
+- `yarn deploy`
+
+## Test

stack/flow1/emitters/infra.yml

@@ -0,0 +1,10 @@
+Resources:
+  EventTopic1:
+    Type: AWS::SNS::Topic
+    Properties:
+      DisplayName: "FLOW_ONE_COMPLETED_TOPIC"
+      FifoTopic: false
+      TopicName: "FLOW_ONE_COMPLETED_TOPIC"
+      KmsMasterKeyId: alias/${self:service}-encryption-key
+      # Good to know : AWS SNS does encryption-in-transit 'by default' over TLS.
+      # By using customer-managed key (see above), we enabled encryption-at-rest as well (Server-side encryption).

stack/flow1/omitters/infra.yml

@@ -0,0 +1,11 @@
+Resources:
+  DLQ1:
+    Type: AWS::SQS::Queue
+    Properties:
+      DelaySeconds: 0
+      QueueName: "FLOW_ONE_DLQ_QUEUE"
+  MDLQ1:
+    Type: AWS::SQS::Queue
+    Properties:
+      DelaySeconds: 0
+      QueueName: "FLOW_ONE_MDLQ_QUEUE"

stack/flow1/saga/infra/roles.yml

@@ -0,0 +1,38 @@
+Resources:
+  Flow1Role:
+    Type: AWS::IAM::Role
+    DependsOn:
+      - Lambda1LambdaFunction
+      - EventTopic1
+    Properties:
+      RoleName: ${self:provider.stackName}-flow1Role
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: states.${aws:region}.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: root
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - lambda:InvokeFunction
+                Resource:
+                  - !GetAtt Lambda1LambdaFunction.Arn
+              - Effect: Allow
+                Action:
+                  - sns:Publish
+                Resource:
+                  - !Ref EventTopic1
+              # Good to know : Step Function's IAM role needs to have the following KMS permissions
+              # on AWS-Managed Key in order to be able to publish to an encrypted SNS topic successfully.
+              - Effect: Allow
+                Action:
+                  - kms:GenerateDataKey
+                  - kms:Decrypt
+                Resource:
+                  - arn:aws:kms:${aws:region}:${aws:accountId}:alias/${self:service}-encryption-key

stack/flow1/saga/infra/stepfunctions.js

@@ -0,0 +1,26 @@
+/* eslint-disable no-console */
+// This is a utility JS module works alongside with stepfunctions.yml & saga.json
+// It acts as a pre-processor for Serverless, wherein it parses saga.json and converts it into single-line JSON before passing it to CloudFormation.
+// More at: https://www.serverless.com/framework/docs/providers/aws/guide/variables/#reference-variables-in-javascript-files
+
+const fs = require("fs");
+
+module.exports.definitionString = (serverless) => {
+  const stage = serverless.options.stage;
+  let fc = fs.readFileSync("flow1/saga/saga.json", "utf8");
+  switch (stage) {
+    case "dev":
+      // stage-specific custom code goes here
+      break;
+    case "prod":
+      // stage-specific custom code goes here
+      break;
+    default:
+      console.log(
+        "*** Parameter missing 'STAGE'. Use --stage <dev|prod> while invoking Serverless. ***"
+      );
+      return false;
+  }
+  fc = fc.replace(/(\r\n|\n|\r)/gm, "").replace(/\s\s+/g, "");
+  return fc;
+};

stack/flow1/saga/infra/stepfunctions.yml

@@ -0,0 +1,20 @@
+Resources:
+  Flow1:
+    Type: "AWS::StepFunctions::StateMachine"
+    Properties:
+      StateMachineName: ${self:service}-flow1
+      DefinitionString: ${file(./flow1/saga/infra/stepfunctions.js):definitionString}
+      # Gotcha!
+      # Unfortunately, AWS CloudFormation only accepts the entire Step Function definition as a single-line JSON string,
+      # hence, it is not very convenient to make changes to it in future if we do it that way.
+      # To overcome this limiation, we are using Serverless Variables feature here by pre-processing a good-looking JSON into a single-line JSON with the help of dashboard.js.
+      # More at: https://www.serverless.com/framework/docs/providers/aws/guide/variables/#reference-variables-in-javascript-files
+
+      # Gotcha!
+      # For a better DevXP, saga.json file can be live-viewed in VS Code using the following plug-in
+      # https://marketplace.visualstudio.com/items?itemName=paulshestakov.aws-step-functions-constructor
+
+      RoleArn: !GetAtt Flow1Role.Arn
+      DefinitionSubstitutions:
+        lambda1Arn: !GetAtt Lambda1LambdaFunction.Arn
+        topicArn: !Ref EventTopic1

stack/flow1/saga/saga.json

@@ -0,0 +1,47 @@
+{
+  "Comment":"Flow1",
+  "StartAt":"Invoke Lambda1",
+  "States":{
+     "Invoke Lambda1":{
+         "Type": "Task",
+         "Resource": "arn:aws:states:::lambda:invoke",
+         "Parameters": {
+            "FunctionName": "${lambda1Arn}",
+            "Payload.$": "$"
+         },
+         "ResultPath": "$.steps.Lambda1",
+         "Next":"Emit Event",
+         "Catch":[
+           {
+              "ErrorEquals":[
+                 "States.ALL"
+              ],
+              "Next":"Catch All"
+           }
+        ]
+     },
+     "Emit Event": {
+      "Type": "Task",
+      "Resource": "arn:aws:states:::sns:publish",
+      "Parameters": {
+        "TopicArn": "${topicArn}",
+        "Message.$": "$",
+        "MessageAttributes": {
+          "attr1": {
+            "DataType": "String",
+            "StringValue": "value1"
+          }
+        }
+      },
+      "End": true
+     },
+     "Catch All":{
+        "Type":"Pass",
+        "Next":"Push To DLQ"
+     },
+     "Push To DLQ":{
+        "Type":"Pass",
+        "End":  true
+     }
+  }
+}

stack/flow1/saga/steps/Lambda1/handler.yml

@@ -0,0 +1,15 @@
+# referenced in ../serverless.yml
+
+Lambda1:
+  handler: flow1/saga/steps/Lambda1/index.handler
+  dependsOn:
+    # - Lambda1SecurityGroup
+    - Lambda1Role
+  role: !GetAtt Lambda1Role.Arn
+  # vpc:
+  #   securityGroupIds:
+  #     - !Ref Lambda1SecurityGroup
+  #   subnetIds:
+  #     - !ImportValue privateSubnet1-dev
+  #     - !ImportValue privateSubnet2-dev
+  #     - !ImportValue privateSubnet3-dev

stack/flow1/saga/steps/Lambda1/index.ts

@@ -0,0 +1,10 @@
+import { Handler } from "aws-lambda";
+
+export const handler: Handler = async (event, context) => {
+  console.log("--- Invoking core systems..");
+  if (Math.round(Math.random()) == 0) {
+    return { result: "OK" };
+  } else {
+    throw new Error("Induced Error");
+  }
+};

stack/flow1/saga/steps/Lambda1/infra.yml

@@ -0,0 +1,53 @@
+Resources:
+  Lambda1Role:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: ${self:provider.stackName}-lambda1Role
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: root
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - logs:CreateLogGroup
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                Resource: arn:aws:logs:*:*:*
+              - Effect: Allow
+                Action:
+                  - ec2:DescribeInstances
+                  - ec2:CreateNetworkInterface
+                  - ec2:AttachNetworkInterface
+                  - ec2:DescribeNetworkInterfaces
+                  - ec2:DeleteNetworkInterface
+                  - ec2:DetachNetworkInterface
+                  - ec2:ModifyNetworkInterfaceAttribute
+                  - ec2:ResetNetworkInterfaceAttribute
+                  - autoscaling:CompleteLifecycleAction
+                Resource: "*"
+              - Effect: Allow
+                Action:
+                  - "xray:PutTraceSegments"
+                  - "xray:PutTelemetryRecords"
+                Resource: "*"
+
+  Lambda1SecurityGroup:
+    Type: "AWS::EC2::SecurityGroup"
+    Properties:
+      GroupDescription: "Lambda Security Group"
+      VpcId: ${env:AWS_VPC_ID}
+      SecurityGroupIngress:
+        - IpProtocol: "-1"
+          Description: "From StepFunction"
+      SecurityGroupEgress:
+        - IpProtocol: "-1"
+          CidrIp: 0.0.0.0/0
+          Description: "To Everywhere"

stack/flow1/triggers/handler.yml

@@ -0,0 +1,18 @@
+# referenced in ../serverless.yml
+
+TriggerLambda1:
+  handler: flow1/triggers/index.handler
+  events:
+    - sqs:
+        arn: !GetAtt TriggerQueue1.Arn
+        batchSize: 1
+  role: !GetAtt TriggerLambda1Role.Arn
+  # vpc:
+  #   securityGroupIds:
+  #     - !Ref TriggerLambda1SecurityGroup
+  #   subnetIds:
+  #     - !ImportValue privateSubnet1-dev
+  #     - !ImportValue privateSubnet2-dev
+  #     - !ImportValue privateSubnet3-dev
+  environment:
+    STATE_MACHINE_ARN: !Ref Flow1 # Gotcha! - Ability to refer ARN of Step Function as a part of same-stack was missing in `serverless-step-functions` plugin, hence the choice of NOT using it.

stack/flow1/triggers/index.ts

@@ -0,0 +1,17 @@
+import { StepFunctions } from "aws-sdk";
+const stepfunctions = new StepFunctions();
+
+export const handler = async (event: any, context: any) => {
+  console.log("--- Invoking step function");
+  try {
+    await stepfunctions
+      .startExecution({
+        stateMachineArn: process.env.STATE_MACHINE_ARN,
+        input: event.Records[0].body,
+      })
+      .promise();
+    console.log("--- Invoked step function successfully");
+  } catch (e) {
+    console.error(e);
+  }
+};