Merge pull request #38 from StanfordBioinformatics/feature/cloudforma…

…tion-stack

Add CF Stack + Change Scheduler to use Cloudformation Stack output

Hummingbird/AWS/compute_environment.json

@@ -1,5 +1,5 @@
 {
-  "computeEnvironmentName": "hummingbird-env",
+  "computeEnvironmentName": "",
   "type": "MANAGED",
   "state": "ENABLED",
   "computeResources": {
@@ -14,15 +14,14 @@
     ],
     "securityGroupIds": [
     ],
-    "ec2KeyPair": "",
-    "instanceRole": "ecsInstanceRole",
+    "instanceRole": "",
     "tags": {
       "KeyName": "hummingbird"
     },
     "launchTemplate": {
-      "launchTemplateName": "hummingbird_disk_launch_template",
+      "launchTemplateName": "hummingbird",
       "version": "$Latest"
     }
   },
-  "serviceRole": "<AWSServiceRoleForBatch>"
+  "serviceRole": ""
 }

Hummingbird/AWS/hummingbird-cloudformation.template

@@ -0,0 +1,347 @@
+Description: This template deploys an environment for running Hummingbird.
+  Resources include VPC, with a pair of public and private subnets spread
+  across two Availability Zones, IAM roles for AWS Batch/ECS.
+  It deploys an internet gateway, with a default route on the public subnets.
+  It deploys a pair of NAT gateways (one in each AZ), and default routes for
+  them in the private subnets.
+
+Parameters:
+  EnvironmentName:
+    Description: An environment name that is prefixed to resource names (e.g. hummingbird)
+    Type: String
+    AllowedPattern: '[-_a-zA-Z0-9]*'
+    ConstraintDescription: Can contain only alphanumeric characters, dashes and underscores.
+
+  VpcCIDR:
+    Description: Please enter the IP range (CIDR notation) for this VPC
+    Type: String
+    Default: 10.192.0.0/16
+
+  PublicSubnet1CIDR:
+    Description: Please enter the IP range (CIDR notation) for the public subnet in the first Availability Zone
+    Type: String
+    Default: 10.192.10.0/24
+
+  PublicSubnet2CIDR:
+    Description: Please enter the IP range (CIDR notation) for the public subnet in the second Availability Zone
+    Type: String
+    Default: 10.192.11.0/24
+
+  PrivateSubnet1CIDR:
+    Description: Please enter the IP range (CIDR notation) for the private subnet in the first Availability Zone
+    Type: String
+    Default: 10.192.20.0/24
+
+  PrivateSubnet2CIDR:
+    Description: Please enter the IP range (CIDR notation) for the private subnet in the second Availability Zone
+    Type: String
+    Default: 10.192.21.0/24
+
+Resources:
+  VPC:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: !Ref VpcCIDR
+      EnableDnsSupport: true
+      EnableDnsHostnames: true
+      Tags:
+        - Key: Name
+          Value: !Ref EnvironmentName
+
+  InternetGateway:
+    Type: AWS::EC2::InternetGateway
+    Properties:
+      Tags:
+        - Key: Name
+          Value: !Ref EnvironmentName
+
+  InternetGatewayAttachment:
+    Type: AWS::EC2::VPCGatewayAttachment
+    Properties:
+      InternetGatewayId: !Ref InternetGateway
+      VpcId: !Ref VPC
+
+  PublicSubnet1:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [ 0, !GetAZs '' ]
+      CidrBlock: !Ref PublicSubnet1CIDR
+      MapPublicIpOnLaunch: true
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Public Subnet (AZ1)
+
+  PublicSubnet2:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
+      CidrBlock: !Ref PublicSubnet2CIDR
+      MapPublicIpOnLaunch: true
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Public Subnet (AZ2)
+
+  PrivateSubnet1:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [ 0, !GetAZs  '' ]
+      CidrBlock: !Ref PrivateSubnet1CIDR
+      MapPublicIpOnLaunch: false
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Subnet (AZ1)
+
+  PrivateSubnet2:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
+      CidrBlock: !Ref PrivateSubnet2CIDR
+      MapPublicIpOnLaunch: false
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Subnet (AZ2)
+
+  NatGateway1EIP:
+    Type: AWS::EC2::EIP
+    DependsOn: InternetGatewayAttachment
+    Properties:
+      Domain: vpc
+
+  NatGateway2EIP:
+    Type: AWS::EC2::EIP
+    DependsOn: InternetGatewayAttachment
+    Properties:
+      Domain: vpc
+
+  NatGateway1:
+    Type: AWS::EC2::NatGateway
+    Properties:
+      AllocationId: !GetAtt NatGateway1EIP.AllocationId
+      SubnetId: !Ref PublicSubnet1
+
+  NatGateway2:
+    Type: AWS::EC2::NatGateway
+    Properties:
+      AllocationId: !GetAtt NatGateway2EIP.AllocationId
+      SubnetId: !Ref PublicSubnet2
+
+  PublicRouteTable:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref VPC
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Public Routes
+
+  DefaultPublicRoute:
+    Type: AWS::EC2::Route
+    DependsOn: InternetGatewayAttachment
+    Properties:
+      RouteTableId: !Ref PublicRouteTable
+      DestinationCidrBlock: 0.0.0.0/0
+      GatewayId: !Ref InternetGateway
+
+  PublicSubnet1RouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PublicRouteTable
+      SubnetId: !Ref PublicSubnet1
+
+  PublicSubnet2RouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PublicRouteTable
+      SubnetId: !Ref PublicSubnet2
+
+
+  PrivateRouteTable1:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref VPC
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Routes (AZ1)
+
+  DefaultPrivateRoute1:
+    Type: AWS::EC2::Route
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable1
+      DestinationCidrBlock: 0.0.0.0/0
+      NatGatewayId: !Ref NatGateway1
+
+  PrivateSubnet1RouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable1
+      SubnetId: !Ref PrivateSubnet1
+
+  PrivateRouteTable2:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref VPC
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Routes (AZ2)
+
+  DefaultPrivateRoute2:
+    Type: AWS::EC2::Route
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable2
+      DestinationCidrBlock: 0.0.0.0/0
+      NatGatewayId: !Ref NatGateway2
+
+  PrivateSubnet2RouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable2
+      SubnetId: !Ref PrivateSubnet2
+
+  BatchEC2SecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: "AWS Batch EC2 Security Group"
+      GroupName: !Sub ${EnvironmentName}-sg
+      SecurityGroupEgress:
+        - IpProtocol: "-1"
+          CidrIp: 0.0.0.0/0
+          Description: Allow all outbound traffic
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName}-SG
+      VpcId: !Ref VPC
+
+  # https://docs.aws.amazon.com/batch/latest/userguide/instance_IAM_role.html
+  ECSInstanceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      Description: "Allows EC2 instances in an ECS cluster to access ECS."
+      AssumeRolePolicyDocument:
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - 'ec2.amazonaws.com'
+            Action:
+              - 'sts:AssumeRole'
+      ManagedPolicyArns:
+        - 'arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role'
+        - 'arn:aws:iam::aws:policy/AmazonS3FullAccess'
+        - 'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleForSSM'
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName}-ECSInstanceRole
+
+  ECSInstanceProfileRole:
+    Type: AWS::IAM::InstanceProfile
+    Properties:
+      InstanceProfileName: !Ref ECSInstanceRole
+      Roles:
+        - !Ref ECSInstanceRole
+
+  # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_execution_IAM_role.html
+  ECSTaskExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      Description: "Allows ECS tasks to call AWS services on your behalf."
+      AssumeRolePolicyDocument:
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - 'ecs-tasks.amazonaws.com'
+            Action:
+              - 'sts:AssumeRole'
+      ManagedPolicyArns:
+        - 'arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy'
+        - 'arn:aws:iam::aws:policy/AmazonS3FullAccess'
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName}-ECSTaskExecutionRole
+
+  # https://docs.aws.amazon.com/batch/latest/userguide/service_IAM_role.html
+  BatchServiceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      Description: "Allows Batch to create and manage AWS resources on your behalf."
+      AssumeRolePolicyDocument:
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - 'batch.amazonaws.com'
+            Action:
+              - 'sts:AssumeRole'
+      ManagedPolicyArns:
+        - 'arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole'
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName}-BatchServiceRole
+
+Outputs:
+  VPC:
+    Description: A reference to the created VPC
+    Value: !Ref VPC
+
+  PublicSubnets:
+    Description: A list of the public subnets
+    Value: !Join [ ",", [ !Ref PublicSubnet1, !Ref PublicSubnet2 ] ]
+
+  PrivateSubnets:
+    Description: A list of the private subnets
+    Value: !Join [ ",", [ !Ref PrivateSubnet1, !Ref PrivateSubnet2 ] ]
+
+  PublicSubnet1:
+    Description: A reference to the public subnet in the 1st Availability Zone
+    Value: !Ref PublicSubnet1
+
+  PublicSubnet2:
+    Description: A reference to the public subnet in the 2nd Availability Zone
+    Value: !Ref PublicSubnet2
+
+  PrivateSubnet1:
+    Description: A reference to the private subnet in the 1st Availability Zone
+    Value: !Ref PrivateSubnet1
+
+  PrivateSubnet2:
+    Description: A reference to the private subnet in the 2nd Availability Zone
+    Value: !Ref PrivateSubnet2
+
+  BatchEC2SecurityGroup:
+    Description: Security Group for AWS Batch EC2
+    Value: !Ref BatchEC2SecurityGroup
+
+  ECSInstanceRole:
+    Description: ECS Instance Role
+    Value: !Ref ECSInstanceRole
+
+  ECSInstanceRoleARN:
+    Description: ECS Instance Role ARN
+    Value: !GetAtt ECSInstanceRole.Arn
+
+  ECSInstanceProfileRole:
+    Description: ECS Instance Profile Role
+    Value: !Ref ECSInstanceProfileRole
+
+  ECSInstanceProfileRoleARN:
+    Description: ECS Instance Profile Role ARN
+    Value: !GetAtt ECSInstanceProfileRole.Arn
+
+  ECSTaskExecutionRole:
+    Description: ECS Task Execution Role
+    Value: !Ref ECSTaskExecutionRole
+
+  ECSTaskExecutionRoleARN:
+    Description: ECS Task Execution Role ARN
+    Value: !GetAtt ECSTaskExecutionRole.Arn
+
+  BatchServiceRole:
+    Description: AWS Batch Service Role
+    Value: !Ref BatchServiceRole
+
+  BatchServiceRoleARN:
+    Description: AWS Batch Service Role ARN
+    Value: !GetAtt BatchServiceRole.Arn

Hummingbird/AWS/job-definition.json

@@ -10,16 +10,14 @@
       "-c",
       "/usr/local/bin/aws_fetch_and_run.sh"
     ],
-    "jobRoleArn": "<ecsTaskExecutionRole>",
+    "jobRoleArn": "",
     "volumes": [
     ],
     "environment": [
     ],
     "mountPoints": [
     ],
     "ulimits": [
-    ],
-    "resourceRequirements": [
     ]
   },
   "retryStrategy": {