Switching to on-demand compute instances

tilezen · May 7, 2021 · 6d99e95 · 6d99e95
1 parent 7e2ef95
commit 6d99e95
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 37 deletions.
diff --git a/batch-setup/batch_setup.py b/batch-setup/batch_setup.py
@@ -148,34 +148,6 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen
         boto_iam, boto_ec2, instanceRoleName, instanceRoleName)
     print("Using ECS instance profile %s" % instanceProfileArn)
 
-    # Create the spot fleet role
-    # https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html
-    spotIamFleetRoleName = "AmazonEC2SpotFleetRole"
-    spotIamRoleDocument = {
-        "Version": "2012-10-17",
-        "Statement": [
-            {
-                "Effect": "Allow",
-                "Principal": {
-                    "Service": "spotfleet.amazonaws.com"
-                },
-                "Action": "sts:AssumeRole"
-            }
-        ]
-    }
-    spotIamFleetRoleArn = get_or_create_role(
-        boto_iam, spotIamFleetRoleName, spotIamRoleDocument)
-    print("Using EC2 Spot Fleet role %s" % spotIamFleetRoleArn)
-
-    spotFleetTaggingRolePolicy = find_policy(
-        boto_iam, 'AmazonEC2SpotFleetTaggingRole')
-    boto_iam.attach_role_policy(
-        PolicyArn=spotFleetTaggingRolePolicy['Arn'],
-        RoleName=spotIamFleetRoleName,
-    )
-    print("Attached Spot Fleet Tagging Role to EC2 Spot Fleet role %s"
-          % spotIamFleetRoleArn)
-
     # Create a compute environment for raw tile rendering
     try:
         response = boto_batch.describe_compute_environments(
@@ -192,7 +164,7 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen
             state="ENABLED",
             serviceRole=serviceRoleArn,
             computeResources=dict(
-                type='SPOT',
+                type='EC2',
                 minvCpus=0,
                 maxvCpus=max_vcpus,
                 desiredvCpus=0,
@@ -205,7 +177,6 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen
                     'cost_resource_group': run_id,
                 },
                 bidPercentage=60,
-                spotIamFleetRole=spotIamFleetRoleArn,
                 subnets=subnet_ids,
                 securityGroupIds=securityGroupIds,
             )

diff --git a/batch-setup/make_meta_tiles.py b/batch-setup/make_meta_tiles.py
@@ -378,16 +378,17 @@ def viable_container_overrides(mem_mb):
     :return: the amount of mem you need to request for AWS batch to honor it, the amount of vcpus you must request
     """
     if mem_mb < 512:
-        return 512
+        return 512, 1
 
     if mem_mb % 1024 == 0:
-        return mem_mb
+        return mem_mb, 1
 
-    # truncate number / 1024, then bump by 1
-    desired_mem_mb = (int(mem_mb) / 1024 + 1) * 1024
+    mem_gb_truncated = mem_mb / 1024
+    next_gb = mem_gb_truncated + 1
+    desired_mem_mb = next_gb * 1024
 
-    max_mem_per_vcpu = 8 * 1024
-    vcpus = (desired_mem_mb - 1) / max_mem_per_vcpu + 1
+    max_mem_per_vcpu: int = 8 * 1024
+    vcpus: int = 1 + (desired_mem_mb - 1)/max_mem_per_vcpu
 
     return desired_mem_mb, vcpus
 
@@ -413,7 +414,7 @@ def enqueue_tiles(config_file, tile_list_file, check_metatile_exists, tile_speci
         mem_mb = int(tile_specifier.get_mem_reqs_mb(coord_line))
         adjusted_mem = mem_mb * mem_multiplier
 
-        # now that we know what we think we want, pick something AWS actually supports
+        # now that we know what we want, pick something AWS actually supports
         viable_mem_request, required_min_cpus = viable_container_overrides(adjusted_mem)
 
         update_container_overrides(cfg, viable_mem_request, mem_max, required_min_cpus)