feat: DEVOPS-1855 implement scalable external reserved APIs (#2349)

infra/tf/main.tf

@@ -78,23 +78,23 @@ resource "google_storage_bucket" "persistence" {
 resource "google_storage_bucket_iam_binding" "persistence_bucket_admins" {
   bucket = google_storage_bucket.persistence.name
   role   = "roles/storage.objectAdmin"
-  members = [
+  members = concat([
     "serviceAccount:${module.bootstraps.service_account.email}",
     "serviceAccount:${module.validators.service_account.email}",
     "serviceAccount:${module.apis.service_account.email}",
     "serviceAccount:${module.checkpoints.service_account.email}",
-    "serviceAccount:${module.persistences.service_account.email}",
-    "serviceAccount:${module.queries.service_account.email}",
-    "serviceAccount:${module.graphs.service_account.email}"
-  ]
+    "serviceAccount:${module.persistences.service_account.email}"
+    ],
+    [for private_api in module.private_apis : "serviceAccount:${private_api.service_account.email}"]
+  )
 }
 
 ################################################################################
 # FIREWALL POLICIES
 ################################################################################
 
-resource "google_compute_firewall" "allow_ingress_from_iap" {
-  name    = "${var.chain_name}-allow-ingress-from-iap"
+resource "google_compute_firewall" "allow_ssh_from_iap" {
+  name    = "${var.chain_name}-allow-ssh-from-iap"
   network = local.network_name
 
   direction     = "INGRESS"
@@ -133,7 +133,7 @@ resource "google_compute_firewall" "allow_external_jsonrpc" {
   network = local.network_name
 
   direction     = "INGRESS"
-  source_ranges = ["0.0.0.0/0"]
+  source_ranges = concat(local.google_load_balancer_ip_ranges, ["0.0.0.0/0", local.iap_ip_range])
 
   target_tags = [var.chain_name]
 
@@ -143,14 +143,29 @@ resource "google_compute_firewall" "allow_external_jsonrpc" {
   }
 }
 
+resource "google_compute_firewall" "allow_jsonrpc_from_iap" {
+  name    = "${var.chain_name}-allow-jsonrpc-from-iap"
+  network = local.network_name
+
+  direction     = "INGRESS"
+  source_ranges = [local.monitoring_ip_range, local.iap_ip_range]
+
+  target_tags = [var.chain_name]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["4202"]
+  }
+}
+
 resource "google_compute_firewall" "allow_monitor_healthcheck" {
   name    = "${var.chain_name}-allow-monitor-healthcheck"
   network = local.network_name
 
   direction     = "INGRESS"
   source_ranges = [local.monitoring_ip_range]
 
-  target_tags = [format("%s", var.chain_name)]
+  target_tags = [var.chain_name]
 
   allow {
     protocol = "tcp"

infra/tf/modules/node/locals.tf

@@ -18,7 +18,7 @@ locals {
         count           = node.count
         region          = node.region != null ? node.region : ([for region in data.google_compute_zones.available : region if contains(region.names, node.zone)][0].region)
         zone            = node.zone != null ? node.zone : ([for region in data.google_compute_zones.available : region if region.region == node.region][0].names[n % length([for region in data.google_compute_zones.available : region if region.region == node.region][0].names)])
-        resource_name   = format("%s-%s-%s-%s-%s", var.chain_name, var.role, var.region_mappings[(node.region != null ? node.region : ([for region in data.google_compute_zones.available : region if contains(region.names, node.zone)][0].region))], n, random_id.name_suffix.hex)
+        resource_name   = format("%s-%s-%s-%s-%s", var.chain_name, lookup(local.labels, "private-api", var.role), var.region_mappings[(node.region != null ? node.region : ([for region in data.google_compute_zones.available : region if contains(region.names, node.zone)][0].region))], n, random_id.name_suffix.hex)
         resource_id     = format("%s-%s-%s-%s", var.chain_name, var.role, var.region_mappings[(node.region != null ? node.region : ([for region in data.google_compute_zones.available : region if contains(region.names, node.zone)][0].region))], n)
       }
     ]

infra/tf/modules/node/variables.tf

@@ -8,8 +8,7 @@ variable "node_role_mappings" {
     validator   = "val",
     checkpoint  = "che",
     persistence = "per",
-    query       = "que",
-    graph       = "gra",
+    private-api = "pap",
     sentry      = "sen",
   }
 }
@@ -72,8 +71,8 @@ variable "role" {
   description = "VM role"
   type        = string
   validation {
-    condition     = contains(["bootstrap", "api", "validator", "apps", "checkpoint", "persistence", "query", "graph", "sentry"], var.role)
-    error_message = "The role value must be one of:  'bootstrap', 'api', 'validator', 'apps', 'checkpoint', 'persistence', 'query', 'graph', 'sentry'."
+    condition     = contains(["bootstrap", "api", "validator", "apps", "checkpoint", "persistence", "private-api", "sentry"], var.role)
+    error_message = "The role value must be one of:  'bootstrap', 'api', 'validator', 'apps', 'checkpoint', 'persistence', 'private-api', 'sentry'."
   }
 }
 

infra/tf/outputs.tf

@@ -35,12 +35,11 @@ output "persistence_external_ip" {
   value       = module.persistences.external_ip
 }
 
-output "query_external_ip" {
-  description = "The provisioned bootstrap external IPs"
-  value       = module.queries.external_ip
-}
-
-output "graph_external_ip" {
-  description = "The provisioned bootstrap external IPs"
-  value       = module.graphs.external_ip
+output "private_api_dns_names" {
+  description = "The provisioned private API DNS names"
+  value = merge(flatten([
+    for name, private_api in module.private_apis : {
+      for idx, key in keys(private_api.external_ip) : format("%s.%s", var.private_api[name].dns_names[idx], var.chain_name) => private_api.external_ip[key]
+    }
+  ])...)
 }

infra/tf/private_api.tf

@@ -0,0 +1,92 @@
+################################################################################
+# PRIVATE API INSTANCES
+################################################################################
+
+resource "google_compute_firewall" "allow_private_api_external_jsonrpc" {
+  for_each = var.private_api
+
+  name    = format("%s-allow-api-%s-external-jsonrpc", var.chain_name, each.key)
+  network = local.network_name
+
+  direction     = "INGRESS"
+  source_ranges = concat([local.iap_ip_range], each.value.firewall_source_ranges)
+  priority      = 990
+
+  target_tags = [format("%s-api-%s", var.chain_name, each.key)]
+
+  allow {
+    protocol = "tcp"
+    ports    = ["4201"]
+  }
+}
+
+resource "google_compute_firewall" "deny_private_api_external_jsonrpc" {
+  for_each = var.private_api
+
+  name    = format("%s-deny-api-%s-external-jsonrpc", var.chain_name, each.key)
+  network = local.network_name
+
+  direction     = "INGRESS"
+  source_ranges = ["0.0.0.0/0"]
+
+  target_tags = [format("%s-api-%s", var.chain_name, each.key)]
+
+  deny {
+    protocol = "tcp"
+    ports    = ["4201"]
+  }
+}
+
+module "private_apis" {
+  source = "./modules/node"
+
+  for_each = var.private_api
+
+  config     = each.value
+  chain_name = var.chain_name
+
+  role         = "private-api"
+  labels       = merge(local.labels, { "private-api" = each.key })
+  network_tags = [format("%s-api-%s", var.chain_name, each.key)]
+
+  metadata = {
+    subdomain   = base64encode(var.subdomain)
+    private-api = base64encode(each.key)
+  }
+
+  service_account_iam = local.default_service_account_iam
+}
+
+resource "google_compute_instance_group" "private_api" {
+  # Flatten the zones across all modules into a single map
+  for_each = {
+    for item in flatten([
+      for idx, private_api in module.private_apis : [
+        for zone in private_api.zones : {
+          idx       = idx
+          zone      = zone
+          instances = [for instance in private_api.instances : instance.self_link if instance.zone == zone]
+        }
+      ]
+    ]) :
+    "${item.idx}-${item.zone}" => {
+      idx       = item.idx
+      zone      = item.zone
+      instances = item.instances
+    }
+  }
+
+  name      = "${var.chain_name}-private-api-${each.value.idx}-${each.value.zone}"
+  zone      = each.value.zone
+  instances = each.value.instances
+
+  named_port {
+    name = "jsonrpc"
+    port = 4201
+  }
+
+  named_port {
+    name = "health"
+    port = 8080
+  }
+}