Skip to content

Commit

Permalink
Merge pull request #4979 from connext/feat/networking-improvements
Browse files Browse the repository at this point in the history
feat: flow logs + networking improvements
  • Loading branch information
carlomazzaferro authored Oct 12, 2023
2 parents 4c26445 + b4e632f commit 37d636d
Show file tree
Hide file tree
Showing 11 changed files with 87 additions and 49 deletions.
5 changes: 4 additions & 1 deletion ops/infra/outputs.tf
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@

output "ecr_admin_role" {
value = module.iam.execution_role_arn
}

output "vpc_flow_logs_role" {
value = module.iam.vpc_flow_logs_role_arn
}
10 changes: 0 additions & 10 deletions ops/modules/ecs/variables.tf
Original file line number Diff line number Diff line change
@@ -1,16 +1,6 @@
variable "ecs_cluster_name_prefix" {
}

variable "private_subnets" {
type = list(string)
}

variable "public_subnets" {
type = list(string)
}

variable "vpc_id" {}

variable "domain" {
description = "domain of deployment"
}
Expand Down
4 changes: 4 additions & 0 deletions ops/modules/iam/outputs.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
output "execution_role_arn" {
value = aws_iam_role.ecr_admin_role.arn
}

output "vpc_flow_logs_role_arn" {
value = aws_iam_role.vpc_flow_logs_role.arn
}
39 changes: 39 additions & 0 deletions ops/modules/iam/vpc.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
data "aws_iam_policy_document" "assume_role_policy_document" {
statement {
effect = "Allow"

principals {
type = "Service"
identifiers = ["vpc-flow-logs.amazonaws.com"]
}

actions = ["sts:AssumeRole"]
}
}

resource "aws_iam_role" "vpc_flow_logs_role" {
name = "vpc_flow_logs_role"
assume_role_policy = data.aws_iam_policy_document.assume_role_policy_document.json
}

data "aws_iam_policy_document" "aws_logs_policy" {
statement {
effect = "Allow"

actions = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogGroups",
"logs:DescribeLogStreams",
]

resources = ["*"]
}
}

resource "aws_iam_role_policy" "vpc_flow_logs_policy" {
name = "vpc_flow_logs_policy"
role = aws_iam_role.vpc_flow_logs_role.id
policy = data.aws_iam_policy_document.aws_logs_policy.json
}
2 changes: 1 addition & 1 deletion ops/modules/lambda/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ resource "aws_lambda_function" "executable" {
dynamic "vpc_config" {
for_each = var.lambda_in_vpc ? [1] : []
content {
subnet_ids = var.private_subnets
subnet_ids = var.subnet_ids
security_group_ids = var.lambda_security_groups
}
}
Expand Down
7 changes: 1 addition & 6 deletions ops/modules/lambda/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,7 @@ variable "lambda_in_vpc" {
default = false
}

variable "public_subnets" {
type = list(string)
default = []
}

variable "private_subnets" {
variable "subnet_ids" {
type = list(string)
default = []
}
Expand Down
30 changes: 30 additions & 0 deletions ops/modules/networking/main.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
data "aws_availability_zones" "available" {}

data "aws_iam_role" "vpc_flow_logs" {
name = "vpc_flow_logs_role"
}

resource "aws_vpc" "main" {
cidr_block = var.cidr_block
enable_dns_hostnames = true
Expand Down Expand Up @@ -117,3 +121,29 @@ resource "aws_security_group" "allow_tls" {
cidr_blocks = ["0.0.0.0/0"]
}
}

resource "aws_cloudwatch_log_group" "flow_logs_log_group_private_subnets" {
count = var.az_count
name = "vpc-flow-logs-${var.environment}-${var.stage}-${var.domain}-private-${count.index}"
}

resource "aws_cloudwatch_log_group" "flow_logs_log_group_public_subnets" {
count = var.az_count
name = "vpc-flow-logs-${var.environment}-${var.stage}-${var.domain}-public-${count.index}"
}

resource "aws_flow_log" "vpc_flow_logs_private_subnets" {
count = var.az_count
iam_role_arn = data.aws_iam_role.vpc_flow_logs.arn
log_destination = aws_cloudwatch_log_group.flow_logs_log_group_private_subnets[count.index].arn
traffic_type = "ALL"
subnet_id = aws_subnet.private[count.index].id
}

resource "aws_flow_log" "vpc_flow_logs_public_subnets" {
count = var.az_count
iam_role_arn = data.aws_iam_role.vpc_flow_logs.arn
log_destination = aws_cloudwatch_log_group.flow_logs_log_group_public_subnets[count.index].arn
traffic_type = "ALL"
subnet_id = aws_subnet.main[count.index].id
}
5 changes: 3 additions & 2 deletions ops/modules/service/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,9 @@ resource "aws_ecs_service" "service" {
task_definition = "${aws_ecs_task_definition.service.family}:${max("${aws_ecs_task_definition.service.revision}", "${aws_ecs_task_definition.service.revision}")}"

network_configuration {
security_groups = flatten([var.service_security_groups, aws_security_group.lb.id])
subnets = var.private_subnets
security_groups = flatten([var.service_security_groups, aws_security_group.lb.id])
subnets = var.lb_subnets
assign_public_ip = var.internal_lb ? false : true
}

load_balancer {
Expand Down
4 changes: 0 additions & 4 deletions ops/modules/service/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@ variable "execution_role_arn" {}
variable "cluster_id" {}
variable "vpc_id" {}

variable "private_subnets" {
type = list(string)
}

variable "lb_subnets" {
type = list(string)
}
Expand Down
5 changes: 0 additions & 5 deletions ops/testnet/staging/backend/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ module "postgrest" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = "postgrest/postgrest:v10.0.0.20221011"
Expand Down Expand Up @@ -102,7 +101,6 @@ module "sdk-server" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = var.full_image_name_sdk_server
Expand Down Expand Up @@ -254,7 +252,4 @@ module "ecs" {
environment = var.environment
domain = var.domain
ecs_cluster_name_prefix = "nxtp-ecs"
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
public_subnets = module.network.public_subnets
}
25 changes: 5 additions & 20 deletions ops/testnet/staging/core/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ module "router_subscriber" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = var.full_image_name_router_subscriber
Expand Down Expand Up @@ -62,7 +61,6 @@ module "router_publisher" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = var.full_image_name_router_publisher
Expand Down Expand Up @@ -92,7 +90,6 @@ module "router_executor" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = var.full_image_name_router_executor
Expand Down Expand Up @@ -122,8 +119,7 @@ module "router_web3signer" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
lb_subnets = module.network.private_subnets
docker_image = "ghcr.io/connext/web3signer:latest"
container_family = "router-web3signer"
health_check_path = "/upcheck"
Expand Down Expand Up @@ -167,7 +163,6 @@ module "sequencer_server" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
docker_image = var.full_image_name_sequencer_server
container_family = "sequencer-server"
Expand Down Expand Up @@ -196,7 +191,6 @@ module "sequencer_publisher" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
docker_image = var.full_image_name_sequencer_publisher
container_family = "sequencer-publisher"
Expand Down Expand Up @@ -236,7 +230,6 @@ module "sequencer_subscriber" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = var.full_image_name_sequencer_subscriber
Expand Down Expand Up @@ -277,8 +270,7 @@ module "sequencer_web3signer" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
lb_subnets = module.network.private_subnets
docker_image = "ghcr.io/connext/web3signer:latest"
container_family = "sequencer-web3signer"
health_check_path = "/upcheck"
Expand Down Expand Up @@ -307,7 +299,6 @@ module "lighthouse_prover_subscriber" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
internal_lb = false
docker_image = var.full_image_name_lighthouse_prover_subscriber
Expand All @@ -325,6 +316,7 @@ module "lighthouse_prover_subscriber" {
cert_arn = var.certificate_arn_testnet
container_env_vars = concat(local.lighthouse_prover_subscriber_env_vars, [{ name = "LIGHTHOUSE_SERVICE", value = "prover-sub" }])
}

module "lighthouse_prover_subscriber_auto_scaling" {
source = "../../../modules/auto-scaling"
stage = var.stage
Expand Down Expand Up @@ -396,7 +388,6 @@ module "relayer" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
docker_image = var.full_image_name_relayer
container_family = "relayer"
Expand Down Expand Up @@ -426,8 +417,7 @@ module "relayer_web3signer" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
lb_subnets = module.network.private_subnets
docker_image = "ghcr.io/connext/web3signer:latest"
container_family = "relayer-web3signer"
health_check_path = "/upcheck"
Expand Down Expand Up @@ -456,7 +446,6 @@ module "watcher" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
docker_image = var.full_image_name_watcher
container_family = "watcher"
Expand Down Expand Up @@ -486,8 +475,7 @@ module "watcher_web3signer" {
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
cluster_id = module.ecs.ecs_cluster_id
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
lb_subnets = module.network.public_subnets
lb_subnets = module.network.private_subnets
docker_image = "ghcr.io/connext/web3signer:latest"
container_family = "watcher-web3signer"
health_check_path = "/upcheck"
Expand Down Expand Up @@ -530,9 +518,6 @@ module "ecs" {
environment = var.environment
domain = var.domain
ecs_cluster_name_prefix = "nxtp-ecs"
vpc_id = module.network.vpc_id
private_subnets = module.network.private_subnets
public_subnets = module.network.public_subnets
}

module "sequencer_cache" {
Expand Down

0 comments on commit 37d636d

Please sign in to comment.