diff --git a/examples/tfvars/ecr_endpoint.tfvars b/examples/tfvars/ecr_endpoint.tfvars
index d921a904..e8defaa4 100644
--- a/examples/tfvars/ecr_endpoint.tfvars
+++ b/examples/tfvars/ecr_endpoint.tfvars
@@ -16,4 +16,5 @@ default_node_groups = {
network = {
create_ecr_endpoint = true
+ create_s3_interface = true
}
diff --git a/modules/eks/README.md b/modules/eks/README.md
index 75081cb4..5fea356c 100644
--- a/modules/eks/README.md
+++ b/modules/eks/README.md
@@ -53,6 +53,7 @@
| [aws_security_group_rule.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
| [aws_security_group_rule.netapp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
| [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
+| [aws_security_group_rule.s3_endpoint](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
| [null_resource.kubeconfig](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
| [terraform_data.run_k8s_pre_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource |
| [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@@ -84,7 +85,7 @@
| [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no |
| [karpenter](#input\_karpenter) | karpenter = {
enabled = Toggle installation of Karpenter.
namespace = Namespace to install Karpenter.
version = Configure the version for Karpenter.
} |
object({| `{}` | no | | [kms\_info](#input\_kms\_info) | key\_id = KMS key id.
enabled = optional(bool, false)
namespace = optional(string, "karpenter")
version = optional(string, "1.0.6")
})
object({| n/a | yes | -| [network\_info](#input\_network\_info) | id = VPC ID.
key_id = string
key_arn = string
enabled = bool
})
object({| n/a | yes | +| [network\_info](#input\_network\_info) | id = VPC ID.
vpc_id = string
ecr_endpoint = optional(object({
security_group_id = optional(string, null)
}), null)
subnets = object({
public = list(object({
name = string
subnet_id = string
az = string
az_id = string
}))
private = list(object({
name = string
subnet_id = string
az = string
az_id = string
}))
pod = list(object({
name = string
subnet_id = string
az = string
az_id = string
}))
})
vpc_cidrs = optional(string, "10.0.0.0/16")
})
object({| n/a | yes | | [node\_iam\_policies](#input\_node\_iam\_policies) | Additional IAM Policy Arns for Nodes | `list(string)` | n/a | yes | | [privatelink](#input\_privatelink) | {
vpc_id = string
ecr_endpoint = optional(object({
security_group_id = optional(string, null)
}), null)
s3_endpoint = optional(object({
security_group_id = optional(string, null)
}), null)
subnets = object({
public = list(object({
name = string
subnet_id = string
az = string
az_id = string
}))
private = list(object({
name = string
subnet_id = string
az = string
az_id = string
}))
pod = list(object({
name = string
subnet_id = string
az = string
az_id = string
}))
})
vpc_cidrs = optional(string, "10.0.0.0/16")
})
object({| `{}` | no | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | diff --git a/modules/eks/node-group.tf b/modules/eks/node-group.tf index 4e5d4b04..4b3589c4 100644 --- a/modules/eks/node-group.tf +++ b/modules/eks/node-group.tf @@ -95,3 +95,14 @@ resource "aws_security_group_rule" "ecr_endpoint" { description = "ECR Endpoint access from EKS nodes." source_security_group_id = aws_security_group.eks_nodes.id } + +resource "aws_security_group_rule" "s3_endpoint" { + count = var.network_info.s3_endpoint != null ? 1 : 0 + security_group_id = var.network_info.s3_endpoint.security_group_id + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + description = "S3 Endpoint access from EKS nodes." + source_security_group_id = aws_security_group.eks_nodes.id +} diff --git a/modules/eks/variables.tf b/modules/eks/variables.tf index 7a47ee4f..e0dc13c7 100644 --- a/modules/eks/variables.tf +++ b/modules/eks/variables.tf @@ -24,6 +24,9 @@ variable "network_info" { ecr_endpoint = { security_group_id = ECR Endpoint security group id. } + s3_endpoint = { + security_group_id = S3 Endpoint security group id. + } subnets = { public = List of public Subnets. [{ @@ -53,6 +56,9 @@ variable "network_info" { ecr_endpoint = optional(object({ security_group_id = optional(string, null) }), null) + s3_endpoint = optional(object({ + security_group_id = optional(string, null) + }), null) subnets = object({ public = list(object({ name = string diff --git a/modules/infra/README.md b/modules/infra/README.md index 0bb11617..2f09d5fe 100644 --- a/modules/infra/README.md +++ b/modules/infra/README.md @@ -62,7 +62,7 @@ | [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no | | [karpenter\_node\_groups](#input\_karpenter\_node\_groups) | Configuration for EKS node groups managed by Karpenter. These node groups are designed to dynamically scale
enabled = optional(bool, false)
namespace = optional(string, "domino-platform")
monitoring_bucket = optional(string, null)
route53_hosted_zone_name = optional(string, null)
vpc_endpoint_services = optional(list(object({
name = optional(string)
ports = optional(list(number))
cert_arn = optional(string)
private_dns = optional(string)
})), [])
})
map(object({| `{}` | no | | [kms](#input\_kms) | enabled = "Toggle, if set use either the specified KMS key\_id or a Domino-generated one"
ami = optional(string, null)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m6a.large"])
spot = optional(bool, false)
min_per_az = optional(number, 1)
max_per_az = optional(number, 1)
max_unavailable_percentage = optional(number, 50)
max_unavailable = optional(number)
desired_per_az = optional(number, 1)
availability_zone_ids = list(string)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "karpenter"
})
taints = optional(list(object({
key = string
value = optional(string)
effect = string
})), [])
tags = optional(map(string), {})
gpu = optional(bool, null)
volume = optional(object({
size = optional(string, "30")
type = optional(string, "gp3")
iops = optional(number)
throughput = optional(number, 500)
}), {})
single_nodegroup = optional(bool, false)
}))
object({| `{}` | no | -| [network](#input\_network) | vpc = {
enabled = optional(bool, true)
key_id = optional(string, null)
additional_policies = optional(list(string), [])
})
object({| `{}` | no | +| [network](#input\_network) | vpc = {
vpc = optional(object({
id = optional(string, null)
subnets = optional(object({
private = optional(list(string), [])
public = optional(list(string), [])
pod = optional(list(string), [])
}), {})
}), {})
network_bits = optional(object({
public = optional(number, 27)
private = optional(number, 19)
pod = optional(number, 19)
}
), {})
cidrs = optional(object({
vpc = optional(string, "10.0.0.0/16")
pod = optional(string, "100.64.0.0/16")
}), {})
use_pod_cidr = optional(bool, true)
create_ecr_endpoint = optional(bool, false)
})
object({| `{}` | no | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | | [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | | [storage](#input\_storage) | storage = {
vpc = optional(object({
id = optional(string, null)
subnets = optional(object({
private = optional(list(string), [])
public = optional(list(string), [])
pod = optional(list(string), [])
}), {})
}), {})
network_bits = optional(object({
public = optional(number, 27)
private = optional(number, 19)
pod = optional(number, 19)
}
), {})
cidrs = optional(object({
vpc = optional(string, "10.0.0.0/16")
pod = optional(string, "100.64.0.0/16")
}), {})
use_pod_cidr = optional(bool, true)
create_ecr_endpoint = optional(bool, false)
create_s3_interface = optional(bool, false)
})
object({| `{}` | no | diff --git a/modules/infra/submodules/network/README.md b/modules/infra/submodules/network/README.md index eab302d1..ed42dc4d 100644 --- a/modules/infra/submodules/network/README.md +++ b/modules/infra/submodules/network/README.md @@ -6,13 +6,13 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | ~> 5.0 | +| [aws](#requirement\_aws) | ~> 5.7 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | ~> 5.0 | +| [aws](#provider\_aws) | ~> 5.7 | ## Modules @@ -35,6 +35,7 @@ No modules. | [aws_route_table_association.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | | [aws_route_table_association.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | | [aws_security_group.ecr_endpoint](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.s3_endpoint](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_subnet.pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_subnet.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_subnet.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | @@ -42,6 +43,7 @@ No modules. | [aws_vpc_endpoint.ecr_api](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | | [aws_vpc_endpoint.ecr_dkr](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | | [aws_vpc_endpoint.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | +| [aws_vpc_endpoint.s3_interface](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | | [aws_vpc_ipv4_cidr_block_association.pod_cidr](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_ipv4_cidr_block_association) | resource | | [aws_availability_zone.zones](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zone) | data source | | [aws_network_acls.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/network_acls) | data source | @@ -58,7 +60,7 @@ No modules. | [add\_eks\_elb\_tags](#input\_add\_eks\_elb\_tags) | Toggle k8s cluster tag on subnet | `bool` | `true` | no | | [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes | | [flow\_log\_bucket\_arn](#input\_flow\_log\_bucket\_arn) | Bucket for vpc flow logging | `object({ arn = string })` | `null` | no | -| [network](#input\_network) | vpc = {
filesystem_type = optional(string, "efs")
efs = optional(object({
access_point_path = optional(string, "/domino")
backup_vault = optional(object({
create = optional(bool, true)
force_destroy = optional(bool, true)
backup = optional(object({
schedule = optional(string, "0 12 * * ? *")
cold_storage_after = optional(number, 35)
delete_after = optional(number, 125)
}), {})
}), {})
}), {})
netapp = optional(object({
migrate_from_efs = optional(object({
enabled = optional(bool, false)
datasync = optional(object({
enabled = optional(bool, false)
target = optional(string, "netapp")
schedule = optional(string, "cron(0 */4 * * ? *)")
verify_mode = optional(string, "ONLY_FILES_TRANSFERRED")
}), {})
}), {})
deployment_type = optional(string, "SINGLE_AZ_1")
storage_capacity = optional(number, 1024)
throughput_capacity = optional(number, 128)
automatic_backup_retention_days = optional(number, 90)
daily_automatic_backup_start_time = optional(string, "00:00")
storage_capacity_autosizing = optional(object({
enabled = optional(bool, false)
threshold = optional(number, 70)
percent_capacity_increase = optional(number, 30)
notification_email_address = optional(string, "")
}), {})
volume = optional(object({
create = optional(bool, true)
name_suffix = optional(string, "domino_shared_storage")
storage_efficiency_enabled = optional(bool, true)
junction_path = optional(string, "/domino")
size_in_megabytes = optional(number, 1048576)
}), {})
}), {})
s3 = optional(object({
force_destroy_on_deletion = optional(bool, true)
}), {})
ecr = optional(object({
force_destroy_on_deletion = optional(bool, true)
}), {}),
enable_remote_backup = optional(bool, false)
costs_enabled = optional(bool, true)
})
object({| n/a | yes | +| [network](#input\_network) | vpc = {
vpc = optional(object({
id = optional(string)
subnets = optional(object({
private = optional(list(string))
public = optional(list(string))
pod = optional(list(string))
}))
}))
network_bits = optional(object({
public = optional(number)
private = optional(number)
pod = optional(number)
}
))
cidrs = optional(object({
vpc = optional(string)
pod = optional(string)
}))
use_pod_cidr = optional(bool)
create_ecr_endpoint = optional(bool, false)
})
object({| n/a | yes | | [node\_groups](#input\_node\_groups) | EKS managed node groups definition. |
vpc = optional(object({
id = optional(string)
subnets = optional(object({
private = optional(list(string))
public = optional(list(string))
pod = optional(list(string))
}))
}))
network_bits = optional(object({
public = optional(number)
private = optional(number)
pod = optional(number)
}
))
cidrs = optional(object({
vpc = optional(string)
pod = optional(string)
}))
use_pod_cidr = optional(bool)
create_ecr_endpoint = optional(bool, false)
create_s3_interface = optional(bool, false)
})
map(object({| n/a | yes | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | diff --git a/modules/infra/submodules/network/main.tf b/modules/infra/submodules/network/main.tf index 3dc5f6df..cabe3b0c 100644 --- a/modules/infra/submodules/network/main.tf +++ b/modules/infra/submodules/network/main.tf @@ -2,6 +2,7 @@ locals { create_vpc = var.network.vpc.id == null provided_vpc = var.network.vpc.id != null create_ecr_endpoint = local.create_vpc && var.network.create_ecr_endpoint + create_s3_interface = local.create_vpc && var.network.create_s3_interface } data "aws_subnet" "public" { diff --git a/modules/infra/submodules/network/outputs.tf b/modules/infra/submodules/network/outputs.tf index 8d5a2435..52b03217 100644 --- a/modules/infra/submodules/network/outputs.tf +++ b/modules/infra/submodules/network/outputs.tf @@ -20,5 +20,8 @@ output "info" { ecr_endpoint = local.create_ecr_endpoint ? { security_group_id = aws_security_group.ecr_endpoint[0].id } : null + s3_endpoint = local.create_s3_interface ? { + security_group_id = aws_security_group.s3_endpoint[0].id + } : null } } diff --git a/modules/infra/submodules/network/variables.tf b/modules/infra/submodules/network/variables.tf index 065453c8..5a5f374e 100644 --- a/modules/infra/submodules/network/variables.tf +++ b/modules/infra/submodules/network/variables.tf @@ -52,6 +52,7 @@ variable "network" { } use_pod_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. create_ecr_endpoint = Create the VPC Endpoint For ECR. + create_s3_interface = Create the VPC Interface Endpoint For S3. EOF type = object({ @@ -75,6 +76,7 @@ variable "network" { })) use_pod_cidr = optional(bool) create_ecr_endpoint = optional(bool, false) + create_s3_interface = optional(bool, false) }) validation { diff --git a/modules/infra/submodules/network/versions.tf b/modules/infra/submodules/network/versions.tf index 50fd4e1c..251ff57b 100644 --- a/modules/infra/submodules/network/versions.tf +++ b/modules/infra/submodules/network/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 5.0" + version = "~> 5.7" } } } diff --git a/modules/infra/submodules/network/vpc.tf b/modules/infra/submodules/network/vpc.tf index e548d194..4e6e4fdd 100644 --- a/modules/infra/submodules/network/vpc.tf +++ b/modules/infra/submodules/network/vpc.tf @@ -49,6 +49,39 @@ resource "aws_vpc_endpoint" "s3" { } } +resource "aws_security_group" "s3_endpoint" { + count = local.create_s3_interface ? 1 : 0 + name = "${var.deploy_id}-s3-endpoint" + description = "S3 Endpoint security group" + vpc_id = aws_vpc.this[0].id + + lifecycle { + create_before_destroy = true + } + tags = { + "Name" = "${var.deploy_id}-s3-endpoint" + } +} + +resource "aws_vpc_endpoint" "s3_interface" { + count = local.create_s3_interface ? 1 : 0 + vpc_id = aws_vpc.this[0].id + service_name = "com.amazonaws.${var.region}.s3" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + subnet_ids = [for s in aws_subnet.pod : s.id] + + security_group_ids = [ + aws_security_group.s3_endpoint[0].id, + ] + + tags = { + "Name" = "${var.deploy_id}-s3" + } + + depends_on = [aws_vpc_endpoint.s3] +} + data "aws_prefix_list" "s3" { count = local.create_vpc ? 1 : 0 prefix_list_id = aws_vpc_endpoint.s3[0].prefix_list_id diff --git a/modules/infra/variables.tf b/modules/infra/variables.tf index ed633ecf..1090b681 100644 --- a/modules/infra/variables.tf +++ b/modules/infra/variables.tf @@ -318,6 +318,7 @@ variable "network" { } use_pod_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. create_ecr_endpoint = Create the VPC Endpoint For ECR. + create_s3_interface = Create the VPC Interface Endpoint For S3. EOF type = object({ @@ -341,6 +342,7 @@ variable "network" { }), {}) use_pod_cidr = optional(bool, true) create_ecr_endpoint = optional(bool, false) + create_s3_interface = optional(bool, false) }) default = {} diff --git a/tests/plan/terraform/README.md b/tests/plan/terraform/README.md index e2b2f400..a62a05aa 100644 --- a/tests/plan/terraform/README.md +++ b/tests/plan/terraform/README.md @@ -48,7 +48,7 @@ | [external\_deployments\_operator](#input\_external\_deployments\_operator) | Config to create IRSA role for the external deployments operator. |
ami = string
bootstrap_extra_args = string
instance_types = list(string)
spot = bool
min_per_az = number
max_per_az = number
desired_per_az = number
availability_zone_ids = list(string)
labels = map(string)
taints = list(object({
key = string
value = string
effect = string
}))
tags = map(string)
instance_tags = map(string)
gpu = bool
volume = object({
size = string
type = string
iops = optional(number)
throughput = optional(number, 500)
})
}))
object({| `{}` | no | | [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no | | [kms](#input\_kms) | enabled = Toggle,if set use either the specified KMS key\_id or a Domino-generated one.
enabled = optional(bool, false)
namespace = optional(string, "domino-compute")
operator_service_account_name = optional(string, "pham-juno-operator")
operator_role_suffix = optional(string, "external-deployments-operator")
repository_suffix = optional(string, "external-deployments")
bucket_suffix = optional(string, "external-deployments")
enable_assume_any_external_role = optional(bool, true)
enable_in_account_deployments = optional(bool, true)
})
object({| `{}` | no | -| [network](#input\_network) | vpc = {
enabled = optional(bool, true)
key_id = optional(string, null)
additional_policies = optional(list(string), [])
})
object({| `{}` | no | +| [network](#input\_network) | vpc = {
vpc = optional(object({
id = optional(string, null)
subnets = optional(object({
private = optional(list(string), [])
public = optional(list(string), [])
pod = optional(list(string), [])
}), {})
}), {})
network_bits = optional(object({
public = optional(number, 27)
private = optional(number, 19)
pod = optional(number, 19)
}
), {})
cidrs = optional(object({
vpc = optional(string, "10.0.0.0/16")
pod = optional(string, "100.64.0.0/16")
}), {})
use_pod_cidr = optional(bool, true)
create_ecr_endpoint = optional(bool, false)
})
object({| `{}` | no | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | | [route53\_hosted\_zone\_name](#input\_route53\_hosted\_zone\_name) | Optional hosted zone for External DNS zone. | `string` | `null` | no | | [single\_node](#input\_single\_node) | Additional EKS managed node groups definition. |
vpc = optional(object({
id = optional(string, null)
subnets = optional(object({
private = optional(list(string), [])
public = optional(list(string), [])
pod = optional(list(string), [])
}), {})
}), {})
network_bits = optional(object({
public = optional(number, 27)
private = optional(number, 19)
pod = optional(number, 19)
}
), {})
cidrs = optional(object({
vpc = optional(string, "10.0.0.0/16")
pod = optional(string, "100.64.0.0/16")
}), {})
use_pod_cidr = optional(bool, true)
create_ecr_endpoint = optional(bool, false)
create_s3_interface = optional(bool, false)
})
object({| `null` | no | diff --git a/tests/plan/terraform/variables.tf b/tests/plan/terraform/variables.tf index f534711b..0dfaaa79 100644 --- a/tests/plan/terraform/variables.tf +++ b/tests/plan/terraform/variables.tf @@ -279,6 +279,7 @@ variable "network" { } use_pod_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. create_ecr_endpoint = Create the VPC Endpoint For ECR. + create_s3_interface = Create the VPC Interface Endpoint For S3. EOF type = object({ @@ -302,6 +303,7 @@ variable "network" { }), {}) use_pod_cidr = optional(bool, true) create_ecr_endpoint = optional(bool, false) + create_s3_interface = optional(bool, false) }) default = {}
name = optional(string, "single-node")
bootstrap_extra_args = optional(string, "")
ami = optional(object({
name_prefix = optional(string, null)
owner = optional(string, null)
}))
instance_type = optional(string, "m6i.2xlarge")
authorized_ssh_ip_ranges = optional(list(string), ["0.0.0.0/0"])
labels = optional(map(string))
taints = optional(list(object({
key = string
value = optional(string)
effect = string
})), [])
volume = optional(object({
size = optional(number, 1000)
type = optional(string, "gp3")
}), {})
})