From 2aacb010a430530d2d9303cbe03e084a9421e1cb Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Wed, 22 Jan 2025 19:37:15 +0000 Subject: [PATCH] feat(autopilot): add insecure_kubelet_readonly_port_enabled --- autogen/main/cluster.tf.tmpl | 11 +++++-- autogen/main/variables.tf.tmpl | 4 +-- examples/simple_autopilot_private/main.tf | 29 ++++++++++--------- .../beta-autopilot-private-cluster/README.md | 1 + .../beta-autopilot-private-cluster/cluster.tf | 11 +++++-- .../variables.tf | 6 ++++ .../beta-autopilot-public-cluster/README.md | 1 + .../beta-autopilot-public-cluster/cluster.tf | 11 +++++-- .../variables.tf | 6 ++++ terraform-google-kubernetes-engine | 1 - .../TestSimpleRegionalWithGatewayAPI.json | 2 +- test/integration/testutils/utils.go | 15 +++------- 12 files changed, 63 insertions(+), 35 deletions(-) delete mode 160000 terraform-google-kubernetes-engine diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index c20bb3335a..086ac4531d 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -278,10 +278,17 @@ resource "google_container_cluster" "primary" { } {% if autopilot_cluster %} dynamic "node_pool_auto_config" { - for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : [] + for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] content { network_tags { - tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags + tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null + } + + dynamic "node_kubelet_config" { + for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + content { + insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled)) + } } } } diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 9a2462408a..e52f17489b 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -108,13 +108,13 @@ variable "service_external_ips" { default = false } -{% if autopilot_cluster != true %} variable "insecure_kubelet_readonly_port_enabled" { type = bool - description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`." + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters.{% if autopilot_cluster != true %} Note: this can be set at the node pool level separately within `node_pools`.{% endif %}" default = null } +{% if autopilot_cluster != true %} variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature." diff --git a/examples/simple_autopilot_private/main.tf b/examples/simple_autopilot_private/main.tf index f656b6cf68..c5b38b1613 100644 --- a/examples/simple_autopilot_private/main.tf +++ b/examples/simple_autopilot_private/main.tf @@ -37,18 +37,19 @@ module "gke" { source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster" version = "~> 35.0" - project_id = var.project_id - name = "${local.cluster_type}-cluster" - regional = true - region = var.region - network = module.gcp-network.network_name - subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] - ip_range_pods = local.pods_range_name - ip_range_services = local.svc_range_name - release_channel = "REGULAR" - enable_vertical_pod_autoscaling = true - enable_private_endpoint = true - enable_private_nodes = true - network_tags = [local.cluster_type] - deletion_protection = false + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = var.region + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + network_tags = [local.cluster_type] + deletion_protection = false + insecure_kubelet_readonly_port_enabled = false } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 962ab2dd8a..8654594835 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -113,6 +113,7 @@ Then perform the following commands on the root folder: | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index 9b2c8059c2..881e278f75 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -134,10 +134,17 @@ resource "google_container_cluster" "primary" { } } dynamic "node_pool_auto_config" { - for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : [] + for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] content { network_tags { - tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags + tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null + } + + dynamic "node_kubelet_config" { + for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + content { + insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled)) + } } } } diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index bc3957f3a3..4d55e0fa55 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters." + default = null +} + variable "maintenance_start_time" { type = string description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 7d48084d7d..bdbca14714 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -105,6 +105,7 @@ Then perform the following commands on the root folder: | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | | http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | | identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. | `bool` | `null` | no | | ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | | ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 92d0eeb2f9..95e6c2cefc 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -134,10 +134,17 @@ resource "google_container_cluster" "primary" { } } dynamic "node_pool_auto_config" { - for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : [] + for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules || var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] content { network_tags { - tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags + tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : length(var.network_tags) > 0 ? var.network_tags : null + } + + dynamic "node_kubelet_config" { + for_each = var.insecure_kubelet_readonly_port_enabled != null ? [1] : [] + content { + insecure_kubelet_readonly_port_enabled = upper(tostring(var.insecure_kubelet_readonly_port_enabled)) + } } } } diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index e50ac78ee1..0ac9f05b5e 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -108,6 +108,12 @@ variable "service_external_ips" { default = false } +variable "insecure_kubelet_readonly_port_enabled" { + type = bool + description = "Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters." + default = null +} + variable "maintenance_start_time" { type = string description = "Time window specified for daily or recurring maintenance operations in RFC3339 format" diff --git a/terraform-google-kubernetes-engine b/terraform-google-kubernetes-engine deleted file mode 160000 index 70a28a04e8..0000000000 --- a/terraform-google-kubernetes-engine +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 70a28a04e870f2cbc2c6bc3f20ff2bd6ae0b15a9 diff --git a/test/integration/simple_regional_with_gateway_api/testdata/TestSimpleRegionalWithGatewayAPI.json b/test/integration/simple_regional_with_gateway_api/testdata/TestSimpleRegionalWithGatewayAPI.json index 1da7d2e0b0..d9edee89ce 100644 --- a/test/integration/simple_regional_with_gateway_api/testdata/TestSimpleRegionalWithGatewayAPI.json +++ b/test/integration/simple_regional_with_gateway_api/testdata/TestSimpleRegionalWithGatewayAPI.json @@ -213,7 +213,7 @@ "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/gke-simple-regional-gatewayapi-cluster-RANDOM_STRING/nodePools/default-pool", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-central1/clusters/simple-regional-gatewayapi-cluster-RANDOM_STRING/nodePools/default-pool", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, diff --git a/test/integration/testutils/utils.go b/test/integration/testutils/utils.go index 0711bca793..2f5a88d1d2 100644 --- a/test/integration/testutils/utils.go +++ b/test/integration/testutils/utils.go @@ -1,4 +1,4 @@ -// Copyright 2022-2024 Google LLC +// Copyright 2022-2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ import ( tfjson "github.com/hashicorp/terraform-json" "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" - "golang.org/x/sync/errgroup" ) var ( @@ -100,16 +99,10 @@ func TGKEAssertGolden(assert *assert.Assertions, golden *golden.GoldenFile, clus nodeCheckPaths := utils.GetTerminalJSONPaths(golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool))) - syncGroup := new(errgroup.Group) - syncGroup.SetLimit(24) for _, nodeCheckPath := range nodeCheckPaths { - nodeCheckPath := nodeCheckPath - syncGroup.Go(func() error { - gotData := golden.ApplySanitizers(clusterJson.Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String()) - gfData := golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String() - assert.Equalf(gfData, gotData, "For node %s path %q expected %q to match fixture %q", nodePool, nodeCheckPath, gotData, gfData) - return nil - }) + gotData := golden.ApplySanitizers(clusterJson.Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String()) + gfData := golden.GetJSON().Get(fmt.Sprintf("nodePools.#(name==%s)", nodePool)).Get(nodeCheckPath).String() + assert.Equalf(gfData, gotData, "For node %q path %q expected %q to match fixture %q", nodePool, nodeCheckPath, gotData, gfData) } } }