metal: simple IPAM for IPv6

cmd/kops-controller/controllers/gceipam.go

@@ -56,7 +56,7 @@ func NewGCEIPAMReconciler(mgr manager.Manager) (*GCEIPAMReconciler, error) {
 	return r, nil
 }
 
-// GCEIPAMReconciler observes Node objects, assigning their`PodCIDRs` from the instance's `ExternalIpv6`.
+// GCEIPAMReconciler observes Node objects, assigning their `PodCIDRs` from the instance's `ExternalIpv6`.
 type GCEIPAMReconciler struct {
 	// client is the controller-runtime client
 	client client.Client

cmd/kops-controller/controllers/metalipam.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/klog/v2"
+	kopsapi "k8s.io/kops/pkg/apis/kops/v1alpha2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// NewMetalIPAMReconciler is the constructor for a MetalIPAMReconciler
+func NewMetalIPAMReconciler(ctx context.Context, mgr manager.Manager) (*MetalIPAMReconciler, error) {
+	klog.Info("starting metal ipam controller")
+	r := &MetalIPAMReconciler{
+		client: mgr.GetClient(),
+		log:    ctrl.Log.WithName("controllers").WithName("metal_ipam"),
+	}
+
+	coreClient, err := corev1client.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return nil, fmt.Errorf("building corev1 client: %w", err)
+	}
+	r.coreV1Client = coreClient
+
+	return r, nil
+}
+
+// MetalIPAMReconciler observes Node objects, assigning their `PodCIDRs` from the instance's `ExternalIpv6`.
+type MetalIPAMReconciler struct {
+	// client is the controller-runtime client
+	client client.Client
+
+	// log is a logr
+	log logr.Logger
+
+	// coreV1Client is a client-go client for patching nodes
+	coreV1Client *corev1client.CoreV1Client
+}
+
+// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+// Reconcile is the main reconciler function that observes node changes.
+func (r *MetalIPAMReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	node := &corev1.Node{}
+	if err := r.client.Get(ctx, req.NamespacedName, node); err != nil {
+		klog.Warningf("unable to fetch node %s: %v", node.Name, err)
+		if apierrors.IsNotFound(err) {
+			// we'll ignore not-found errors, since they can't be fixed by an immediate
+			// requeue (we'll need to wait for a new notification), and we can get them
+			// on deleted requests.
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	host := &kopsapi.Host{}
+	id := types.NamespacedName{
+		Namespace: "kops-system",
+		Name:      node.Name,
+	}
+	if err := r.client.Get(ctx, id, host); err != nil {
+		klog.Warningf("unable to fetch host %s: %v", id, err)
+		return ctrl.Result{}, err
+	}
+
+	if len(node.Spec.PodCIDRs) == 0 {
+		if err := patchNodePodCIDRs(r.coreV1Client, ctx, node, host.Spec.PodCIDRs); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *MetalIPAMReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("metal_ipam").
+		For(&corev1.Node{}).
+		Complete(r)
+}

cmd/kops-controller/main.go

@@ -392,6 +392,12 @@ func setupCloudIPAM(ctx context.Context, mgr manager.Manager, opt *config.Option
 			return fmt.Errorf("creating gce IPAM controller: %w", err)
 		}
 		controller = ipamController
+	case "metal":
+		ipamController, err := controllers.NewMetalIPAMReconciler(ctx, mgr)
+		if err != nil {
+			return fmt.Errorf("creating metal IPAM controller: %w", err)
+		}
+		controller = ipamController
 	default:
 		return fmt.Errorf("kOps IPAM controller is not supported on cloud %q", opt.Cloud)
 	}

cmd/kops/toolbox_enroll.go

@@ -45,6 +45,7 @@ func NewCmdToolboxEnroll(f commandutils.Factory, out io.Writer) *cobra.Command {
 
 	cmd.Flags().StringVar(&options.ClusterName, "cluster", options.ClusterName, "Name of cluster to join")
 	cmd.Flags().StringVar(&options.InstanceGroup, "instance-group", options.InstanceGroup, "Name of instance-group to join")
+	cmd.Flags().StringSliceVar(&options.PodCIDRs, "pod-cidr", options.PodCIDRs, "IP Address range to use for pods that run on this node")
 
 	cmd.Flags().StringVar(&options.Host, "host", options.Host, "IP/hostname for machine to add")
 	cmd.Flags().StringVar(&options.SSHUser, "ssh-user", options.SSHUser, "user for ssh")

k8s/crds/kops.k8s.io_hosts.yaml

@@ -42,6 +42,12 @@ spec:
             properties:
               instanceGroup:
                 type: string
+              podCIDRs:
+                description: PodCIDRs configures the IP ranges to be used for pods
+                  on this node/host.
+                items:
+                  type: string
+                type: array
               publicKey:
                 type: string
             type: object

nodeup/pkg/model/kube_apiserver.go

@@ -19,10 +19,12 @@ package model
 import (
 	"context"
 	"fmt"
+	"net"
 	"path/filepath"
 	"sort"
 	"strings"
 
+	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/k8scodecs"
@@ -77,6 +79,55 @@ func (b *KubeAPIServerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		}
 	}
 
+	if b.CloudProvider() == kops.CloudProviderMetal {
+		// Workaround for https://github.com/kubernetes/kubernetes/issues/111671
+		if b.IsIPv6Only() {
+			interfaces, err := net.Interfaces()
+			if err != nil {
+				return fmt.Errorf("getting local network interfaces: %w", err)
+			}
+			var ipv6s []net.IP
+			for _, intf := range interfaces {
+				addresses, err := intf.Addrs()
+				if err != nil {
+					return fmt.Errorf("getting addresses for network interface %q: %w", intf.Name, err)
+				}
+				for _, addr := range addresses {
+					ip, _, err := net.ParseCIDR(addr.String())
+					if ip == nil {
+						return fmt.Errorf("parsing ip address %q (bound to network %q): %w", addr.String(), intf.Name, err)
+					}
+					if ip.To4() != nil {
+						// We're only looking for ipv6
+						continue
+					}
+					if ip.IsLinkLocalUnicast() {
+						klog.V(4).Infof("ignoring link-local unicast addr %v", addr)
+						continue
+					}
+					if ip.IsLinkLocalMulticast() {
+						klog.V(4).Infof("ignoring link-local multicast addr %v", addr)
+						continue
+					}
+					if ip.IsLoopback() {
+						klog.V(4).Infof("ignoring loopback addr %v", addr)
+						continue
+					}
+					ipv6s = append(ipv6s, ip)
+				}
+			}
+			if len(ipv6s) > 1 {
+				klog.Warningf("found multiple ipv6s, choosing first: %v", ipv6s)
+			}
+			if len(ipv6s) == 0 {
+				klog.Warningf("did not find ipv6 address for kube-apiserver --advertise-address")
+			}
+			if len(ipv6s) > 0 {
+				kubeAPIServer.AdvertiseAddress = ipv6s[0].String()
+			}
+		}
+	}
+
 	b.configureOIDC(&kubeAPIServer)
 	if err := b.writeAuthenticationConfig(c, &kubeAPIServer); err != nil {
 		return err
@@ -697,10 +748,9 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
 	image := b.RemapImage(kubeAPIServer.Image)
 
 	container := &v1.Container{
-		Name:           "kube-apiserver",
-		Image:          image,
-		Env:            append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...),
-		LivenessProbe:  livenessProbe,
+		Name:  "kube-apiserver",
+		Image: image,
+		Env:   append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...), LivenessProbe: livenessProbe,
 		ReadinessProbe: readinessProbe,
 		StartupProbe:   startupProbe,
 		Ports: []v1.ContainerPort{

nodeup/pkg/model/prefix.go

@@ -41,6 +41,8 @@ func (b *PrefixBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		})
 	case kops.CloudProviderGCE:
 		// Prefix is assigned by GCE
+	case kops.CloudProviderMetal:
+		// IPv6 must be configured externally (not by nodeup)
 	default:
 		return fmt.Errorf("kOps IPAM controller not supported on cloud %q", b.CloudProvider())
 	}

pkg/apis/kops/v1alpha2/host.go

@@ -36,6 +36,9 @@ type Host struct {
 type HostSpec struct {
 	PublicKey     string `json:"publicKey,omitempty"`
 	InstanceGroup string `json:"instanceGroup,omitempty"`
+
+	// PodCIDRs configures the IP ranges to be used for pods on this node/host.
+	PodCIDRs []string `json:"podCIDRs,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/commands/toolbox_enroll.go

@@ -67,6 +67,9 @@ type ToolboxEnrollOptions struct {
 
 	SSHUser string
 	SSHPort int
+
+	// PodCIDRs is the list of IP Address ranges to use for pods that run on this node
+	PodCIDRs []string
 }
 
 func (o *ToolboxEnrollOptions) InitDefaults() {
@@ -209,6 +212,7 @@ func createHostResourceInAPIServer(ctx context.Context, options *ToolboxEnrollOp
 	host.Name = nodeName
 	host.Spec.InstanceGroup = options.InstanceGroup
 	host.Spec.PublicKey = string(publicKey)
+	host.Spec.PodCIDRs = options.PodCIDRs
 
 	if err := client.Create(ctx, host); err != nil {
 		return fmt.Errorf("failed to create host %s/%s: %w", host.Namespace, host.Name, err)

pkg/kubeconfig/create_kubecfg.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"crypto/x509/pkix"
 	"fmt"
+	"net"
 	"os/user"
 	"sort"
 	"time"
@@ -41,7 +42,7 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
 		server = "https://" + cluster.APIInternalName()
 	} else {
 		if cluster.Spec.API.PublicName != "" {
-			server = "https://" + cluster.Spec.API.PublicName
+			server = "https://" + wrapIPv6Address(cluster.Spec.API.PublicName)
 		} else {
 			server = "https://api." + clusterName
 		}
@@ -82,7 +83,7 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
 				if len(targets) != 1 {
 					klog.Warningf("Found multiple API endpoints (%v), choosing arbitrarily", targets)
 				}
-				server = "https://" + targets[0]
+				server = "https://" + wrapIPv6Address(targets[0])
 			}
 		}
 	}
@@ -171,3 +172,14 @@ func BuildKubecfg(ctx context.Context, cluster *kops.Cluster, keyStore fi.Keysto
 
 	return b, nil
 }
+
+// wrapIPv6Address will wrap IPv6 addresses in square brackets,
+// for use in URLs; other endpoints are unchanged.
+func wrapIPv6Address(endpoint string) string {
+	ip := net.ParseIP(endpoint)
+	// IPv6 addresses are wrapped in square brackets in URLs
+	if ip != nil && ip.To4() == nil {
+		return "[" + endpoint + "]"
+	}
+	return endpoint
+}

tests/e2e/scenarios/bare-metal/scenario-ipv6

@@ -70,18 +70,24 @@ ${REPO_ROOT}/tests/e2e/scenarios/bare-metal/start-vms
 
 . hack/dev-build-metal.sh
 
+IPV6_PREFIX=fd00:10:123:45:
+IPV4_PREFIX=10.123.45.
+
 echo "Waiting 10 seconds for VMs to start"
 sleep 10
 
+VM0_IP=${IPV4_PREFIX}10
+VM1_IP=${IPV4_PREFIX}11
+VM2_IP=${IPV4_PREFIX}12
+
 # Remove from known-hosts in case of reuse
-ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.10 || true
-ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.11 || true
-ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.12 || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM0_IP} || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM1_IP} || true
+ssh-keygen -f ~/.ssh/known_hosts -R ${VM2_IP} || true
 
-# Check SSH is working and accept the keys
-ssh -o StrictHostKeyChecking=accept-new root@${VM0_IP} uptime
-ssh -o StrictHostKeyChecking=accept-new root@${VM1_IP} uptime
-ssh -o StrictHostKeyChecking=accept-new root@${VM2_IP} uptime
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${VM0_IP} uptime
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${VM1_IP} uptime
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${VM2_IP} uptime
 
 cd ${REPO_ROOT}
 
@@ -206,8 +212,8 @@ ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group control-plane-
 cat <<EOF | ssh root@${VM0_IP} tee -a /etc/hosts
 
 # Hosts added for etcd discovery
-10.123.45.10 node0.main.${CLUSTER_NAME}
-10.123.45.10 node0.events.${CLUSTER_NAME}
+${VM0_IP} node0.main.${CLUSTER_NAME}
+${VM0_IP} node0.events.${CLUSTER_NAME}
 EOF
 
 ssh root@${VM0_IP} cat /etc/hosts
@@ -278,17 +284,18 @@ EOF
 
 function enroll_node() {
   local node_ip=$1
+  local pod_cidr=$2
 
 # Manual "discovery" for control-plane endpoints
 # TODO: Replace with well-known IP
 cat <<EOF | ssh root@${node_ip} tee -a /etc/hosts
 
 # Hosts added for leader discovery
-10.123.45.10 kops-controller.internal.${CLUSTER_NAME}
-10.123.45.10 api.internal.${CLUSTER_NAME}
+${VM0_IP} kops-controller.internal.${CLUSTER_NAME}
+${VM0_IP} api.internal.${CLUSTER_NAME}
 EOF
 
-timeout 10m ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group nodes-main --host ${node_ip} --v=2
+timeout 10m ${KOPS} toolbox enroll --cluster ${CLUSTER_NAME} --instance-group nodes-main --host ${node_ip} --pod-cidr ${pod_cidr} --v=2
 }
 
 enroll_node ${VM1_IP} ${VM1_POD_CIDR}