Merge pull request #940 from traPtitech/chores/add-svc-ipfamily-conf

chores(k8s-backend): add svc ip families config

.local-dev/config/ns.yaml

@@ -126,6 +126,10 @@ components:
         type: traefik
         traefik:
           priorityOffset: 0
+      service:
+        ipFamilies:
+          - IPv4
+        ipFamilyPolicy: SingleStack
       tls:
         type: traefik
         traefik:

.local-manifest/traefik/dashboard-ingress-route.yaml

@@ -9,12 +9,12 @@ spec:
     - websecure
   routes:
     - kind: Rule
-      match: Host(`traefik-dashboard.local.trapti.tech`)
+      match: Host(`traefik.local.trapti.tech`)
       services:
-        - namespace: traefik
-          kind: Service
-          name: dashboard
-          port: dashboard
-          scheme: http
-          strategy: RoundRobin
-          weight: 1
+        - kind: TraefikService
+          name: dashboard@internal
+    - kind: Rule
+      match: Host(`traefik.local.trapti.tech`) && PathPrefix(`/api`)
+      services:
+        - kind: TraefikService
+          name: api@internal

.local-manifest/traefik/kustomization.yaml

@@ -6,5 +6,4 @@ resources:
   - traefik-role-binding.yaml
   - traefik-stateful-set.yaml
   - traefik-service.yaml
-  - dashboard-service.yaml
   - dashboard-ingress-route.yaml

.local-manifest/traefik/traefik-cluster-role.yaml

@@ -8,8 +8,8 @@ rules:
       - ""
     resources:
       - services
-      - endpoints
       - secrets
+      - nodes
     verbs:
       - get
       - list
@@ -33,7 +33,6 @@ rules:
       - update
   - apiGroups:
       - traefik.io
-      - traefik.containo.us
     resources:
       - middlewares
       - middlewaretcps
@@ -49,3 +48,10 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch

.local-manifest/traefik/traefik-stateful-set.yaml

@@ -21,7 +21,7 @@ spec:
 
       containers:
         - name: traefik
-          image: traefik:2.10
+          image: traefik:3.1
           args:
             - --api.insecure
             - --providers.kubernetescrd

cmd/config.go

@@ -187,6 +187,10 @@ func init() {
 
 	viper.SetDefault("components.controller.k8s.routing.type", "traefik")
 	viper.SetDefault("components.controller.k8s.routing.traefik.priorityOffset", 0)
+
+	viper.SetDefault("components.controller.k8s.service.ipFamilies", nil)
+	viper.SetDefault("components.controller.k8s.service.ipFamilyPolicy", "PreferDualStack")
+
 	viper.SetDefault("components.controller.k8s.tls.type", "traefik")
 	viper.SetDefault("components.controller.k8s.tls.traefik.certResolver", "nsresolver")
 	viper.SetDefault("components.controller.k8s.tls.traefik.wildcard.domains", nil)

pkg/infrastructure/backend/k8simpl/config.go

@@ -181,6 +181,15 @@ type Config struct {
 			PriorityOffset int `mapstructure:"priorityOffset" yaml:"priorityOffset"`
 		} `mapstructure:"traefik" yaml:"traefik"`
 	} `mapstructure:"routing" yaml:"routing"`
+	// Service section defines Service (L4) routing settings.
+	Service struct {
+		// IPFamilies defines ipFamilies field for the service objects.
+		// Allowed values: IPv4, IPv6
+		IPFamilies []v1.IPFamily `mapstructure:"ipFamilies" yaml:"ipFamilies"`
+		// IPFamilyPolicy defines ipFamilyPolicy field for the service objects.
+		// Allowed values: "", "SingleStack", "PreferDualStack", "RequireDualStack"
+		IPFamilyPolicy v1.IPFamilyPolicy `mapstructure:"ipFamilyPolicy" yaml:"ipFamilyPolicy"`
+	} `mapstructure:"service" yaml:"service"`
 	// TLS section defines tls setting for user app ingress.
 	TLS struct {
 		// Type defines which provider is responsible for obtaining http certificates.
@@ -264,6 +273,17 @@ func (c *Config) selectNode(appID string) map[string]string {
 	return map[string]string{hostnameNodeSelectorLabel: host}
 }
 
+func (c *Config) serviceIPFamilies() []v1.IPFamily {
+	return c.Service.IPFamilies
+}
+
+func (c *Config) serviceIPFamilyPolicy() *v1.IPFamilyPolicy {
+	if c.Service.IPFamilyPolicy == "" {
+		return nil
+	}
+	return lo.ToPtr(c.Service.IPFamilyPolicy)
+}
+
 var tolerationOperatorMapper = mapper.MustNewValueMapper(map[string]v1.TolerationOperator{
 	string(v1.TolerationOpEqual):  v1.TolerationOpEqual,
 	string(v1.TolerationOpExists): v1.TolerationOpExists,
@@ -347,6 +367,22 @@ func (c *Config) Validate() error {
 	default:
 		return errors.New(fmt.Sprintf("k8s.routing.type is invalid: %s", c.Routing.Type))
 	}
+
+	for _, family := range c.Service.IPFamilies {
+		if !lo.Contains([]v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, family) {
+			return errors.New(fmt.Sprintf("invalid IPFamily %s", family))
+		}
+	}
+	if !lo.Contains([]v1.IPFamilyPolicy{
+		// Allow empty value
+		"",
+		v1.IPFamilyPolicySingleStack,
+		v1.IPFamilyPolicyPreferDualStack,
+		v1.IPFamilyPolicyRequireDualStack,
+	}, c.Service.IPFamilyPolicy) {
+		return errors.New(fmt.Sprintf("invalid IPFamily policy: %s", c.Service.IPFamilyPolicy))
+	}
+
 	switch c.TLS.Type {
 	case tlsTypeTraefik:
 		if err := c.TLS.Traefik.Wildcard.Domains.Validate(); err != nil {

pkg/infrastructure/backend/k8simpl/synchronize_runtime.go

@@ -140,7 +140,8 @@ func (b *Backend) runtimeSpec(app *domain.RuntimeDesiredState) (*appsv1.Stateful
 			},
 			Spec: v1.ServiceSpec{
 				Type:           "ClusterIP",
-				IPFamilyPolicy: lo.ToPtr(v1.IPFamilyPolicyPreferDualStack),
+				IPFamilies:     b.config.serviceIPFamilies(),
+				IPFamilyPolicy: b.config.serviceIPFamilyPolicy(),
 				Selector:       appSelector(app.App.ID),
 				Ports: ds.Map(cont.Ports, func(port v1.ContainerPort) v1.ServicePort {
 					return v1.ServicePort{
@@ -186,8 +187,10 @@ func (b *Backend) runtimePortService(app *domain.Application, port *domain.PortP
 			Labels:    b.appLabel(app.ID),
 		},
 		Spec: v1.ServiceSpec{
-			Type:     "LoadBalancer",
-			Selector: appSelector(app.ID),
+			Type:           "LoadBalancer",
+			IPFamilies:     b.config.serviceIPFamilies(),
+			IPFamilyPolicy: b.config.serviceIPFamilyPolicy(),
+			Selector:       appSelector(app.ID),
 			Ports: []v1.ServicePort{{
 				Protocol:   protocolMapper.IntoMust(port.Protocol),
 				Port:       int32(port.InternetPort),