Add init containers for exec node

ytsaurus · Jun 19, 2024 · f8fb36f · f8fb36f
1 parent b3cca0a
commit f8fb36f
Show file tree

Hide file tree

Showing 10 changed files with 92 additions and 34 deletions.
diff --git a/api/v1/ytsaurus_types.go b/api/v1/ytsaurus_types.go
@@ -276,6 +276,7 @@ type MastersSpec struct {
 	MaxSnapshotCountToKeep  *int `json:"maxSnapshotCountToKeep,omitempty"`
 	MaxChangelogCountToKeep *int `json:"maxChangelogCountToKeep,omitempty"`
 
+	// List of sidecar containers as yaml of core/v1 Container.
 	Sidecars []string `json:"sidecars,omitempty"`
 }
 
@@ -415,7 +416,9 @@ type ExecNodesSpec struct {
 	//+kubebuilder:default:=default
 	//+kubebuilder:validation:MinLength:=1
 	Name string `json:"name,omitempty"`
-	// List of sidecar containers as yaml of corev1.Container.
+	// List of init containers as yaml of core/v1 Container.
+	InitContainers []string `json:"initContainers,omitempty"`
+	// List of sidecar containers as yaml of core/v1 Container.
 	Sidecars []string `json:"sidecars,omitempty"`
 	//+kubebuilder:default:=true
 	//+optional

diff --git a/api/v1/ytsaurus_webhook.go b/api/v1/ytsaurus_webhook.go
@@ -241,7 +241,7 @@ func validateSidecars(sidecars []string, path *field.Path) field.ErrorList {
 	names := make(map[string]bool)
 	for i, sidecarSpec := range sidecars {
 		sidecar := corev1.Container{}
-		if err := yaml.Unmarshal([]byte(sidecarSpec), &sidecar); err != nil {
+		if err := yaml.UnmarshalStrict([]byte(sidecarSpec), &sidecar); err != nil {
 			allErrors = append(allErrors, field.Invalid(path.Index(i), sidecarSpec, err.Error()))
 		}
 		if _, exists := names[sidecar.Name]; exists {
@@ -275,6 +275,9 @@ func (r *Ytsaurus) validateExecNodes(*Ytsaurus) field.ErrorList {
 			allErrors = append(allErrors, field.NotFound(path.Child("locations"), LocationTypeSlots))
 		}
 
+		if en.InitContainers != nil {
+			allErrors = append(allErrors, validateSidecars(en.InitContainers, path.Child("initContainers"))...)
+		}
 		if en.Sidecars != nil {
 			allErrors = append(allErrors, validateSidecars(en.Sidecars, path.Child("sidecars"))...)
 		}

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml b/config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml
@@ -690,6 +690,11 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              initContainers:
+                description: List of init containers as yaml of core/v1 Container.
+                items:
+                  type: string
+                type: array
               instanceCount:
                 format: int32
                 type: integer
@@ -1078,7 +1083,7 @@ spec:
               runtimeClassName:
                 type: string
               sidecars:
-                description: List of sidecar containers as yaml of corev1.Container.
+                description: List of sidecar containers as yaml of core/v1 Container.
                 items:
                   type: string
                 type: array

diff --git a/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml b/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml
@@ -7803,6 +7803,11 @@ spec:
                     image:
                       description: Overrides coreImage for component.
                       type: string
+                    initContainers:
+                      description: List of init containers as yaml of core/v1 Container.
+                      items:
+                        type: string
+                      type: array
                     instanceCount:
                       format: int32
                       type: integer
@@ -8183,7 +8188,7 @@ spec:
                     runtimeClassName:
                       type: string
                     sidecars:
-                      description: List of sidecar containers as yaml of corev1.Container.
+                      description: List of sidecar containers as yaml of core/v1 Container.
                       items:
                         type: string
                       type: array
@@ -15339,6 +15344,7 @@ spec:
                   runtimeClassName:
                     type: string
                   sidecars:
+                    description: List of sidecar containers as yaml of core/v1 Container.
                     items:
                       type: string
                     type: array
@@ -27074,6 +27080,7 @@ spec:
                     runtimeClassName:
                       type: string
                     sidecars:
+                      description: List of sidecar containers as yaml of core/v1 Container.
                       items:
                         type: string
                       type: array

diff --git a/docs/api.md b/docs/api.md
@@ -458,7 +458,8 @@ _Appears in:_
 | `tags` _string array_ | List of the node tags. |  |  |
 | `rack` _string_ | Name of the node rack. |  |  |
 | `name` _string_ |  | default | MinLength: 1 <br /> |
-| `sidecars` _string array_ | List of sidecar containers as yaml of corev1.Container. |  |  |
+| `initContainers` _string array_ | List of init containers as yaml of core/v1 Container. |  |  |
+| `sidecars` _string array_ | List of sidecar containers as yaml of core/v1 Container. |  |  |
 | `privileged` _boolean_ |  | true |  |
 | `jobProxyLoggers` _[TextLoggerSpec](#textloggerspec) array_ |  |  |  |
 | `jobResources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#resourcerequirements-v1-core)_ | Resources dedicated for running jobs. |  |  |
@@ -865,7 +866,7 @@ _Appears in:_
 | `hostAddressLabel` _string_ |  |  |  |
 | `maxSnapshotCountToKeep` _integer_ |  |  |  |
 | `maxChangelogCountToKeep` _integer_ |  |  |  |
-| `sidecars` _string array_ |  |  |  |
+| `sidecars` _string array_ | List of sidecar containers as yaml of core/v1 Container. |  |  |
 
 
 #### OauthServiceSpec
@@ -1115,7 +1116,8 @@ _Appears in:_
 | `tags` _string array_ | List of the node tags. |  |  |
 | `rack` _string_ | Name of the node rack. |  |  |
 | `name` _string_ |  | default | MinLength: 1 <br /> |
-| `sidecars` _string array_ | List of sidecar containers as yaml of corev1.Container. |  |  |
+| `initContainers` _string array_ | List of init containers as yaml of core/v1 Container. |  |  |
+| `sidecars` _string array_ | List of sidecar containers as yaml of core/v1 Container. |  |  |
 | `privileged` _boolean_ |  | true |  |
 | `jobProxyLoggers` _[TextLoggerSpec](#textloggerspec) array_ |  |  |  |
 | `jobResources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#resourcerequirements-v1-core)_ | Resources dedicated for running jobs. |  |  |

diff --git a/pkg/components/exec_node_base.go b/pkg/components/exec_node_base.go
@@ -44,6 +44,39 @@ func (n *baseExecNode) doBuildBase() error {
 	statefulSet := n.server.buildStatefulSet()
 	podSpec := &statefulSet.Spec.Template.Spec
 
+	if len(podSpec.Containers) != 1 {
+		log.Panicf("Number of exec node containers is expected to be 1, actual %v", len(podSpec.Containers))
+	}
+
+	if err := AddInitContainersToPodSpec(n.spec.InitContainers, podSpec); err != nil {
+		return err
+	}
+
+	if err := AddSidecarsToPodSpec(n.spec.Sidecars, podSpec); err != nil {
+		return err
+	}
+
+	setContainerPrivileged := func(ct *corev1.Container) {
+		if ct.SecurityContext == nil {
+			ct.SecurityContext = &corev1.SecurityContext{}
+		}
+		if ct.SecurityContext.Privileged == nil {
+			ct.SecurityContext.Privileged = ptr.Bool(n.spec.Privileged)
+		}
+	}
+
+	for i := range podSpec.InitContainers {
+		setContainerPrivileged(&podSpec.InitContainers[i])
+	}
+
+	for i := range podSpec.Containers {
+		setContainerPrivileged(&podSpec.Containers[i])
+
+		if envSpec := n.spec.JobEnvironment; envSpec != nil && envSpec.CRI != nil {
+			n.addEnvironmentForCRITools(&podSpec.Containers[i])
+		}
+	}
+
 	// Pour job resources into node container if jobs are not isolated.
 	if n.spec.JobResources != nil && !n.IsJobEnvironmentIsolated() {
 		addResourceList := func(list, newList corev1.ResourceList) {
@@ -61,26 +94,6 @@ func (n *baseExecNode) doBuildBase() error {
 		addResourceList(podSpec.Containers[0].Resources.Limits, n.spec.JobResources.Limits)
 	}
 
-	setContainerPrivileged := func(ct *corev1.Container) {
-		if ct.SecurityContext == nil {
-			ct.SecurityContext = &corev1.SecurityContext{}
-		}
-		ct.SecurityContext.Privileged = ptr.Bool(n.spec.Privileged)
-	}
-
-	if len(podSpec.Containers) != 1 {
-		log.Panicf("Number of exec node containers is expected to be 1, actual %v", len(podSpec.Containers))
-	}
-	setContainerPrivileged(&podSpec.Containers[0])
-
-	if envSpec := n.spec.JobEnvironment; envSpec != nil && envSpec.CRI != nil {
-		n.addEnvironmentForCRITools(&podSpec.Containers[0])
-	}
-
-	for i := range podSpec.InitContainers {
-		setContainerPrivileged(&podSpec.InitContainers[i])
-	}
-
 	if n.IsJobEnvironmentIsolated() {
 		// Add sidecar container for running jobs.
 		if envSpec := n.spec.JobEnvironment; envSpec != nil && envSpec.CRI != nil {
@@ -97,10 +110,6 @@ func (n *baseExecNode) doBuildBase() error {
 			})
 	}
 
-	if err := AddSidecarsToPodSpec(n.spec.Sidecars, podSpec); err != nil {
-		return err
-	}
-
 	if n.sidecarConfig != nil {
 		podSpec.Volumes = append(podSpec.Volumes, createConfigVolume(consts.ContainerdConfigVolumeName,
 			n.sidecarConfig.labeller.GetSidecarConfigMapName(consts.JobsContainerName), nil))

diff --git a/pkg/components/helpers.go b/pkg/components/helpers.go
@@ -209,10 +209,22 @@ func AddAffinity(statefulSet *appsv1.StatefulSet,
 func AddSidecarsToPodSpec(sidecar []string, podSpec *corev1.PodSpec) error {
 	for _, sidecarSpec := range sidecar {
 		sidecar := corev1.Container{}
-		if err := yaml.Unmarshal([]byte(sidecarSpec), &sidecar); err != nil {
+		if err := yaml.UnmarshalStrict([]byte(sidecarSpec), &sidecar); err != nil {
 			return err
 		}
 		podSpec.Containers = append(podSpec.Containers, sidecar)
 	}
 	return nil
 }
+
+func AddInitContainersToPodSpec(initContainers []string, podSpec *corev1.PodSpec) error {
+	containers := make([]corev1.Container, len(initContainers), len(initContainers)+len(podSpec.InitContainers))
+	for i, spec := range initContainers {
+		if err := yaml.UnmarshalStrict([]byte(spec), &containers[i]); err != nil {
+			return err
+		}
+	}
+	// Insert new containers into head
+	podSpec.InitContainers = append(containers, podSpec.InitContainers...)
+	return nil
+}
diff --git a/ytop-chart/templates/remoteexecnodes-crd.yaml b/ytop-chart/templates/remoteexecnodes-crd.yaml
@@ -674,6 +674,11 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              initContainers:
+                description: List of init containers as yaml of core/v1 Container.
+                items:
+                  type: string
+                type: array
               instanceCount:
                 format: int32
                 type: integer
@@ -1062,7 +1067,7 @@ spec:
               runtimeClassName:
                 type: string
               sidecars:
-                description: List of sidecar containers as yaml of corev1.Container.
+                description: List of sidecar containers as yaml of core/v1 Container.
                 items:
                   type: string
                 type: array

diff --git a/ytop-chart/templates/ytsaurus-crd.yaml b/ytop-chart/templates/ytsaurus-crd.yaml
@@ -7759,6 +7759,11 @@ spec:
                     image:
                       description: Overrides coreImage for component.
                       type: string
+                    initContainers:
+                      description: List of init containers as yaml of core/v1 Container.
+                      items:
+                        type: string
+                      type: array
                     instanceCount:
                       format: int32
                       type: integer
@@ -8139,7 +8144,7 @@ spec:
                     runtimeClassName:
                       type: string
                     sidecars:
-                      description: List of sidecar containers as yaml of corev1.Container.
+                      description: List of sidecar containers as yaml of core/v1 Container.
                       items:
                         type: string
                       type: array
@@ -15252,6 +15257,7 @@ spec:
                   runtimeClassName:
                     type: string
                   sidecars:
+                    description: List of sidecar containers as yaml of core/v1 Container.
                     items:
                       type: string
                     type: array
@@ -26896,6 +26902,7 @@ spec:
                     runtimeClassName:
                       type: string
                     sidecars:
+                      description: List of sidecar containers as yaml of core/v1 Container.
                       items:
                         type: string
                       type: array