From cd529f16f33cd1f39603c2161af09fd34e91061b Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Sun, 9 Jun 2024 19:27:38 +0200 Subject: [PATCH] Add CRI image cache config --- api/v1/ytsaurus_types.go | 12 ++++++ api/v1/zz_generated.deepcopy.go | 20 ++++++++++ ...cluster.ytsaurus.tech_remoteexecnodes.yaml | 17 +++++++++ .../bases/cluster.ytsaurus.tech_ytsaurus.yaml | 17 +++++++++ docs/api.md | 4 ++ .../test.canondata | 3 ++ .../test.canondata | 3 ++ .../test.canondata | 3 ++ .../test.canondata | 3 ++ pkg/ytconfig/node.go | 37 +++++++++++++++++-- ytop-chart/templates/remoteexecnodes-crd.yaml | 17 +++++++++ ytop-chart/templates/ytsaurus-crd.yaml | 17 +++++++++ 12 files changed, 149 insertions(+), 4 deletions(-) diff --git a/api/v1/ytsaurus_types.go b/api/v1/ytsaurus_types.go index c3305a26..8b5c528b 100644 --- a/api/v1/ytsaurus_types.go +++ b/api/v1/ytsaurus_types.go @@ -375,6 +375,18 @@ type CRIJobEnvironmentSpec struct { // See: https://github.com/containerd/containerd/blob/main/docs/hosts.md //+optional RegistryConfigPath *string `json:"registryConfigPath,omitempty"` + // Initial estimation for space required for pulling image into cache. + //+optional + ImageSizeEstimation *int64 `json:"imageSizeEstimation,omitempty"` + // Multiplier for image size to account space used by unpacked images. + //+optional + ImageCompressionRatioEstimation *int32 `json:"imageCompressionRatioEstimation,omitempty"` + // Always pull "latest" images. + //+optional + AlwaysPullLatestImage *bool `json:"alwaysPullLatestImage,omitempty"` + // Pull images periodically. + //+optional + ImagePullPeriodSeconds *int32 `json:"imagePullPeriodSeconds,omitempty"` } type JobEnvironmentSpec struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 113eaed9..01488f19 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -149,6 +149,26 @@ func (in *CRIJobEnvironmentSpec) DeepCopyInto(out *CRIJobEnvironmentSpec) { *out = new(string) **out = **in } + if in.ImageSizeEstimation != nil { + in, out := &in.ImageSizeEstimation, &out.ImageSizeEstimation + *out = new(int64) + **out = **in + } + if in.ImageCompressionRatioEstimation != nil { + in, out := &in.ImageCompressionRatioEstimation, &out.ImageCompressionRatioEstimation + *out = new(int32) + **out = **in + } + if in.AlwaysPullLatestImage != nil { + in, out := &in.AlwaysPullLatestImage, &out.AlwaysPullLatestImage + *out = new(bool) + **out = **in + } + if in.ImagePullPeriodSeconds != nil { + in, out := &in.ImagePullPeriodSeconds, &out.ImagePullPeriodSeconds + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRIJobEnvironmentSpec. diff --git a/config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml b/config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml index aec89adc..fe1161d9 100644 --- a/config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml +++ b/config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml @@ -699,6 +699,9 @@ spec: description: CRI service configuration for running jobs in sidecar container. properties: + alwaysPullLatestImage: + description: Always pull "latest" images. + type: boolean apiRetryTimeoutSeconds: description: Timeout for retrying CRI API calls. format: int32 @@ -715,6 +718,20 @@ spec: items: type: string type: array + imageCompressionRatioEstimation: + description: Multiplier for image size to account space used + by unpacked images. + format: int32 + type: integer + imagePullPeriodSeconds: + description: Pull images periodically. + format: int32 + type: integer + imageSizeEstimation: + description: Initial estimation for space required for pulling + image into cache. + format: int64 + type: integer registryConfigPath: description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md' type: string diff --git a/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml b/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml index a74badb4..071e3446 100644 --- a/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml +++ b/config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml @@ -7812,6 +7812,9 @@ spec: description: CRI service configuration for running jobs in sidecar container. properties: + alwaysPullLatestImage: + description: Always pull "latest" images. + type: boolean apiRetryTimeoutSeconds: description: Timeout for retrying CRI API calls. format: int32 @@ -7828,6 +7831,20 @@ spec: items: type: string type: array + imageCompressionRatioEstimation: + description: Multiplier for image size to account space + used by unpacked images. + format: int32 + type: integer + imagePullPeriodSeconds: + description: Pull images periodically. + format: int32 + type: integer + imageSizeEstimation: + description: Initial estimation for space required for + pulling image into cache. + format: int64 + type: integer registryConfigPath: description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md' type: string diff --git a/docs/api.md b/docs/api.md index 528bedab..f103f203 100644 --- a/docs/api.md +++ b/docs/api.md @@ -109,6 +109,10 @@ _Appears in:_ | `criNamespace` _string_ | CRI namespace for jobs containers. | | | | `baseCgroup` _string_ | Base cgroup for jobs. | | | | `registryConfigPath` _string_ | See: https://github.com/containerd/containerd/blob/main/docs/hosts.md | | | +| `imageSizeEstimation` _integer_ | Initial estimation for space required for pulling image into cache. | | | +| `imageCompressionRatioEstimation` _integer_ | Multiplier for image size to account space used by unpacked images. | | | +| `alwaysPullLatestImage` _boolean_ | Always pull "latest" images. | | | +| `imagePullPeriodSeconds` _integer_ | Pull images periodically. | | | #### CategoriesFilter diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata index 352ae496..2afb3cee 100644 --- a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata @@ -127,6 +127,9 @@ namespace=yt; "base_cgroup"="/yt"; }; + "cri_image_cache"={ + capacity=4294967296; + }; "use_job_proxy_from_image"=%false; }; "do_not_set_user_id"=%true; diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata index 950f3e6f..9a568de6 100644 --- a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata @@ -127,6 +127,9 @@ namespace=yt; "base_cgroup"="/yt"; }; + "cri_image_cache"={ + capacity=4294967296; + }; "use_job_proxy_from_image"=%false; }; "do_not_set_user_id"=%true; diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata index 352ae496..2afb3cee 100644 --- a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata @@ -127,6 +127,9 @@ namespace=yt; "base_cgroup"="/yt"; }; + "cri_image_cache"={ + capacity=4294967296; + }; "use_job_proxy_from_image"=%false; }; "do_not_set_user_id"=%true; diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata index 950f3e6f..9a568de6 100644 --- a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata @@ -127,6 +127,9 @@ namespace=yt; "base_cgroup"="/yt"; }; + "cri_image_cache"={ + capacity=4294967296; + }; "use_job_proxy_from_image"=%false; }; "do_not_set_user_id"=%true; diff --git a/pkg/ytconfig/node.go b/pkg/ytconfig/node.go index 02b2c9db..9087f76e 100644 --- a/pkg/ytconfig/node.go +++ b/pkg/ytconfig/node.go @@ -89,11 +89,26 @@ type CriExecutor struct { CpuPeriod yson.Duration `yson:"cpu_period,omitempty"` } +type CriImageCache struct { + Capacity *int64 `yson:"capacity,omitempty"` + + ManagedPrefixes []string `yson:"managed_prefixes,omitempty"` + UnmanagedPrefixes []string `yson:"managed_prefixes,omitempty"` + PinnedImages []string `yson:"pinned_image,omitempty"` + + AlwaysPullLatest *bool `yson:"always_pull_latest,omitempty"` + PullPeriod yson.Duration `yson:"pullPeriod,omitempty"` + ImageSizeEstimation *int64 `yson:"image_size_estimation,omitempty"` + ImageCompressionRatioEstimation *float32 `yson:"image_compression_ratio_estimation,omitempty"` + YoungerSizeFraction *float32 `yson:"younger_size_fraction,omitempty"` +} + type CriJobEnvironment struct { - CriExecutor *CriExecutor `yson:"cri_executor,omitempty"` - JobProxyImage string `yson:"job_proxy_image,omitempty"` - JobProxyBindMounts []BindMount `yson:"job_proxy_bind_mounts,omitempty"` - UseJobProxyFromImage *bool `yson:"use_job_proxy_from_image,omitempty"` + CriExecutor *CriExecutor `yson:"cri_executor,omitempty"` + CriImageCache *CriImageCache `yson:"cri_image_cache,omitempty"` + JobProxyImage string `yson:"job_proxy_image,omitempty"` + JobProxyBindMounts []BindMount `yson:"job_proxy_bind_mounts,omitempty"` + UseJobProxyFromImage *bool `yson:"use_job_proxy_from_image,omitempty"` } type JobEnvironment struct { @@ -404,6 +419,20 @@ func fillJobEnvironment(execNode *ExecNode, spec *ytv1.ExecNodesSpec, commonSpec } } + if location := ytv1.FindFirstLocation(spec.Locations, ytv1.LocationTypeImageCache); location != nil { + jobEnv.CriImageCache = &CriImageCache{ + Capacity: findQuotaForLocation(*location, spec.InstanceSpec), + ImageSizeEstimation: envSpec.CRI.ImageSizeEstimation, + AlwaysPullLatest: envSpec.CRI.AlwaysPullLatestImage, + } + if ratio := envSpec.CRI.ImageCompressionRatioEstimation; ratio != nil { + jobEnv.CriImageCache.ImageCompressionRatioEstimation = ptr.Float32(float32(*ratio)) + } + if period := envSpec.CRI.ImagePullPeriodSeconds; period != nil { + jobEnv.CriImageCache.PullPeriod = yson.Duration(time.Duration(*period) * time.Second) + } + } + // NOTE: Default was "false", now it's "true" and option was moved into dynamic config. execNode.UseArtifactBindsLegacy = ptr.Bool(ptr.BoolDeref(envSpec.UseArtifactBinds, true)) if !*execNode.UseArtifactBindsLegacy { diff --git a/ytop-chart/templates/remoteexecnodes-crd.yaml b/ytop-chart/templates/remoteexecnodes-crd.yaml index 0467d558..24247fde 100644 --- a/ytop-chart/templates/remoteexecnodes-crd.yaml +++ b/ytop-chart/templates/remoteexecnodes-crd.yaml @@ -683,6 +683,9 @@ spec: description: CRI service configuration for running jobs in sidecar container. properties: + alwaysPullLatestImage: + description: Always pull "latest" images. + type: boolean apiRetryTimeoutSeconds: description: Timeout for retrying CRI API calls. format: int32 @@ -699,6 +702,20 @@ spec: items: type: string type: array + imageCompressionRatioEstimation: + description: Multiplier for image size to account space used + by unpacked images. + format: int32 + type: integer + imagePullPeriodSeconds: + description: Pull images periodically. + format: int32 + type: integer + imageSizeEstimation: + description: Initial estimation for space required for pulling + image into cache. + format: int64 + type: integer registryConfigPath: description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md' type: string diff --git a/ytop-chart/templates/ytsaurus-crd.yaml b/ytop-chart/templates/ytsaurus-crd.yaml index 6014842d..dfadea07 100644 --- a/ytop-chart/templates/ytsaurus-crd.yaml +++ b/ytop-chart/templates/ytsaurus-crd.yaml @@ -7768,6 +7768,9 @@ spec: description: CRI service configuration for running jobs in sidecar container. properties: + alwaysPullLatestImage: + description: Always pull "latest" images. + type: boolean apiRetryTimeoutSeconds: description: Timeout for retrying CRI API calls. format: int32 @@ -7784,6 +7787,20 @@ spec: items: type: string type: array + imageCompressionRatioEstimation: + description: Multiplier for image size to account space + used by unpacked images. + format: int32 + type: integer + imagePullPeriodSeconds: + description: Pull images periodically. + format: int32 + type: integer + imageSizeEstimation: + description: Initial estimation for space required for + pulling image into cache. + format: int64 + type: integer registryConfigPath: description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md' type: string