Skip to content

Commit

Permalink
Add CRI image cache config
Browse files Browse the repository at this point in the history
  • Loading branch information
@koct9i
koct9i committed Jun 12, 2024
1 parent 22954cf commit cd529f1
Show file tree
Hide file tree
Showing 12 changed files with 149 additions and 4 deletions.
12 changes: 12 additions & 0 deletions api/v1/ytsaurus_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,18 @@ type CRIJobEnvironmentSpec struct {
// See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
//+optional
RegistryConfigPath *string `json:"registryConfigPath,omitempty"`
// Initial estimation for space required for pulling image into cache.
//+optional
ImageSizeEstimation *int64 `json:"imageSizeEstimation,omitempty"`
// Multiplier for image size to account space used by unpacked images.
//+optional
ImageCompressionRatioEstimation *int32 `json:"imageCompressionRatioEstimation,omitempty"`
// Always pull "latest" images.
//+optional
AlwaysPullLatestImage *bool `json:"alwaysPullLatestImage,omitempty"`
// Pull images periodically.
//+optional
ImagePullPeriodSeconds *int32 `json:"imagePullPeriodSeconds,omitempty"`
}

type JobEnvironmentSpec struct {
Expand Down
20 changes: 20 additions & 0 deletions api/v1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_remoteexecnodes.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,9 @@ spec:
description: CRI service configuration for running jobs in sidecar
container.
properties:
alwaysPullLatestImage:
description: Always pull "latest" images.
type: boolean
apiRetryTimeoutSeconds:
description: Timeout for retrying CRI API calls.
format: int32
Expand All @@ -715,6 +718,20 @@ spec:
items:
type: string
type: array
imageCompressionRatioEstimation:
description: Multiplier for image size to account space used
by unpacked images.
format: int32
type: integer
imagePullPeriodSeconds:
description: Pull images periodically.
format: int32
type: integer
imageSizeEstimation:
description: Initial estimation for space required for pulling
image into cache.
format: int64
type: integer
registryConfigPath:
description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md'
type: string
Expand Down
17 changes: 17 additions & 0 deletions config/crd/bases/cluster.ytsaurus.tech_ytsaurus.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7812,6 +7812,9 @@ spec:
description: CRI service configuration for running jobs
in sidecar container.
properties:
alwaysPullLatestImage:
description: Always pull "latest" images.
type: boolean
apiRetryTimeoutSeconds:
description: Timeout for retrying CRI API calls.
format: int32
Expand All @@ -7828,6 +7831,20 @@ spec:
items:
type: string
type: array
imageCompressionRatioEstimation:
description: Multiplier for image size to account space
used by unpacked images.
format: int32
type: integer
imagePullPeriodSeconds:
description: Pull images periodically.
format: int32
type: integer
imageSizeEstimation:
description: Initial estimation for space required for
pulling image into cache.
format: int64
type: integer
registryConfigPath:
description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md'
type: string
Expand Down
4 changes: 4 additions & 0 deletions docs/api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ _Appears in:_
| `criNamespace` _string_ | CRI namespace for jobs containers. | | |
| `baseCgroup` _string_ | Base cgroup for jobs. | | |
| `registryConfigPath` _string_ | See: https://github.com/containerd/containerd/blob/main/docs/hosts.md | | |
| `imageSizeEstimation` _integer_ | Initial estimation for space required for pulling image into cache. | | |
| `imageCompressionRatioEstimation` _integer_ | Multiplier for image size to account space used by unpacked images. | | |
| `alwaysPullLatestImage` _boolean_ | Always pull "latest" images. | | |
| `imagePullPeriodSeconds` _integer_ | Pull images periodically. | | |


#### CategoriesFilter
Expand Down
3 changes: 3 additions & 0 deletions ...ondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@
namespace=yt;
"base_cgroup"="/yt";
};
"cri_image_cache"={
capacity=4294967296;
};
"use_job_proxy_from_image"=%false;
};
"do_not_set_user_id"=%true;
Expand Down
3 changes: 3 additions & 0 deletions ...ata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@
namespace=yt;
"base_cgroup"="/yt";
};
"cri_image_cache"={
capacity=4294967296;
};
"use_job_proxy_from_image"=%false;
};
"do_not_set_user_id"=%true;
Expand Down
3 changes: 3 additions & 0 deletions ...canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@
namespace=yt;
"base_cgroup"="/yt";
};
"cri_image_cache"={
capacity=4294967296;
};
"use_job_proxy_from_image"=%false;
};
"do_not_set_user_id"=%true;
Expand Down
3 changes: 3 additions & 0 deletions ...ondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@
namespace=yt;
"base_cgroup"="/yt";
};
"cri_image_cache"={
capacity=4294967296;
};
"use_job_proxy_from_image"=%false;
};
"do_not_set_user_id"=%true;
Expand Down
37 changes: 33 additions & 4 deletions pkg/ytconfig/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,26 @@ type CriExecutor struct {
CpuPeriod yson.Duration `yson:"cpu_period,omitempty"`
}

type CriImageCache struct {
Capacity *int64 `yson:"capacity,omitempty"`

ManagedPrefixes []string `yson:"managed_prefixes,omitempty"`
UnmanagedPrefixes []string `yson:"managed_prefixes,omitempty"`
PinnedImages []string `yson:"pinned_image,omitempty"`

AlwaysPullLatest *bool `yson:"always_pull_latest,omitempty"`
PullPeriod yson.Duration `yson:"pullPeriod,omitempty"`
ImageSizeEstimation *int64 `yson:"image_size_estimation,omitempty"`
ImageCompressionRatioEstimation *float32 `yson:"image_compression_ratio_estimation,omitempty"`
YoungerSizeFraction *float32 `yson:"younger_size_fraction,omitempty"`
}

type CriJobEnvironment struct {
CriExecutor *CriExecutor `yson:"cri_executor,omitempty"`
JobProxyImage string `yson:"job_proxy_image,omitempty"`
JobProxyBindMounts []BindMount `yson:"job_proxy_bind_mounts,omitempty"`
UseJobProxyFromImage *bool `yson:"use_job_proxy_from_image,omitempty"`
CriExecutor *CriExecutor `yson:"cri_executor,omitempty"`
CriImageCache *CriImageCache `yson:"cri_image_cache,omitempty"`
JobProxyImage string `yson:"job_proxy_image,omitempty"`
JobProxyBindMounts []BindMount `yson:"job_proxy_bind_mounts,omitempty"`
UseJobProxyFromImage *bool `yson:"use_job_proxy_from_image,omitempty"`
}

type JobEnvironment struct {
Expand Down Expand Up @@ -404,6 +419,20 @@ func fillJobEnvironment(execNode *ExecNode, spec *ytv1.ExecNodesSpec, commonSpec
}
}

if location := ytv1.FindFirstLocation(spec.Locations, ytv1.LocationTypeImageCache); location != nil {
jobEnv.CriImageCache = &CriImageCache{
Capacity: findQuotaForLocation(*location, spec.InstanceSpec),
ImageSizeEstimation: envSpec.CRI.ImageSizeEstimation,
AlwaysPullLatest: envSpec.CRI.AlwaysPullLatestImage,
}
if ratio := envSpec.CRI.ImageCompressionRatioEstimation; ratio != nil {
jobEnv.CriImageCache.ImageCompressionRatioEstimation = ptr.Float32(float32(*ratio))
}
if period := envSpec.CRI.ImagePullPeriodSeconds; period != nil {
jobEnv.CriImageCache.PullPeriod = yson.Duration(time.Duration(*period) * time.Second)
}
}

// NOTE: Default was "false", now it's "true" and option was moved into dynamic config.
execNode.UseArtifactBindsLegacy = ptr.Bool(ptr.BoolDeref(envSpec.UseArtifactBinds, true))
if !*execNode.UseArtifactBindsLegacy {
Expand Down
17 changes: 17 additions & 0 deletions ytop-chart/templates/remoteexecnodes-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,6 +683,9 @@ spec:
description: CRI service configuration for running jobs in sidecar
container.
properties:
alwaysPullLatestImage:
description: Always pull "latest" images.
type: boolean
apiRetryTimeoutSeconds:
description: Timeout for retrying CRI API calls.
format: int32
Expand All @@ -699,6 +702,20 @@ spec:
items:
type: string
type: array
imageCompressionRatioEstimation:
description: Multiplier for image size to account space used
by unpacked images.
format: int32
type: integer
imagePullPeriodSeconds:
description: Pull images periodically.
format: int32
type: integer
imageSizeEstimation:
description: Initial estimation for space required for pulling
image into cache.
format: int64
type: integer
registryConfigPath:
description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md'
type: string
Expand Down
17 changes: 17 additions & 0 deletions ytop-chart/templates/ytsaurus-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7768,6 +7768,9 @@ spec:
description: CRI service configuration for running jobs in
sidecar container.
properties:
alwaysPullLatestImage:
description: Always pull "latest" images.
type: boolean
apiRetryTimeoutSeconds:
description: Timeout for retrying CRI API calls.
format: int32
Expand All @@ -7784,6 +7787,20 @@ spec:
items:
type: string
type: array
imageCompressionRatioEstimation:
description: Multiplier for image size to account space
used by unpacked images.
format: int32
type: integer
imagePullPeriodSeconds:
description: Pull images periodically.
format: int32
type: integer
imageSizeEstimation:
description: Initial estimation for space required for
pulling image into cache.
format: int64
type: integer
registryConfigPath:
description: 'See: https://github.com/containerd/containerd/blob/main/docs/hosts.md'
type: string
Expand Down

0 comments on commit cd529f1

Please sign in to comment.