diff --git a/pkg/components/canondata/TestDoBuildExecNodeBase/isolated-containers-with-job-resources/test.canondata b/pkg/components/canondata/TestDoBuildExecNodeBase/isolated-containers-with-job-resources/test.canondata new file mode 100644 index 00000000..fe80a21c --- /dev/null +++ b/pkg/components/canondata/TestDoBuildExecNodeBase/isolated-containers-with-job-resources/test.canondata @@ -0,0 +1,265 @@ +{ + "address_resolver"={ + "enable_ipv4"=%false; + "enable_ipv6"=%true; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + ]; + }; + "master_cache"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=111669149696; + "total_cpu"=119.000000; + "node_dedicated_cpu"=20.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/components/canondata/TestDoBuildExecNodeBase/isolated-containers-without-job-resources/test.canondata b/pkg/components/canondata/TestDoBuildExecNodeBase/isolated-containers-without-job-resources/test.canondata new file mode 100644 index 00000000..a22eab66 --- /dev/null +++ b/pkg/components/canondata/TestDoBuildExecNodeBase/isolated-containers-without-job-resources/test.canondata @@ -0,0 +1,265 @@ +{ + "address_resolver"={ + "enable_ipv4"=%false; + "enable_ipv6"=%true; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + ]; + }; + "master_cache"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=5368709120; + "total_cpu"=20.000000; + "node_dedicated_cpu"=0.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/components/canondata/TestDoBuildExecNodeBase/single-container-with-job-resources/test.canondata b/pkg/components/canondata/TestDoBuildExecNodeBase/single-container-with-job-resources/test.canondata new file mode 100644 index 00000000..fe80a21c --- /dev/null +++ b/pkg/components/canondata/TestDoBuildExecNodeBase/single-container-with-job-resources/test.canondata @@ -0,0 +1,265 @@ +{ + "address_resolver"={ + "enable_ipv4"=%false; + "enable_ipv6"=%true; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + ]; + }; + "master_cache"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=111669149696; + "total_cpu"=119.000000; + "node_dedicated_cpu"=20.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/components/canondata/TestDoBuildExecNodeBase/single-container-without-job-resources/test.canondata b/pkg/components/canondata/TestDoBuildExecNodeBase/single-container-without-job-resources/test.canondata new file mode 100644 index 00000000..a22eab66 --- /dev/null +++ b/pkg/components/canondata/TestDoBuildExecNodeBase/single-container-without-job-resources/test.canondata @@ -0,0 +1,265 @@ +{ + "address_resolver"={ + "enable_ipv4"=%false; + "enable_ipv6"=%true; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + ]; + }; + "master_cache"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=5368709120; + "total_cpu"=20.000000; + "node_dedicated_cpu"=0.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/components/exec_node_base.go b/pkg/components/exec_node_base.go index d0b6c10d..1fd7b90b 100644 --- a/pkg/components/exec_node_base.go +++ b/pkg/components/exec_node_base.go @@ -146,7 +146,7 @@ func (n *baseExecNode) doBuildCRISidecar(envSpec *ytv1.JobEnvironmentSpec, podSp } if n.spec.JobResources != nil { - jobsContainer.Resources = *n.spec.JobResources + jobsContainer.Resources = *n.spec.JobResources.DeepCopy() } else { // Without dedicated job resources enforce same limits as for node. jobsContainer.Resources.Limits = n.spec.Resources.Limits diff --git a/pkg/components/exec_node_base_test.go b/pkg/components/exec_node_base_test.go new file mode 100644 index 00000000..935a6404 --- /dev/null +++ b/pkg/components/exec_node_base_test.go @@ -0,0 +1,320 @@ +package components + +import ( + "testing" + + "github.com/stretchr/testify/require" + ytv1 "github.com/ytsaurus/yt-k8s-operator/api/v1" + apiProxy "github.com/ytsaurus/yt-k8s-operator/pkg/apiproxy" + "github.com/ytsaurus/yt-k8s-operator/pkg/canonize" + "github.com/ytsaurus/yt-k8s-operator/pkg/consts" + "github.com/ytsaurus/yt-k8s-operator/pkg/labeller" + "github.com/ytsaurus/yt-k8s-operator/pkg/ytconfig" + "go.ytsaurus.tech/library/go/ptr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + testClusterDomain = "fake.zone" + + testNamespace = "fake" + testYtsaurusName = "test" +) + +var ( + testLogRotationPeriod int64 = 900000 + testTotalLogSize = 10 * int64(1<<30) + + testStorageClassname = "yc-network-hdd" + + testResourceReqs = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(20, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(5*1024*1024*1024, resource.BinarySI), + }, + } + testJobResourceReqs = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(99, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(99*1024*1024*1024, resource.BinarySI), + }, + } + + testLocationChunkCache = ytv1.LocationSpec{ + LocationType: "ChunkCache", + Path: "/yt/hdd1/chunk-cache", + } + testLocationSlots = ytv1.LocationSpec{ + LocationType: "Slots", + Path: "/yt/hdd2/slots", + } + testLocationImageCache = ytv1.LocationSpec{ + LocationType: "ImageCache", + Path: "/yt/hdd1/images", + } + testVolumeMounts = []corev1.VolumeMount{ + { + Name: "hdd1", + MountPath: "/yt/hdd1", + }, + } + testVolumeClaimTemplates = []ytv1.EmbeddedPersistentVolumeClaim{ + { + EmbeddedObjectMetadata: ytv1.EmbeddedObjectMetadata{ + Name: "hdd1", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: &testStorageClassname, + }, + }, + } + testClusterNodeSpec = ytv1.ClusterNodesSpec{ + Tags: []string{"rack:xn-a"}, + Rack: "fake", + } +) + +func getCommonSpec() ytv1.CommonSpec { + return ytv1.CommonSpec{ + UseIPv6: true, + } +} + +func getMasterConnectionSpec() ytv1.MasterConnectionSpec { + return ytv1.MasterConnectionSpec{ + CellTag: 0, + } +} + +func getYtsaurus() *ytv1.Ytsaurus { + return &ytv1.Ytsaurus{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: testYtsaurusName, + }, + Spec: ytv1.YtsaurusSpec{ + CommonSpec: getCommonSpec(), + + PrimaryMasters: ytv1.MastersSpec{ + MasterConnectionSpec: getMasterConnectionSpec(), + MaxSnapshotCountToKeep: ptr.Int(1543), + InstanceSpec: ytv1.InstanceSpec{ + InstanceCount: 1, + MonitoringPort: ptr.Int32(consts.MasterMonitoringPort), + + VolumeMounts: []corev1.VolumeMount{ + { + Name: "master-data", + MountPath: "/yt/master-data", + }, + }, + + Locations: []ytv1.LocationSpec{ + { + LocationType: ytv1.LocationTypeMasterChangelogs, + Path: "/yt/master-data/master-changelogs", + }, + { + LocationType: ytv1.LocationTypeMasterSnapshots, + Path: "/yt/master-data/master-snapshots", + }, + }, + + VolumeClaimTemplates: []ytv1.EmbeddedPersistentVolumeClaim{ + { + EmbeddedObjectMetadata: ytv1.EmbeddedObjectMetadata{ + Name: "master-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + }, + + Loggers: []ytv1.TextLoggerSpec{ + { + BaseLoggerSpec: ytv1.BaseLoggerSpec{ + Name: "info", + MinLogLevel: ytv1.LogLevelInfo, + Compression: ytv1.LogCompressionNone, + Format: ytv1.LogFormatPlainText, + }, + WriterType: ytv1.LogWriterTypeFile, + }, + { + BaseLoggerSpec: ytv1.BaseLoggerSpec{ + Name: "error", + MinLogLevel: ytv1.LogLevelError, + Compression: ytv1.LogCompressionNone, + Format: ytv1.LogFormatPlainText, + }, + WriterType: ytv1.LogWriterTypeFile, + }, + { + BaseLoggerSpec: ytv1.BaseLoggerSpec{ + Name: "debug", + MinLogLevel: ytv1.LogLevelDebug, + Compression: ytv1.LogCompressionZstd, + Format: ytv1.LogFormatPlainText, + + RotationPolicy: &ytv1.LogRotationPolicy{ + RotationPeriodMilliseconds: &testLogRotationPeriod, + MaxTotalSizeToKeep: &testTotalLogSize, + }, + }, + WriterType: ytv1.LogWriterTypeFile, + CategoriesFilter: &ytv1.CategoriesFilter{ + Type: ytv1.CategoriesFilterTypeExclude, + Values: []string{"Bus"}, + }, + }, + }, + }, + + Sidecars: []string{ + "{name: sleep, image: fakeimage:stable, command: [/bin/sleep], args: [inf]}", + }, + }, + }, + } +} + +func getExecNodeSpec(jobResources *corev1.ResourceRequirements) ytv1.ExecNodesSpec { + rotationPolicyMS := int64(900000) + rotationPolicyMaxTotalSize := int64(3145728) + return ytv1.ExecNodesSpec{ + InstanceSpec: ytv1.InstanceSpec{ + InstanceCount: 50, + MonitoringPort: ptr.Int32(consts.ExecNodeMonitoringPort), + Resources: testResourceReqs, + Locations: []ytv1.LocationSpec{ + testLocationChunkCache, + testLocationSlots, + }, + VolumeMounts: testVolumeMounts, + VolumeClaimTemplates: testVolumeClaimTemplates, + }, + JobResources: jobResources, + ClusterNodesSpec: testClusterNodeSpec, + JobProxyLoggers: []ytv1.TextLoggerSpec{ + { + BaseLoggerSpec: ytv1.BaseLoggerSpec{ + Name: "debug", + Format: ytv1.LogFormatPlainText, + MinLogLevel: ytv1.LogLevelDebug, + Compression: ytv1.LogCompressionZstd, + RotationPolicy: &ytv1.LogRotationPolicy{ + RotationPeriodMilliseconds: &rotationPolicyMS, + MaxTotalSizeToKeep: &rotationPolicyMaxTotalSize, + }, + }, + WriterType: ytv1.LogWriterTypeFile, + CategoriesFilter: &ytv1.CategoriesFilter{ + Type: ytv1.CategoriesFilterTypeExclude, + Values: []string{"Bus", "Concurrency"}, + }, + }, + }, + Name: "end-a", + } +} + +func withCri(spec ytv1.ExecNodesSpec, jobResources *corev1.ResourceRequirements, isolated bool) ytv1.ExecNodesSpec { + spec.Locations = append(spec.Locations, testLocationImageCache) + spec.JobResources = jobResources + spec.JobEnvironment = &ytv1.JobEnvironmentSpec{ + UserSlots: ptr.Int(42), + CRI: &ytv1.CRIJobEnvironmentSpec{ + SandboxImage: ptr.String("registry.k8s.io/pause:3.8"), + APIRetryTimeoutSeconds: ptr.Int32(120), + CRINamespace: ptr.String("yt"), + BaseCgroup: ptr.String("/yt"), + }, + UseArtifactBinds: ptr.Bool(true), + DoNotSetUserId: ptr.Bool(true), + Isolated: ptr.Bool(isolated), + } + return spec +} + +func TestDoBuildExecNodeBase(t *testing.T) { + yt := getYtsaurus() + ytsaurus := apiProxy.NewYtsaurus(yt, nil, nil, nil) + + l := labeller.Labeller{ + ObjectMeta: ptr.T(ytsaurus.GetResource().ObjectMeta), + APIProxy: ytsaurus.APIProxy(), + ComponentLabel: consts.YTComponentLabelExecNode, + ComponentName: consts.YTComponentLabelExecNode, + } + + g := ytconfig.NewLocalNodeGenerator(yt, testClusterDomain) + + cases := map[string]struct { + JobResources *corev1.ResourceRequirements + Isolated bool + }{ + "isolated-containers-without-job-resources": { + JobResources: nil, + Isolated: true, + }, + "isolated-containers-with-job-resources": { + JobResources: &testJobResourceReqs, + Isolated: true, + }, + "single-container-without-job-resources": { + JobResources: nil, + Isolated: false, + }, + "single-container-with-job-resources": { + JobResources: &testJobResourceReqs, + Isolated: false, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + execNodeSpec := withCri(getExecNodeSpec(nil), test.JobResources, test.Isolated) + + srv := newServer( + &l, + ytsaurus, + ptr.T(execNodeSpec.InstanceSpec), + "/usr/bin/ytserver-node", + "ytserver-exec-node.yson", + consts.YTComponentLabelExecNode, + consts.YTComponentLabelExecNode, + func() ([]byte, error) { + return g.GetExecNodeConfig(execNodeSpec) + }, + WithContainerPorts(corev1.ContainerPort{ + Name: consts.YTRPCPortName, + ContainerPort: consts.ExecNodeRPCPort, + Protocol: corev1.ProtocolTCP, + }), + ) + + bEN := baseExecNode{ + server: srv, + cfgen: g, + spec: &execNodeSpec, + sidecarConfig: nil, + } + + err := bEN.doBuildBase() + require.NoError(t, err) + + cfg, err := g.GetExecNodeConfig(execNodeSpec) + require.NoError(t, err) + canonize.Assert(t, cfg) + }) + } +} diff --git a/pkg/components/server.go b/pkg/components/server.go index facd53ec..4833f302 100644 --- a/pkg/components/server.go +++ b/pkg/components/server.go @@ -301,7 +301,7 @@ func (s *serverImpl) rebuildStatefulSet() *appsv1.StatefulSet { Command: command, VolumeMounts: volumeMounts, Ports: s.componentContainerPorts, - Resources: s.instanceSpec.Resources, + Resources: *s.instanceSpec.Resources.DeepCopy(), ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfig/with-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfig/with-job-resources/test.canondata new file mode 100644 index 00000000..168eaa0e --- /dev/null +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfig/with-job-resources/test.canondata @@ -0,0 +1,255 @@ +{ + "address_resolver"={ + "enable_ipv4"=%true; + "enable_ipv6"=%false; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + "ds-test-0.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-1.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-2.discovery-test.fake.svc.fake.zone:9020"; + ]; + }; + "master_cache"={ + addresses=[ + "msc-test-0.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-1.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-2.master-caches-test.fake.svc.fake.zone:9018"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=111669149696; + "total_cpu"=119.000000; + "node_dedicated_cpu"=20.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=495; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=porto; + "start_uid"=19500; + }; + "enable_tmpfs"=%true; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=495; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfig/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfig/without-job-resources/test.canondata similarity index 100% rename from pkg/ytconfig/canondata/TestGetExecNodeConfig/test.canondata rename to pkg/ytconfig/canondata/TestGetExecNodeConfig/without-job-resources/test.canondata diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata similarity index 99% rename from pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/test.canondata rename to pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata index ebbd897b..5bfdc910 100644 --- a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/test.canondata +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-with-job-resources/test.canondata @@ -90,8 +90,8 @@ exec; ]; "resource_limits"={ - "total_memory"=10737418240; - "total_cpu"=40.000000; + "total_memory"=111669149696; + "total_cpu"=119.000000; "node_dedicated_cpu"=20.000000; }; tags=[ diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata new file mode 100644 index 00000000..2a7d32c7 --- /dev/null +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/isolated-containers-without-job-resources/test.canondata @@ -0,0 +1,270 @@ +{ + "address_resolver"={ + "enable_ipv4"=%true; + "enable_ipv6"=%false; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + "ds-test-0.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-1.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-2.discovery-test.fake.svc.fake.zone:9020"; + ]; + }; + "master_cache"={ + addresses=[ + "msc-test-0.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-1.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-2.master-caches-test.fake.svc.fake.zone:9018"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=5368709120; + "total_cpu"=20.000000; + "node_dedicated_cpu"=0.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata new file mode 100644 index 00000000..5bfdc910 --- /dev/null +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-with-job-resources/test.canondata @@ -0,0 +1,270 @@ +{ + "address_resolver"={ + "enable_ipv4"=%true; + "enable_ipv6"=%false; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + "ds-test-0.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-1.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-2.discovery-test.fake.svc.fake.zone:9020"; + ]; + }; + "master_cache"={ + addresses=[ + "msc-test-0.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-1.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-2.master-caches-test.fake.svc.fake.zone:9018"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=111669149696; + "total_cpu"=119.000000; + "node_dedicated_cpu"=20.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata new file mode 100644 index 00000000..2a7d32c7 --- /dev/null +++ b/pkg/ytconfig/canondata/TestGetExecNodeConfigWithCri/single-container-without-job-resources/test.canondata @@ -0,0 +1,270 @@ +{ + "address_resolver"={ + "enable_ipv4"=%true; + "enable_ipv6"=%false; + retries=1000; + }; + "solomon_exporter"={ + host="{POD_SHORT_HOSTNAME}"; + "instance_tags"={ + pod="{K8S_POD_NAME}"; + }; + }; + logging={ + writers={ + info={ + type=file; + "file_name"="/var/log/exec-node.info.log"; + format="plain_text"; + "enable_system_messages"=%true; + }; + stderr={ + type=stderr; + format="plain_text"; + "enable_system_messages"=%true; + }; + }; + rules=[ + { + "min_level"=info; + writers=[ + info; + ]; + family="plain_text"; + }; + { + "min_level"=error; + writers=[ + stderr; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "monitoring_port"=10029; + "rpc_port"=9029; + "timestamp_provider"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + }; + "cluster_connection"={ + "cluster_name"=test; + "primary_master"={ + addresses=[ + "ms-test-0.masters-test.fake.svc.fake.zone:9010"; + ]; + peers=[ + { + address="ms-test-0.masters-test.fake.svc.fake.zone:9010"; + voting=%true; + }; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + }; + "discovery_connection"={ + addresses=[ + "ds-test-0.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-1.discovery-test.fake.svc.fake.zone:9020"; + "ds-test-2.discovery-test.fake.svc.fake.zone:9020"; + ]; + }; + "master_cache"={ + addresses=[ + "msc-test-0.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-1.master-caches-test.fake.svc.fake.zone:9018"; + "msc-test-2.master-caches-test.fake.svc.fake.zone:9018"; + ]; + "cell_id"="65726e65-ad6b7562-259-79747361"; + "enable_master_cache_discovery"=%false; + }; + }; + "cypress_annotations"={ + "k8s_node_name"="{K8S_NODE_NAME}"; + "k8s_pod_name"="{K8S_POD_NAME}"; + "k8s_pod_namespace"="{K8S_POD_NAMESPACE}"; + "physical_host"="{K8S_NODE_NAME}"; + }; + flavors=[ + exec; + ]; + "resource_limits"={ + "total_memory"=5368709120; + "total_cpu"=20.000000; + "node_dedicated_cpu"=0.000000; + }; + tags=[ + "rack:xn-a"; + ]; + rack=fake; + "skynet_http_port"=11029; + "job_resource_manager"={ + "resource_limits"={ + "user_slots"=42; + }; + }; + "exec_node"={ + "slot_manager"={ + locations=[ + { + path="/yt/hdd2/slots"; + "medium_name"=""; + "disk_usage_watermark"=0; + "enable_disk_quota"=%false; + }; + ]; + "job_environment"={ + type=cri; + "start_uid"=19500; + "cri_executor"={ + "retry_backoff_time"=1000; + "retry_attempts"=120; + "retry_timeout"=120000; + "runtime_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + "image_endpoint"="unix:///yt/hdd1/images/containerd.sock"; + namespace=yt; + "base_cgroup"="/yt"; + }; + "use_job_proxy_from_image"=%false; + }; + "do_not_set_user_id"=%true; + "enable_tmpfs"=%false; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + "job_controller"={ + "resource_limits"={ + "user_slots"=42; + }; + "gpu_manager"={ + "gpu_info_source"={ + type="nvidia_smi"; + }; + }; + }; + "job_proxy"={ + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "forward_all_environment_variables"=%true; + }; + "job_proxy_authentication_manager"={ + "cypress_cookie_manager"={ + }; + "cypress_user_manager"={ + }; + "cypress_token_authenticator"={ + secure=%true; + }; + "require_authentication"=%true; + }; + "job_proxy_logging"={ + writers={ + debug={ + type=file; + "file_name"="job-proxy.debug.log.zstd"; + format="plain_text"; + "compression_method"=zstd; + "enable_compression"=%true; + "enable_system_messages"=%true; + "rotation_policy"={ + "rotation_period"=900000; + "max_total_size_to_keep"=3145728; + }; + }; + }; + rules=[ + { + "exclude_categories"=[ + Bus; + Concurrency; + ]; + "min_level"=debug; + writers=[ + debug; + ]; + family="plain_text"; + }; + ]; + "flush_period"=3000; + }; + "do_not_set_user_id"=%true; + "forward_all_environment_variables"=%true; + "use_artifact_binds"=%true; + }; + "data_node"={ + "store_locations"=[ + ]; + "cache_locations"=[ + { + path="/yt/hdd1/chunk-cache"; + }; + ]; + "block_cache"={ + "compressed_data"={ + capacity=0; + }; + "uncompressed_data"={ + capacity=0; + }; + }; + "blocks_ext_cache"={ + capacity=0; + }; + "chunk_meta_cache"={ + capacity=0; + }; + "block_meta_cache"={ + capacity=0; + }; + }; + "tablet_node"={ + "versioned_chunk_meta_cache"={ + capacity=0; + }; + }; + "caching_object_service"={ + capacity=0; + }; +} \ No newline at end of file diff --git a/pkg/ytconfig/generator_test.go b/pkg/ytconfig/generator_test.go index 49244fde..85fb81f1 100644 --- a/pkg/ytconfig/generator_test.go +++ b/pkg/ytconfig/generator_test.go @@ -45,6 +45,12 @@ var ( corev1.ResourceMemory: *resource.NewQuantity(5*1024*1024*1024, resource.BinarySI), }, } + testJobResourceReqs = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewQuantity(99, resource.BinarySI), + corev1.ResourceMemory: *resource.NewQuantity(99*1024*1024*1024, resource.BinarySI), + }, + } testLocationChunkStore = ytv1.LocationSpec{ LocationType: "ChunkStore", Path: "/yt/hdd1/chunk-store", @@ -129,23 +135,66 @@ func TestGetDiscoveryConfig(t *testing.T) { } func TestGetExecNodeConfig(t *testing.T) { - g := NewLocalNodeGenerator(getYtsaurusWithEverything(), testClusterDomain) - cfg, err := g.GetExecNodeConfig(getExecNodeSpec()) - require.NoError(t, err) - canonize.Assert(t, cfg) + cases := map[string]struct { + JobResources *corev1.ResourceRequirements + }{ + "without-job-resources": { + JobResources: nil, + }, + "with-job-resources": { + JobResources: &testJobResourceReqs, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + g := NewLocalNodeGenerator(getYtsaurusWithEverything(), testClusterDomain) + cfg, err := g.GetExecNodeConfig(getExecNodeSpec(test.JobResources)) + require.NoError(t, err) + canonize.Assert(t, cfg) + }) + } } func TestGetExecNodeConfigWithCri(t *testing.T) { g := NewLocalNodeGenerator(getYtsaurusWithEverything(), testClusterDomain) - spec := withCri(getExecNodeSpec()) - cfg, err := g.GetExecNodeConfig(spec) - require.NoError(t, err) - canonize.Assert(t, cfg) + + cases := map[string]struct { + JobResources *corev1.ResourceRequirements + Isolated bool + }{ + "isolated-containers-without-job-resources": { + JobResources: nil, + Isolated: true, + }, + "isolated-containers-with-job-resources": { + JobResources: &testJobResourceReqs, + Isolated: true, + }, + "single-container-without-job-resources": { + JobResources: nil, + Isolated: false, + }, + "single-container-with-job-resources": { + JobResources: &testJobResourceReqs, + Isolated: false, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + spec := withCri(getExecNodeSpec(nil), test.JobResources, test.Isolated) + cfg, err := g.GetExecNodeConfig(spec) + require.NoError(t, err) + canonize.Assert(t, cfg) + }) + } } func TestGetContainerdConfig(t *testing.T) { g := NewLocalNodeGenerator(getYtsaurusWithEverything(), testClusterDomain) - spec := withCri(getExecNodeSpec()) + + spec := withCri(getExecNodeSpec(nil), nil, false) cfg, err := g.GetContainerdConfig(&spec) require.NoError(t, err) canonize.Assert(t, cfg) @@ -159,7 +208,7 @@ func TestGetExecNodeWithoutYtsaurusConfig(t *testing.T) { getMasterConnectionSpecWithFixedMasterHosts(), getMasterCachesSpecWithFixedHosts(), ) - cfg, err := g.GetExecNodeConfig(getExecNodeSpec()) + cfg, err := g.GetExecNodeConfig(getExecNodeSpec(nil)) require.NoError(t, err) canonize.Assert(t, cfg) } @@ -556,7 +605,7 @@ func getDataNodeSpec() ytv1.DataNodesSpec { } } -func getExecNodeSpec() ytv1.ExecNodesSpec { +func getExecNodeSpec(jobResources *corev1.ResourceRequirements) ytv1.ExecNodesSpec { rotationPolicyMS := int64(900000) rotationPolicyMaxTotalSize := int64(3145728) return ytv1.ExecNodesSpec{ @@ -571,6 +620,7 @@ func getExecNodeSpec() ytv1.ExecNodesSpec { VolumeMounts: testVolumeMounts, VolumeClaimTemplates: testVolumeClaimTemplates, }, + JobResources: jobResources, ClusterNodesSpec: testClusterNodeSpec, JobProxyLoggers: []ytv1.TextLoggerSpec{ { @@ -595,9 +645,9 @@ func getExecNodeSpec() ytv1.ExecNodesSpec { } } -func withCri(spec ytv1.ExecNodesSpec) ytv1.ExecNodesSpec { +func withCri(spec ytv1.ExecNodesSpec, jobResources *corev1.ResourceRequirements, isolated bool) ytv1.ExecNodesSpec { spec.Locations = append(spec.Locations, testLocationImageCache) - spec.JobResources = &testResourceReqs + spec.JobResources = jobResources spec.JobEnvironment = &ytv1.JobEnvironmentSpec{ UserSlots: ptr.Int(42), CRI: &ytv1.CRIJobEnvironmentSpec{ @@ -608,6 +658,7 @@ func withCri(spec ytv1.ExecNodesSpec) ytv1.ExecNodesSpec { }, UseArtifactBinds: ptr.Bool(true), DoNotSetUserId: ptr.Bool(true), + Isolated: ptr.Bool(isolated), } return spec }