diff --git a/docs/docs/40-admin/03-computeclasses.md b/docs/docs/40-admin/03-computeclasses.md index 1445417d8e..09b8d7f03e 100644 --- a/docs/docs/40-admin/03-computeclasses.md +++ b/docs/docs/40-admin/03-computeclasses.md @@ -24,6 +24,11 @@ memory: values: # Specific values that are only allowed to be used. Default must be included in these values and max/min cannot be set. - 1.5Gi cpuScaler: 1 # This is used as a ratio of how many VCPUs to schedule per Gibibyte of memory. In this case it is 1 to 1. +resources: # The same resources fields for Pods: memory and CPU values will be overwritten by memory and cpuScaler fields + limits: + gpu-vendor.example/example-limit: 1 + requests: + gpu-vendor.example/example-request: 1 priorityClassName: foo # The priority class to use for Pods runtimeClassName: bar # The runtime class name to use for Pods tolerations: # The same toleration fields for Pods @@ -42,7 +47,7 @@ affinity: # The same affinity fields for Pods - bar ``` -If `memory.min`, `memory.max`, `memory.values`, `affinity`, and `tolerations` are not given, then there are no scheduling rules for workloads using the compute class. +If `memory.min`, `memory.max`, `memory.values`, `resources`, `affinity`, and `tolerations` are not given, then there are no scheduling rules for workloads using the compute class. ## Cluster Compute Classes diff --git a/integration/client/computeclass/computeclass_test.go b/integration/client/computeclass/computeclass_test.go index 15cb953aed..5f6c9359e7 100644 --- a/integration/client/computeclass/computeclass_test.go +++ b/integration/client/computeclass/computeclass_test.go @@ -162,26 +162,6 @@ func TestCreatingComputeClasses(t *testing.T) { }, fail: true, }, - // // Raise error to avoid conflicts with - // // the "first class" fields for memory and cpu scaling - // { - // name: "invalid-custom-resources-limits", - // resources: corev1.ResourceRequirements{ - // Limits: corev1.ResourceList{ - // "cpu": resource.MustParse("1"), - // }, - // }, - // fail: true, - // }, - // { - // name: "invalid-custom-resources-requests", - // resources: corev1.ResourceRequirements{ - // Requests: corev1.ResourceList{ - // "memory": resource.MustParse("1"), - // }, - // }, - // fail: true, - // }, } for _, tt := range checks { diff --git a/pkg/apis/admin.acorn.io/v1/zz_generated.deepcopy.go b/pkg/apis/admin.acorn.io/v1/zz_generated.deepcopy.go index 00b3b67f07..02127b5c4c 100644 --- a/pkg/apis/admin.acorn.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/admin.acorn.io/v1/zz_generated.deepcopy.go @@ -34,6 +34,11 @@ func (in *ClusterComputeClass) DeepCopyInto(out *ClusterComputeClass) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterComputeClass. @@ -295,6 +300,11 @@ func (in *ProjectComputeClass) DeepCopyInto(out *ProjectComputeClass) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectComputeClass. diff --git a/pkg/apis/api.acorn.io/v1/zz_generated.deepcopy.go b/pkg/apis/api.acorn.io/v1/zz_generated.deepcopy.go index a319d08d39..3f62594c4e 100644 --- a/pkg/apis/api.acorn.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/api.acorn.io/v1/zz_generated.deepcopy.go @@ -7,6 +7,7 @@ package v1 import ( internal_acorn_iov1 "github.com/acorn-io/runtime/pkg/apis/internal.acorn.io/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -351,6 +352,11 @@ func (in *ComputeClass) DeepCopyInto(out *ComputeClass) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Memory.DeepCopyInto(&out.Memory) + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } if in.SupportedRegions != nil { in, out := &in.SupportedRegions, &out.SupportedRegions *out = make([]string, len(*in)) diff --git a/pkg/apis/internal.admin.acorn.io/v1/zz_generated.deepcopy.go b/pkg/apis/internal.admin.acorn.io/v1/zz_generated.deepcopy.go index 7d742d2408..edb131bc32 100644 --- a/pkg/apis/internal.admin.acorn.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/internal.admin.acorn.io/v1/zz_generated.deepcopy.go @@ -52,6 +52,11 @@ func (in *ClusterComputeClassInstance) DeepCopyInto(out *ClusterComputeClassInst *out = make([]string, len(*in)) copy(*out, *in) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterComputeClassInstance. @@ -365,6 +370,11 @@ func (in *ProjectComputeClassInstance) DeepCopyInto(out *ProjectComputeClassInst *out = make([]string, len(*in)) copy(*out, *in) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectComputeClassInstance. diff --git a/pkg/controller/appdefinition/testdata/computeclass/generic-resources/expected.golden b/pkg/controller/appdefinition/testdata/computeclass/generic-resources/expected.golden index c8439e019c..2f9dcc0325 100644 --- a/pkg/controller/appdefinition/testdata/computeclass/generic-resources/expected.golden +++ b/pkg/controller/appdefinition/testdata/computeclass/generic-resources/expected.golden @@ -86,9 +86,11 @@ spec: port: 81 resources: limits: + gpu-vendor.example/example-limit: "1" memory: 1Mi requests: cpu: 1m + gpu-vendor.example/example-limit: "1" memory: 1Mi - image: foo name: left @@ -259,9 +261,11 @@ status: - bar requirements: limits: + gpu-vendor.example/example-limit: "1" memory: 1Mi requests: cpu: 1m + gpu-vendor.example/example-limit: "1" memory: 1Mi staged: appImage: diff --git a/pkg/controller/appdefinition/testdata/computeclass/generic-resources/input.yaml b/pkg/controller/appdefinition/testdata/computeclass/generic-resources/input.yaml index 44a71471a4..7707929d0a 100644 --- a/pkg/controller/appdefinition/testdata/computeclass/generic-resources/input.yaml +++ b/pkg/controller/appdefinition/testdata/computeclass/generic-resources/input.yaml @@ -23,9 +23,11 @@ status: - bar requirements: limits: + gpu-vendor.example/example-limit: 1 memory: 1Mi requests: cpu: 1m + gpu-vendor.example/example-limit: 1 memory: 1Mi left: requirements: diff --git a/pkg/controller/scheduling/scheduling.go b/pkg/controller/scheduling/scheduling.go index 6a29ef56c4..d58be28d04 100644 --- a/pkg/controller/scheduling/scheduling.go +++ b/pkg/controller/scheduling/scheduling.go @@ -71,10 +71,8 @@ func addScheduling(req router.Request, appInstance *v1.AppInstance, workloads ma for sidecarName, sidecarContainer := range container.Sidecars { // disable extra resource indication (GPU et al. for sidecars) - if computeClass != nil { - if computeClass.Resources != nil { - computeClass.Resources = &corev1.ResourceRequirements{Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}} - } + if computeClass != nil && computeClass.Resources != nil { + computeClass.Resources = &corev1.ResourceRequirements{} } sidecarRequirements, err := ResourceRequirements(req, appInstance, sidecarName, sidecarContainer, computeClass) if err != nil { @@ -161,14 +159,12 @@ func ResourceRequirements(req router.Request, app *v1.AppInstance, containerName } requirements := &corev1.ResourceRequirements{Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}} - if computeClass != nil { - if computeClass.Resources != nil { - if computeClass.Resources.Requests != nil { - requirements.Requests = computeClass.Resources.Requests - } - if computeClass.Resources.Limits != nil { - requirements.Limits = computeClass.Resources.Limits - } + if computeClass != nil && computeClass.Resources != nil { + if computeClass.Resources.Requests != nil { + requirements.Requests = computeClass.Resources.Requests + } + if computeClass.Resources.Limits != nil { + requirements.Limits = computeClass.Resources.Limits } } diff --git a/pkg/openapi/generated/openapi_generated.go b/pkg/openapi/generated/openapi_generated.go index 985f6ed937..86589dc7fc 100644 --- a/pkg/openapi/generated/openapi_generated.go +++ b/pkg/openapi/generated/openapi_generated.go @@ -632,12 +632,17 @@ func schema_pkg_apis_adminacornio_v1_ClusterComputeClass(ref common.ReferenceCal Format: "", }, }, + "resources": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), + }, + }, }, Required: []string{"default"}, }, }, Dependencies: []string{ - "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, } } @@ -1115,12 +1120,17 @@ func schema_pkg_apis_adminacornio_v1_ProjectComputeClass(ref common.ReferenceCal Format: "", }, }, + "resources": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), + }, + }, }, Required: []string{"default"}, }, }, Dependencies: []string{ - "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, } } @@ -2009,6 +2019,11 @@ func schema_pkg_apis_apiacornio_v1_ComputeClass(ref common.ReferenceCallback) co Ref: ref("github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory"), }, }, + "resources": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), + }, + }, "description": { SchemaProps: spec.SchemaProps{ Type: []string{"string"}, @@ -2041,7 +2056,7 @@ func schema_pkg_apis_apiacornio_v1_ComputeClass(ref common.ReferenceCallback) co }, }, Dependencies: []string{ - "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, } } @@ -13528,12 +13543,17 @@ func schema_pkg_apis_internaladminacornio_v1_ClusterComputeClassInstance(ref com Format: "", }, }, + "resources": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), + }, + }, }, Required: []string{"default"}, }, }, Dependencies: []string{ - "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, } } @@ -14099,12 +14119,17 @@ func schema_pkg_apis_internaladminacornio_v1_ProjectComputeClassInstance(ref com Format: "", }, }, + "resources": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), + }, + }, }, Required: []string{"default"}, }, }, Dependencies: []string{ - "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + "github.com/acorn-io/runtime/pkg/apis/internal.admin.acorn.io/v1.ComputeClassMemory", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, } }