From 661dd71df888a7cb8e2b0891408b34225dc2ee85 Mon Sep 17 00:00:00 2001 From: xulinfei1996 <1187938268@qq.com> Date: Fri, 24 Nov 2023 18:29:54 +0800 Subject: [PATCH] scheduler: Add non-preemptible Request in quotaInfo's QuotaCalculateInfo (#1755) Signed-off-by: xulinfei.xlf Co-authored-by: xulinfei.xlf --- apis/extension/elastic_quota.go | 68 +++++--- .../plugins/elasticquota/controller.go | 40 ++++- .../elasticquota/core/group_quota_manager.go | 63 +++---- .../core/group_quota_manager_test.go | 155 ++++++++++++------ .../plugins/elasticquota/core/quota_info.go | 62 ++++--- .../elasticquota/core/quota_summary.go | 23 +-- .../core/runtime_quota_calculator_test.go | 8 +- 7 files changed, 273 insertions(+), 146 deletions(-) diff --git a/apis/extension/elastic_quota.go b/apis/extension/elastic_quota.go index b13e1f4ef..b061eb8ee 100644 --- a/apis/extension/elastic_quota.go +++ b/apis/extension/elastic_quota.go @@ -27,29 +27,31 @@ import ( // RootQuotaName means quotaTree's root\head. const ( - SystemQuotaName = "koordinator-system-quota" - RootQuotaName = "koordinator-root-quota" - DefaultQuotaName = "koordinator-default-quota" - QuotaKoordinatorPrefix = "quota.scheduling.koordinator.sh" - LabelQuotaIsParent = QuotaKoordinatorPrefix + "/is-parent" - LabelQuotaParent = QuotaKoordinatorPrefix + "/parent" - LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource" - LabelQuotaName = QuotaKoordinatorPrefix + "/name" - LabelQuotaProfile = QuotaKoordinatorPrefix + "/profile" - LabelQuotaIsRoot = QuotaKoordinatorPrefix + "/is-root" - LabelQuotaTreeID = QuotaKoordinatorPrefix + "/tree-id" - LabelQuotaIgnoreDefaultTree = QuotaKoordinatorPrefix + "/ignore-default-tree" - LabelPreemptible = QuotaKoordinatorPrefix + "/preemptible" - LabelAllowForceUpdate = QuotaKoordinatorPrefix + "/allow-force-update" - AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight" - AnnotationRuntime = QuotaKoordinatorPrefix + "/runtime" - AnnotationRequest = QuotaKoordinatorPrefix + "/request" - AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request" - AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys" - AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource" - AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces" - AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed" - AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated" + SystemQuotaName = "koordinator-system-quota" + RootQuotaName = "koordinator-root-quota" + DefaultQuotaName = "koordinator-default-quota" + QuotaKoordinatorPrefix = "quota.scheduling.koordinator.sh" + LabelQuotaIsParent = QuotaKoordinatorPrefix + "/is-parent" + LabelQuotaParent = QuotaKoordinatorPrefix + "/parent" + LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource" + LabelQuotaName = QuotaKoordinatorPrefix + "/name" + LabelQuotaProfile = QuotaKoordinatorPrefix + "/profile" + LabelQuotaIsRoot = QuotaKoordinatorPrefix + "/is-root" + LabelQuotaTreeID = QuotaKoordinatorPrefix + "/tree-id" + LabelQuotaIgnoreDefaultTree = QuotaKoordinatorPrefix + "/ignore-default-tree" + LabelPreemptible = QuotaKoordinatorPrefix + "/preemptible" + LabelAllowForceUpdate = QuotaKoordinatorPrefix + "/allow-force-update" + AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight" + AnnotationRuntime = QuotaKoordinatorPrefix + "/runtime" + AnnotationRequest = QuotaKoordinatorPrefix + "/request" + AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request" + AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys" + AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource" + AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces" + AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed" + AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated" + AnnotationNonPreemptibleRequest = QuotaKoordinatorPrefix + "/non-preemptible-request" + AnnotationNonPreemptibleUsed = QuotaKoordinatorPrefix + "/non-preemptible-used" ) func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string { @@ -124,6 +126,26 @@ func GetAnnotationQuotaNamespaces(quota *v1alpha1.ElasticQuota) []string { return namespaces } +func GetNonPreemptibleRequest(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) { + nonPreemptibleRequest := corev1.ResourceList{} + if quota.Annotations[AnnotationNonPreemptibleRequest] != "" { + if err := json.Unmarshal([]byte(quota.Annotations[AnnotationNonPreemptibleRequest]), &nonPreemptibleRequest); err != nil { + return nonPreemptibleRequest, err + } + } + return nonPreemptibleRequest, nil +} + +func GetNonPreemptibleUsed(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) { + nonPreemptibleUsed := corev1.ResourceList{} + if quota.Annotations[AnnotationNonPreemptibleUsed] != "" { + if err := json.Unmarshal([]byte(quota.Annotations[AnnotationNonPreemptibleUsed]), &nonPreemptibleUsed); err != nil { + return nonPreemptibleUsed, err + } + } + return nonPreemptibleUsed, nil +} + func GetGuaranteed(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) { guaranteed := corev1.ResourceList{} if quota.Annotations[AnnotationGuaranteed] != "" { diff --git a/pkg/scheduler/plugins/elasticquota/controller.go b/pkg/scheduler/plugins/elasticquota/controller.go index 594b1fb72..7259550b8 100644 --- a/pkg/scheduler/plugins/elasticquota/controller.go +++ b/pkg/scheduler/plugins/elasticquota/controller.go @@ -106,7 +106,7 @@ func (ctrl *Controller) syncHandler() []error { errors = append(errors, fmt.Errorf("failed get quota manager for %v", eq.Name)) return } - used, request, childRequest, runtime, guaranteed, allocated, err := mgr.GetQuotaInformationForSyncHandler(eq.Name) + used, request, childRequest, runtime, guaranteed, allocated, nonPreemptibleRequest, nonPreemptibleUsed, err := mgr.GetQuotaInformationForSyncHandler(eq.Name) if err != nil { errors = append(errors, err) return @@ -121,7 +121,7 @@ func (ctrl *Controller) syncHandler() []error { decorateResource(eq, guaranteed) decorateResource(eq, allocated) - var oriRuntime, oriRequest, oriChildRequest, oriGuaranteed, oriAllocated v1.ResourceList + var oriRuntime, oriRequest, oriChildRequest, oriGuaranteed, oriAllocated, oriNonPreemptibleRequest, oriNonPreemptibleUsed v1.ResourceList oriRequest, err = extension.GetRequest(eq) if err != nil { @@ -153,13 +153,27 @@ func (ctrl *Controller) syncHandler() []error { return } + oriNonPreemptibleRequest, err = extension.GetNonPreemptibleRequest(eq) + if err != nil { + errors = append(errors, err) + return + } + + oriNonPreemptibleUsed, err = extension.GetNonPreemptibleUsed(eq) + if err != nil { + errors = append(errors, err) + return + } + // Ignore this loop if the runtime/request/used doesn't change if quotav1.Equals(quotav1.RemoveZeros(eq.Status.Used), quotav1.RemoveZeros(used)) && quotav1.Equals(quotav1.RemoveZeros(oriRuntime), quotav1.RemoveZeros(runtime)) && quotav1.Equals(quotav1.RemoveZeros(oriRequest), quotav1.RemoveZeros(request)) && quotav1.Equals(quotav1.RemoveZeros(oriChildRequest), quotav1.RemoveZeros(childRequest)) && quotav1.Equals(quotav1.RemoveZeros(oriGuaranteed), quotav1.RemoveZeros(guaranteed)) && - quotav1.Equals(quotav1.RemoveZeros(oriAllocated), quotav1.RemoveZeros(allocated)) { + quotav1.Equals(quotav1.RemoveZeros(oriAllocated), quotav1.RemoveZeros(allocated)) && + quotav1.Equals(quotav1.RemoveZeros(oriNonPreemptibleRequest), quotav1.RemoveZeros(nonPreemptibleRequest)) && + quotav1.Equals(quotav1.RemoveZeros(oriNonPreemptibleUsed), quotav1.RemoveZeros(nonPreemptibleUsed)) { return } newEQ := eq.DeepCopy() @@ -191,19 +205,35 @@ func (ctrl *Controller) syncHandler() []error { errors = append(errors, err) return } + nonPreemptibleRequestBytes, err := json.Marshal(nonPreemptibleRequest) + if err != nil { + errors = append(errors, err) + return + } + nonPreemptibleUsedBytes, err := json.Marshal(nonPreemptibleUsed) + if err != nil { + errors = append(errors, err) + return + } newEQ.Annotations[extension.AnnotationRuntime] = string(runtimeBytes) newEQ.Annotations[extension.AnnotationRequest] = string(requestBytes) newEQ.Annotations[extension.AnnotationChildRequest] = string(childRequestBytes) newEQ.Annotations[extension.AnnotationGuaranteed] = string(guaranteedBytes) newEQ.Annotations[extension.AnnotationAllocated] = string(allocatedBytes) + newEQ.Annotations[extension.AnnotationNonPreemptibleRequest] = string(nonPreemptibleRequestBytes) + newEQ.Annotations[extension.AnnotationNonPreemptibleUsed] = string(nonPreemptibleUsedBytes) newEQ.Status.Used = used - klog.V(5).Infof("quota: %v, oldUsed: %v, newUsed: %v, oldRuntime: %v, newRuntime: %v, oldRequest: %v, newRequest: %v, oldChildRequest: %v, newChildRequest: %v, oldGuarantee: %v, newGuarantee: %v, oldAllocated: %v, newAllocated: %v", + klog.V(5).Infof("quota: %v, oldUsed: %v, newUsed: %v, oldRuntime: %v, newRuntime: %v, oldRequest: %v, newRequest: %v, "+ + "oldChildRequest: %v, newChildRequest: %v, oldGuarantee: %v, newGuarantee: %v, oldAllocated: %v, newAllocated: %v, "+ + "oldNonPreemptibleRequest: %v, newNonPreemptibleRequest: %v, oldNonPreemptibleUsed: %v, newNonPreemptibleUsed: %v", eq.Name, eq.Status.Used, used, eq.Annotations[extension.AnnotationRuntime], string(runtimeBytes), eq.Annotations[extension.AnnotationRequest], string(requestBytes), eq.Annotations[extension.AnnotationChildRequest], string(childRequestBytes), eq.Annotations[extension.AnnotationGuaranteed], string(guaranteedBytes), - eq.Annotations[extension.AnnotationAllocated], string(allocatedBytes)) + eq.Annotations[extension.AnnotationAllocated], string(allocatedBytes), + eq.Annotations[extension.AnnotationNonPreemptibleRequest], string(nonPreemptibleRequestBytes), + eq.Annotations[extension.AnnotationNonPreemptibleUsed], string(nonPreemptibleUsedBytes)) patch, err := util.CreateMergePatch(eq, newEQ) if err != nil { diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go index eed921ae5..f712eb2be 100644 --- a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go +++ b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go @@ -120,7 +120,7 @@ func (gqm *GroupQuotaManager) UpdateClusterTotalResource(deltaRes v1.ResourceLis func (gqm *GroupQuotaManager) updateClusterTotalResourceNoLock(deltaRes v1.ResourceList) { gqm.totalResource = quotav1.Add(gqm.totalResource, deltaRes) - sysAndDefaultUsed := v1.ResourceList{} + var sysAndDefaultUsed v1.ResourceList defaultQuota := gqm.quotaInfoMap[extension.DefaultQuotaName] if defaultQuota != nil { sysAndDefaultUsed = quotav1.Add(sysAndDefaultUsed, defaultQuota.CalculateInfo.Used.DeepCopy()) @@ -167,7 +167,7 @@ func (gqm *GroupQuotaManager) SetTotalResourceForTree(total v1.ResourceList) v1. } // updateGroupDeltaRequestNoLock no need lock gqm.lock -func (gqm *GroupQuotaManager) updateGroupDeltaRequestNoLock(quotaName string, deltaReq v1.ResourceList) { +func (gqm *GroupQuotaManager) updateGroupDeltaRequestNoLock(quotaName string, deltaReq, deltaNonPreemptibleRequest v1.ResourceList) { curToAllParInfos := gqm.getCurToAllParentGroupQuotaInfoNoLock(quotaName) allQuotaInfoLen := len(curToAllParInfos) if allQuotaInfoLen <= 0 { @@ -176,16 +176,16 @@ func (gqm *GroupQuotaManager) updateGroupDeltaRequestNoLock(quotaName string, de defer gqm.scopedLockForQuotaInfo(curToAllParInfos)() - gqm.recursiveUpdateGroupTreeWithDeltaRequest(deltaReq, curToAllParInfos) + gqm.recursiveUpdateGroupTreeWithDeltaRequest(deltaReq, deltaNonPreemptibleRequest, curToAllParInfos) } // recursiveUpdateGroupTreeWithDeltaRequest update the quota of a node, also need update all parentNode, the lock operation // of all quotaInfo is done by gqm. scopedLockForQuotaInfo, so just get treeWrappers' lock when calling treeWrappers' function -func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaRequest(deltaReq v1.ResourceList, curToAllParInfos []*QuotaInfo) { +func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaRequest(deltaReq, deltaNonPreemptibleRequest v1.ResourceList, curToAllParInfos []*QuotaInfo) { for i := 0; i < len(curToAllParInfos); i++ { curQuotaInfo := curToAllParInfos[i] oldSubLimitReq := curQuotaInfo.getLimitRequestNoLock() - curQuotaInfo.addRequestNonNegativeNoLock(deltaReq) + curQuotaInfo.addRequestNonNegativeNoLock(deltaReq, deltaNonPreemptibleRequest) if curQuotaInfo.Name == extension.RootQuotaName { return } @@ -250,7 +250,7 @@ func (gqm *GroupQuotaManager) updateGroupDeltaUsedNoLock(quotaName string, delta // if systemQuotaGroup or DefaultQuotaGroup's used change, update cluster total resource. if quotaName == extension.SystemQuotaName || quotaName == extension.DefaultQuotaName { - gqm.updateClusterTotalResourceNoLock(v1.ResourceList{}) + gqm.updateClusterTotalResourceNoLock(nil) } } @@ -472,7 +472,8 @@ func (gqm *GroupQuotaManager) buildSubParGroupTopoNoLock() { // ResetAllGroupQuotaNoLock no need to lock gqm.lock func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() { - childRequestMap, childNonPreemptibleUsedMap, childUsedMap := make(quotaResMapType), make(quotaResMapType), make(quotaResMapType) + childRequestMap, childNonPreemptibleUsedMap, childUsedMap, childNonPreemptibleRequestMap := + make(quotaResMapType), make(quotaResMapType), make(quotaResMapType), make(quotaResMapType) for quotaName, topoNode := range gqm.quotaTopoNodeMap { if quotaName == extension.RootQuotaName { gqm.resetRootQuotaUsedAndRequest() @@ -481,6 +482,7 @@ func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() { topoNode.quotaInfo.lock.Lock() if !topoNode.quotaInfo.IsParent { childRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.ChildRequest.DeepCopy() + childNonPreemptibleRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.NonPreemptibleRequest.DeepCopy() childNonPreemptibleUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.NonPreemptibleUsed.DeepCopy() childUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.Used.DeepCopy() } @@ -500,7 +502,7 @@ func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() { // subGroup's topo relation may change; refresh the request/used from bottom to top for quotaName, topoNode := range gqm.quotaTopoNodeMap { if !topoNode.quotaInfo.IsParent { - gqm.updateGroupDeltaRequestNoLock(quotaName, childRequestMap[quotaName]) + gqm.updateGroupDeltaRequestNoLock(quotaName, childRequestMap[quotaName], childNonPreemptibleRequestMap[quotaName]) gqm.updateGroupDeltaUsedNoLock(quotaName, childUsedMap[quotaName], childNonPreemptibleUsedMap[quotaName]) } } @@ -585,24 +587,27 @@ func (gqm *GroupQuotaManager) GetAllQuotaNames() map[string]struct{} { } func (gqm *GroupQuotaManager) updatePodRequestNoLock(quotaName string, oldPod, newPod *v1.Pod) { - var oldPodReq, newPodReq v1.ResourceList + var oldPodReq, newPodReq, oldNonPreemptibleRequest, newNonPreemptibleRequest v1.ResourceList if oldPod != nil { oldPodReq, _ = PodRequestsAndLimits(oldPod) - } else { - oldPodReq = make(v1.ResourceList) + if extension.IsPodNonPreemptible(oldPod) { + oldNonPreemptibleRequest = oldPodReq + } } if newPod != nil { newPodReq, _ = PodRequestsAndLimits(newPod) - } else { - newPodReq = make(v1.ResourceList) + if extension.IsPodNonPreemptible(newPod) { + newNonPreemptibleRequest = newPodReq + } } deltaReq := quotav1.Subtract(newPodReq, oldPodReq) - if quotav1.IsZero(deltaReq) { + deltaNonPreemptibleRequest := quotav1.Subtract(newNonPreemptibleRequest, oldNonPreemptibleRequest) + if quotav1.IsZero(deltaReq) && quotav1.IsZero(deltaNonPreemptibleRequest) { return } - gqm.updateGroupDeltaRequestNoLock(quotaName, deltaReq) + gqm.updateGroupDeltaRequestNoLock(quotaName, deltaReq, deltaNonPreemptibleRequest) } func (gqm *GroupQuotaManager) updatePodUsedNoLock(quotaName string, oldPod, newPod *v1.Pod) { @@ -622,9 +627,6 @@ func (gqm *GroupQuotaManager) updatePodUsedNoLock(quotaName string, oldPod, newP if extension.IsPodNonPreemptible(oldPod) { oldNonPreemptibleUsed = oldPodUsed } - } else { - oldPodUsed = make(v1.ResourceList) - oldNonPreemptibleUsed = make(v1.ResourceList) } if newPod != nil { @@ -632,9 +634,6 @@ func (gqm *GroupQuotaManager) updatePodUsedNoLock(quotaName string, oldPod, newP if extension.IsPodNonPreemptible(newPod) { newNonPreemptibleUsed = newPodUsed } - } else { - newPodUsed = make(v1.ResourceList) - newNonPreemptibleUsed = make(v1.ResourceList) } deltaUsed := quotav1.Subtract(newPodUsed, oldPodUsed) @@ -806,17 +805,20 @@ func (gqm *GroupQuotaManager) UnreservePod(quotaName string, p *v1.Pod) { gqm.updatePodIsAssignedNoLock(quotaName, p, false) } -func (gqm *GroupQuotaManager) GetQuotaInformationForSyncHandler(quotaName string) (used, request, childRequest, runtime, guaranteed, allocated v1.ResourceList, err error) { +func (gqm *GroupQuotaManager) GetQuotaInformationForSyncHandler(quotaName string) (used, request, childRequest, runtime, + guaranteed, allocated, nonPreemptibleRequest, nonPreemptibleUsed v1.ResourceList, err error) { gqm.hierarchyUpdateLock.RLock() defer gqm.hierarchyUpdateLock.RUnlock() quotaInfo := gqm.getQuotaInfoByNameNoLock(quotaName) if quotaInfo == nil { - return nil, nil, nil, nil, nil, nil, fmt.Errorf("groupQuotaManager doesn't have this quota:%v", quotaName) + return nil, nil, nil, nil, + nil, nil, nil, nil, fmt.Errorf("groupQuotaManager doesn't have this quota:%v", quotaName) } runtime = gqm.RefreshRuntimeNoLock(quotaName) - return quotaInfo.GetUsed(), quotaInfo.GetRequest(), quotaInfo.GetChildRequest(), runtime, quotaInfo.GetGuaranteed(), quotaInfo.GetAllocated(), nil + return quotaInfo.GetUsed(), quotaInfo.GetRequest(), quotaInfo.GetChildRequest(), runtime, + quotaInfo.GetGuaranteed(), quotaInfo.GetAllocated(), quotaInfo.GetNonPreemptibleRequest(), quotaInfo.GetNonPreemptibleUsed(), nil } func getPodName(oldPod, newPod *v1.Pod) string { @@ -871,7 +873,7 @@ func (gqm *GroupQuotaManager) OnNodeDelete(node *v1.Node) { return } - delta := quotav1.Subtract(v1.ResourceList{}, node.Status.Allocatable) + delta := quotav1.Subtract(nil, node.Status.Allocatable) gqm.UpdateClusterTotalResource(delta) delete(gqm.nodeResourceMap, node.Name) klog.V(5).Infof("OnNodeDeleteFunc success: %v [%v]", node.Name, delta) @@ -886,15 +888,14 @@ func (gqm *GroupQuotaManager) resetRootQuotaUsedAndRequest() { rootQuotaInfo.lock.Lock() defer rootQuotaInfo.lock.Unlock() - used := v1.ResourceList{} - request := v1.ResourceList{} - nonPreemptUsed := v1.ResourceList{} + var used, request, nonPreemptUsed, nonPreemptRequest v1.ResourceList systemQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.SystemQuotaName) if systemQuotaInfo != nil { used = quotav1.Add(used, systemQuotaInfo.GetUsed()) request = quotav1.Add(request, systemQuotaInfo.GetRequest()) nonPreemptUsed = quotav1.Add(nonPreemptUsed, systemQuotaInfo.GetNonPreemptibleUsed()) + nonPreemptRequest = quotav1.Add(nonPreemptRequest, systemQuotaInfo.GetNonPreemptibleRequest()) } defaultQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.DefaultQuotaName) @@ -902,11 +903,13 @@ func (gqm *GroupQuotaManager) resetRootQuotaUsedAndRequest() { used = quotav1.Add(used, defaultQuotaInfo.GetUsed()) request = quotav1.Add(request, defaultQuotaInfo.GetRequest()) nonPreemptUsed = quotav1.Add(nonPreemptUsed, defaultQuotaInfo.GetNonPreemptibleUsed()) + nonPreemptRequest = quotav1.Add(nonPreemptRequest, defaultQuotaInfo.GetNonPreemptibleRequest()) } rootQuotaInfo.CalculateInfo.Used = used rootQuotaInfo.CalculateInfo.Request = request rootQuotaInfo.CalculateInfo.NonPreemptibleUsed = nonPreemptUsed + rootQuotaInfo.CalculateInfo.NonPreemptibleRequest = nonPreemptRequest } func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaAllocated(deltaAllocated v1.ResourceList, curToAllParInfos []*QuotaInfo) { @@ -986,7 +989,7 @@ func (gqm *GroupQuotaManager) doUpdateOneGroupMaxQuotaNoLock(quotaName string, n newSubLimitReq := curQuotaInfo.getLimitRequestNoLock() deltaRequest := quotav1.Subtract(newSubLimitReq, oldSubLimitReq) - gqm.recursiveUpdateGroupTreeWithDeltaRequest(deltaRequest, curToAllParInfos[1:]) + gqm.recursiveUpdateGroupTreeWithDeltaRequest(deltaRequest, nil, curToAllParInfos[1:]) } } @@ -1024,7 +1027,7 @@ func (gqm *GroupQuotaManager) doUpdateOneGroupMinQuotaNoLock(quotaName string, n newSubLimitReq := curQuotaInfo.getLimitRequestNoLock() deltaRequest := quotav1.Subtract(newSubLimitReq, oldSubLimitReq) - gqm.recursiveUpdateGroupTreeWithDeltaRequest(deltaRequest, curToAllParInfos[1:]) + gqm.recursiveUpdateGroupTreeWithDeltaRequest(deltaRequest, nil, curToAllParInfos[1:]) } // update the guarantee. diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go index 436ae0f4f..cb1972ae3 100644 --- a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go +++ b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go @@ -162,7 +162,7 @@ func TestGroupQuotaManager_UpdateQuotaInternalAndRequest(t *testing.T) { // test1 request[120, 290] runtime == maxQuota request := createResourceList(120, 290*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test1", request) + gqm.updateGroupDeltaRequestNoLock("test1", request, request) runtime := gqm.RefreshRuntime("test1") assert.Equal(t, deltaRes, runtime) @@ -225,13 +225,13 @@ func TestGroupQuotaManager_UpdateQuotaDeltaRequest(t *testing.T) { // test1 request[120, 290] runtime == maxQuota request := createResourceList(120, 200*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test1", request) + gqm.updateGroupDeltaRequestNoLock("test1", request, request) runtime := gqm.RefreshRuntime("test1") assert.Equal(t, deltaRes, runtime) // test2 request[150, 210] runtime request = createResourceList(150, 210*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test2", request) + gqm.updateGroupDeltaRequestNoLock("test2", request, request) runtime = gqm.RefreshRuntime("test1") assert.Equal(t, createResourceList(53, 80*GigaByte), runtime) runtime = gqm.RefreshRuntime("test2") @@ -248,7 +248,7 @@ func TestGroupQuotaManager_NotAllowLentResource(t *testing.T) { AddQuotaToManager(t, gqm, "test2", extension.RootQuotaName, 96, 0, 40, 0, false, false) request := createResourceList(120, 0) - gqm.updateGroupDeltaRequestNoLock("test1", request) + gqm.updateGroupDeltaRequestNoLock("test1", request, request) runtime := gqm.RefreshRuntime("test1") assert.Equal(t, int64(60), runtime.Cpu().Value()) runtime = gqm.RefreshRuntime("test2") @@ -279,7 +279,7 @@ func TestGroupQuotaManager_NotAllowLentResource_2(t *testing.T) { // add 40 request request := createResourceList(40, 0) - gqm.updateGroupDeltaRequestNoLock("test-child1", request) + gqm.updateGroupDeltaRequestNoLock("test-child1", request, request) rootRuntime = gqm.RefreshRuntime("test-root") child1Runtime = gqm.RefreshRuntime("test-child1") @@ -291,7 +291,7 @@ func TestGroupQuotaManager_NotAllowLentResource_2(t *testing.T) { // add 20 request request2 := createResourceList(20, 0) - gqm.updateGroupDeltaRequestNoLock("test-child1", request2) + gqm.updateGroupDeltaRequestNoLock("test-child1", request2, request2) rootRuntime = gqm.RefreshRuntime("test-root") child1Runtime = gqm.RefreshRuntime("test-child1") @@ -326,7 +326,7 @@ func TestGroupQuotaManager_NotAllowLentResource_3(t *testing.T) { // add 40 request request := createResourceList(40, 0) - gqm.updateGroupDeltaRequestNoLock("test-child1", request) + gqm.updateGroupDeltaRequestNoLock("test-child1", request, request) rootRuntime = gqm.RefreshRuntime("test-root") child1Runtime = gqm.RefreshRuntime("test-child1") @@ -338,7 +338,7 @@ func TestGroupQuotaManager_NotAllowLentResource_3(t *testing.T) { // add 20 request request2 := createResourceList(20, 0) - gqm.updateGroupDeltaRequestNoLock("test-child1", request2) + gqm.updateGroupDeltaRequestNoLock("test-child1", request2, request2) rootRuntime = gqm.RefreshRuntime("test-root") child1Runtime = gqm.RefreshRuntime("test-child1") @@ -373,7 +373,7 @@ func TestGroupQuotaManager_NotAllowLentResource_4(t *testing.T) { // add 40 request request := createResourceList(40, 0) - gqm.updateGroupDeltaRequestNoLock("test-child1", request) + gqm.updateGroupDeltaRequestNoLock("test-child1", request, request) rootRuntime = gqm.RefreshRuntime("test-root") child1Runtime = gqm.RefreshRuntime("test-child1") @@ -385,7 +385,7 @@ func TestGroupQuotaManager_NotAllowLentResource_4(t *testing.T) { // add 20 request request2 := createResourceList(20, 0) - gqm.updateGroupDeltaRequestNoLock("test-child1", request2) + gqm.updateGroupDeltaRequestNoLock("test-child1", request2, request2) rootRuntime = gqm.RefreshRuntime("test-root") child1Runtime = gqm.RefreshRuntime("test-child1") @@ -411,19 +411,19 @@ func TestGroupQuotaManager_UpdateQuotaRequest(t *testing.T) { // 1. initial test1 request [60, 100] request := createResourceList(60, 100*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test1", request) + gqm.updateGroupDeltaRequestNoLock("test1", request, request) runtime := gqm.RefreshRuntime("test1") assert.Equal(t, request, runtime) // test1 request[120, 290] runtime == maxQuota newRequest := createResourceList(120, 200*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test1", newRequest) + gqm.updateGroupDeltaRequestNoLock("test1", request, newRequest) runtime = gqm.RefreshRuntime("test1") assert.Equal(t, deltaRes, runtime) // test2 request[150, 210] runtime request = createResourceList(150, 210*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test2", request) + gqm.updateGroupDeltaRequestNoLock("test2", request, request) runtime = gqm.RefreshRuntime("test1") assert.Equal(t, createResourceList(53, 80*GigaByte), runtime) runtime = gqm.RefreshRuntime("test2") @@ -503,7 +503,7 @@ func TestGroupQuotaManager_MultiUpdateQuotaRequest(t *testing.T) { AddQuotaToManager(t, gqm, "a-123", "test1-a", 96, 160*GigaByte, 50, 80*GigaByte, true, false) request := createResourceList(96, 130*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a-123", request) + gqm.updateGroupDeltaRequestNoLock("a-123", request, request) runtime := gqm.RefreshRuntime("a-123") assert.Equal(t, request, runtime) runtime = gqm.RefreshRuntime("test1-a") @@ -571,7 +571,7 @@ func TestGroupQuotaManager_MultiUpdateQuotaRequest2(t *testing.T) { // a-123 request[10,30] request < min request := createResourceList(10, 30*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a-123", request) + gqm.updateGroupDeltaRequestNoLock("a-123", request, request) runtime := gqm.RefreshRuntime("a-123") assert.Equal(t, request, runtime) runtime = gqm.RefreshRuntime("test1-a") @@ -581,7 +581,7 @@ func TestGroupQuotaManager_MultiUpdateQuotaRequest2(t *testing.T) { // a-123 add request[15,15] totalRequest[25,45] request > min request = createResourceList(15, 15*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a-123", request) + gqm.updateGroupDeltaRequestNoLock("a-123", request, request) runtime = gqm.RefreshRuntime("a-123") assert.Equal(t, createResourceList(25, 45*GigaByte), runtime) quotaInfo := gqm.GetQuotaInfoByName("test1-a") @@ -591,7 +591,7 @@ func TestGroupQuotaManager_MultiUpdateQuotaRequest2(t *testing.T) { // a-123 add request[30,30] totalRequest[55,75] request > max request = createResourceList(30, 30*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a-123", request) + gqm.updateGroupDeltaRequestNoLock("a-123", request, request) runtime = gqm.RefreshRuntime("a-123") assert.Equal(t, createResourceList(30, 60*GigaByte), runtime) quotaInfo = gqm.GetQuotaInfoByName("test1-a") @@ -612,9 +612,9 @@ func TestGroupQuotaManager_MultiUpdateQuotaRequest_WithScaledMinQuota1(t *testin AddQuotaToManager(t, gqm, "c", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) request := createResourceList(200, 200*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a", request) - gqm.updateGroupDeltaRequestNoLock("b", request) - gqm.updateGroupDeltaRequestNoLock("c", request) + gqm.updateGroupDeltaRequestNoLock("a", request, request) + gqm.updateGroupDeltaRequestNoLock("b", request, request) + gqm.updateGroupDeltaRequestNoLock("c", request, request) deltaRes := createResourceList(200, 200*GigaByte) gqm.UpdateClusterTotalResource(deltaRes) @@ -685,9 +685,9 @@ func TestGroupQuotaManager_MultiUpdateQuotaRequest_WithScaledMinQuota2(t *testin AddQuotaToManager(t, gqm, "c", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) request := createResourceList(200, 200*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a", request) - gqm.updateGroupDeltaRequestNoLock("b", createResourceList(0, 0)) - gqm.updateGroupDeltaRequestNoLock("c", request) + gqm.updateGroupDeltaRequestNoLock("a", request, request) + gqm.updateGroupDeltaRequestNoLock("b", createResourceList(0, 0), createResourceList(0, 0)) + gqm.updateGroupDeltaRequestNoLock("c", request, request) gqm.UpdateClusterTotalResource(createResourceList(199, 199*GigaByte)) runtime := gqm.RefreshRuntime("p") assert.Equal(t, createResourceList(200, 200*GigaByte), runtime) @@ -744,6 +744,34 @@ func TestGroupQuotaManager_MultiUpdateQuotaUsedAndNonPreemptibleUsed(t *testing. assert.Equal(t, nonPreemptibleUsed, quotaInfo.CalculateInfo.NonPreemptibleUsed) } +func TestGroupQuotaManager_MultiUpdateQuotaRequestAndNonPreemptibleRequest(t *testing.T) { + gqm := NewGroupQuotaManagerForTest() + + AddQuotaToManager(t, gqm, "test1", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "test1-sub1", "test1", 96, 160*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "test1-sub1-1", "test1-sub1", 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + request := createResourceList(120, 290*GigaByte) + nonPreemptibleRequest := createResourceList(50, 100*GigaByte) + limitedRequest := createResourceList(96, 160*GigaByte) + + gqm.updateGroupDeltaRequestNoLock("test1-sub1-1", request, nonPreemptibleRequest) + quotaInfo := gqm.GetQuotaInfoByName("test1-sub1-1") + assert.True(t, quotaInfo != nil) + assert.Equal(t, request, quotaInfo.CalculateInfo.Request) + assert.Equal(t, nonPreemptibleRequest, quotaInfo.CalculateInfo.NonPreemptibleRequest) + + quotaInfo = gqm.GetQuotaInfoByName("test1-sub1") + assert.True(t, quotaInfo != nil) + assert.Equal(t, limitedRequest, quotaInfo.CalculateInfo.Request) + assert.Equal(t, nonPreemptibleRequest, quotaInfo.CalculateInfo.NonPreemptibleRequest) + + quotaInfo = gqm.GetQuotaInfoByName("test1") + assert.True(t, quotaInfo != nil) + assert.Equal(t, limitedRequest, quotaInfo.CalculateInfo.Request) + assert.Equal(t, nonPreemptibleRequest, quotaInfo.CalculateInfo.NonPreemptibleRequest) +} + func TestGroupQuotaManager_UpdateQuotaParentName(t *testing.T) { gqm := NewGroupQuotaManagerForTest() @@ -767,7 +795,7 @@ func TestGroupQuotaManager_UpdateQuotaParentName(t *testing.T) { // a-123 request [60,100] request := createResourceList(60, 100*GigaByte) - gqm.updateGroupDeltaRequestNoLock("a-123", request) + gqm.updateGroupDeltaRequestNoLock("a-123", request, request) gqm.updateGroupDeltaUsedNoLock("a-123", request, createResourceList(0, 0)) runtime := gqm.RefreshRuntime("a-123") assert.Equal(t, request, runtime) @@ -780,7 +808,7 @@ func TestGroupQuotaManager_UpdateQuotaParentName(t *testing.T) { // test2-a request [20,40] request = createResourceList(20, 40*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test2-a", request) + gqm.updateGroupDeltaRequestNoLock("test2-a", request, request) gqm.updateGroupDeltaUsedNoLock("test2-a", request, createResourceList(0, 0)) runtime = gqm.RefreshRuntime("test2-a") assert.Equal(t, request, runtime) @@ -967,7 +995,7 @@ func TestGroupQuotaManager_MultiChildMaxGreaterParentMax_MaxGreaterThanTotalRes( // test1 Request [500, 4096] limitRequest [500, 2048] // test1-sub Request [500,2048] limitedRequest [500, 2048] limited by rootRes [300, 8000] -> [300,2048] request := createResourceList(500, 4096) - gqm.updateGroupDeltaRequestNoLock("test1-sub1", request) + gqm.updateGroupDeltaRequestNoLock("test1-sub1", request, request) runtime := gqm.RefreshRuntime("test1-sub1") assert.Equal(t, createResourceList(300, 2048), runtime) fmt.Printf("quota1 runtime:%v\n", runtime) @@ -975,7 +1003,7 @@ func TestGroupQuotaManager_MultiChildMaxGreaterParentMax_MaxGreaterThanTotalRes( // test1 Request [1050, 8192] limitRequest [500, 2048] // test1-sub1 Request [500,2048] limitedRequest [500, 2048] limited by rootRes [300, 8000] -> [300,2048] request = createResourceList(550, 4096) - gqm.updateGroupDeltaRequestNoLock("test1-sub1", request) + gqm.updateGroupDeltaRequestNoLock("test1-sub1", request, request) runtime = gqm.RefreshRuntime("test1-sub") fmt.Printf("quota1 runtime:%v\n", runtime) @@ -1006,7 +1034,7 @@ func TestGroupQuotaManager_MultiChildMaxGreaterParentMax(t *testing.T) { // test1-sub1 Request[400, 1500] limitedRequest [400, 1500] // test1 Request [400,1500] limitedRequest [300, 1024] request := createResourceList(400, 1500*GigaByte) - gqm.updateGroupDeltaRequestNoLock("test1-sub1", request) + gqm.updateGroupDeltaRequestNoLock("test1-sub1", request, request) quotaInfo := gqm.GetQuotaInfoByName("test1") assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(400, 1500*GigaByte)) quotaInfo = gqm.GetQuotaInfoByName("test1-sub1") @@ -1018,7 +1046,7 @@ func TestGroupQuotaManager_MultiChildMaxGreaterParentMax(t *testing.T) { // test1 max < test1-sub1 max < request // test1-sub1 Request[800, 3000] limitedRequest [500, 2048] // test1 Request [500, 2048] limitedRequest [300, 1024] - gqm.updateGroupDeltaRequestNoLock("test1-sub1", request) + gqm.updateGroupDeltaRequestNoLock("test1-sub1", request, request) runtime = gqm.RefreshRuntime("test1-sub1") assert.Equal(t, createResourceList(300, 1024*GigaByte), runtime) fmt.Printf("quota1 runtime:%v\n", runtime) @@ -1062,8 +1090,8 @@ func TestGroupQuotaManager_RefreshAndGetRuntimeQuota_UpdateQuota(t *testing.T) { // case2: no existed group assert.Nil(t, gqm.RefreshRuntime("5")) - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(5, 5)) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(5, 5)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(5, 5), createResourceList(5, 5)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(5, 5), createResourceList(5, 5)) gq1 := gqm.GetQuotaInfoByName("1") gq2 := gqm.GetQuotaInfoByName("2") @@ -1080,8 +1108,8 @@ func TestGroupQuotaManager_RefreshAndGetRuntimeQuota_UpdateQuota(t *testing.T) { assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(5, 5)) // case5: request is larger than min - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(25, 25)) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(25, 25)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(25, 25), createResourceList(25, 25)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(25, 25), createResourceList(25, 25)) // 1 min [10,10] -> toPartitionRes [40,40] -> runtime [30,30] // 2 min [0,0] -> toPartitionRes [40,40] -> runtime [20,20] assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(30, 30)) @@ -1129,8 +1157,8 @@ func TestGroupQuotaManager_UpdateOneGroupMaxQuota_UpdateQuota(t *testing.T) { gqm.UpdateQuota(qi2, false) // case1: min < req < max - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(35, 35)) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(35, 35)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(35, 35), createResourceList(35, 35)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(35, 35), createResourceList(35, 35)) assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(25, 25)) assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(25, 25)) assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(35, 35)) @@ -1169,8 +1197,8 @@ func TestGroupQuotaManager_UpdateOneGroupMinQuota_UpdateQuota(t *testing.T) { gqm.UpdateQuota(qi2, false) // case1: min < req < max - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(15, 15)) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(15, 15)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(15, 15), createResourceList(15, 15)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(15, 15), createResourceList(15, 15)) assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(15, 15)) assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(15, 15)) assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) @@ -1215,8 +1243,8 @@ func TestGroupQuotaManager_UpdateOneGroupMinQuota_UpdateQuota(t *testing.T) { qi2.Spec.Min = createResourceList(5, 5) gqm.UpdateQuota(qi1, false) gqm.UpdateQuota(qi2, false) - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(100, 100)) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(100, 100)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(100, 100), createResourceList(100, 100)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(100, 100), createResourceList(100, 100)) assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(40, 40)) assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(10, 10)) assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) @@ -1235,8 +1263,8 @@ func TestGroupQuotaManager_DeleteOneGroup_UpdateQuota(t *testing.T) { qi2 := createQuota("2", extension.RootQuotaName, 40, 40, 10, 10) gqm.UpdateQuota(qi1, false) gqm.UpdateQuota(qi2, false) - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(15, 15)) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(15, 15)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(15, 15), createResourceList(15, 15)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(15, 15), createResourceList(15, 15)) // delete one group gqm.UpdateQuota(q1, true) @@ -1277,7 +1305,7 @@ func BenchmarkGroupQuotaManager_RefreshRuntime(b *testing.B) { request := createResourceList(reqCpu, reqMem) totalReqMem += float64(reqMem) totalReqCpu += float64(reqCpu) - gqm.updateGroupDeltaRequestNoLock(fmt.Sprintf("%v", j), request) + gqm.updateGroupDeltaRequestNoLock(fmt.Sprintf("%v", j), request, request) } totalRes := createResourceList(int64(totalReqCpu/1.5), int64(totalReqMem/1.5)) gqm.UpdateClusterTotalResource(totalRes) @@ -1339,14 +1367,21 @@ func TestGroupQuotaManager_UpdatePodCache_UpdatePodIsAssigned_GetPodIsAssigned_U gqm.updatePodIsAssignedNoLock("2", pod1, true) assert.True(t, gqm.getPodIsAssignedNoLock("2", pod1)) + pod1.Labels = map[string]string{ + extension.LabelPreemptible: "false", + } gqm.updatePodRequestNoLock("1", nil, pod1) assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("1").GetRequest()) + assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("1").GetNonPreemptibleRequest()) gqm.updatePodUsedNoLock("2", nil, pod1) assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("2").GetUsed()) + assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("2").GetNonPreemptibleUsed()) gqm.updatePodRequestNoLock("1", pod1, nil) assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetRequest()) + assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetNonPreemptibleRequest()) gqm.updatePodUsedNoLock("2", pod1, nil) assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("2").GetUsed()) + assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("2").GetNonPreemptibleUsed()) } func TestGroupQuotaManager_OnPodUpdate(t *testing.T) { @@ -1436,10 +1471,10 @@ func TestGroupQuotaManager_GetQuotaInformationForSyncHandler(t *testing.T) { qi1 := createQuota("1", extension.RootQuotaName, 400, 400, 10, 10) gqm.UpdateQuota(qi1, false) gqm.UpdateClusterTotalResource(createResourceList(1000, 1000)) - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(100, 100)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(100, 100), createResourceList(100, 100)) gqm.RefreshRuntime("1") gqm.updateGroupDeltaUsedNoLock("1", createResourceList(10, 10), createResourceList(0, 0)) - used, request, childRequest, runtime, _, _, err := gqm.GetQuotaInformationForSyncHandler("1") + used, request, childRequest, runtime, _, _, _, _, err := gqm.GetQuotaInformationForSyncHandler("1") assert.Nil(t, err) assert.Equal(t, used, createResourceList(10, 10)) assert.Equal(t, request, createResourceList(100, 100)) @@ -1454,10 +1489,10 @@ func TestGroupQuotaManager_GetQuotaInformationForSyncHandlerWithUsageGuarantee(t qi1 := createQuota("1", extension.RootQuotaName, 400, 400, 10, 10) gqm.UpdateQuota(qi1, false) gqm.UpdateClusterTotalResource(createResourceList(1000, 1000)) - gqm.updateGroupDeltaRequestNoLock("1", createResourceList(100, 100)) + gqm.updateGroupDeltaRequestNoLock("1", createResourceList(100, 100), createResourceList(100, 100)) gqm.RefreshRuntime("1") gqm.updateGroupDeltaUsedNoLock("1", createResourceList(10, 10), createResourceList(0, 0)) - used, request, childRequest, runtime, guaranteed, allocated, err := gqm.GetQuotaInformationForSyncHandler("1") + used, request, childRequest, runtime, guaranteed, allocated, nonPreemptibleRequest, nonPreemptibleUsed, err := gqm.GetQuotaInformationForSyncHandler("1") assert.Nil(t, err) assert.Equal(t, used, createResourceList(10, 10)) assert.Equal(t, request, createResourceList(100, 100)) @@ -1465,6 +1500,8 @@ func TestGroupQuotaManager_GetQuotaInformationForSyncHandlerWithUsageGuarantee(t assert.Equal(t, runtime, createResourceList(100, 100)) assert.Equal(t, guaranteed, createResourceList(10, 10)) assert.Equal(t, allocated, createResourceList(10, 10)) + assert.Equal(t, nonPreemptibleRequest, createResourceList(100, 100)) + assert.Equal(t, nonPreemptibleUsed, createResourceList(0, 0)) } func TestGetPodName(t *testing.T) { @@ -1497,23 +1534,31 @@ func TestGroupQuotaManager_IsParent(t *testing.T) { func TestGroupQuotaManager_UpdateRootQuotaUsed(t *testing.T) { expectedTotalUsed := createResourceList(0, 0) + expectedTotalNonpreemptibleRequest := createResourceList(0, 0) gqm := NewGroupQuotaManagerForTest() gqm.UpdateClusterTotalResource(createResourceList(1000, 1000)) sysUsed := createResourceList(10, 30) expectedTotalUsed = quotav1.Add(expectedTotalUsed, sysUsed) + expectedTotalNonpreemptibleRequest = quotav1.Add(expectedTotalNonpreemptibleRequest, sysUsed) gqm.updateGroupDeltaUsedNoLock(extension.SystemQuotaName, sysUsed, createResourceList(0, 0)) + gqm.updateGroupDeltaRequestNoLock(extension.SystemQuotaName, sysUsed, sysUsed) assert.Equal(t, sysUsed, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetUsed()) + assert.Equal(t, sysUsed, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetNonPreemptibleRequest()) defaultUsed := createResourceList(2, 5) expectedTotalUsed = quotav1.Add(expectedTotalUsed, defaultUsed) + expectedTotalNonpreemptibleRequest = quotav1.Add(expectedTotalNonpreemptibleRequest, defaultUsed) gqm.updateGroupDeltaUsedNoLock(extension.DefaultQuotaName, defaultUsed, createResourceList(0, 0)) + gqm.updateGroupDeltaRequestNoLock(extension.DefaultQuotaName, defaultUsed, defaultUsed) assert.Equal(t, defaultUsed, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetUsed()) + assert.Equal(t, defaultUsed, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetNonPreemptibleRequest()) //case1: no quota, root quota used rootQuotaUsed := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName).GetUsed() sysAndDefaultUsed := quotav1.Add(sysUsed, defaultUsed) assert.Equal(t, rootQuotaUsed, sysAndDefaultUsed) + assert.Equal(t, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetNonPreemptibleRequest(), sysAndDefaultUsed) //case2: just pod no quota, root quota used pod1 := schetesting.MakePod().Name("1").Obj() @@ -1526,12 +1571,16 @@ func TestGroupQuotaManager_UpdateRootQuotaUsed(t *testing.T) { } pod1Used := pod1.Spec.Containers[0].Resources.Requests expectedTotalUsed = quotav1.Add(expectedTotalUsed, pod1Used) + pod1.Labels = map[string]string{extension.LabelPreemptible: "false"} + expectedTotalNonpreemptibleRequest = quotav1.Add(expectedTotalNonpreemptibleRequest, pod1Used) gqm.updatePodCacheNoLock(extension.DefaultQuotaName, pod1, true) assert.Equal(t, 1, len(gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetPodCache())) gqm.updatePodIsAssignedNoLock(extension.DefaultQuotaName, pod1, true) assert.True(t, gqm.getPodIsAssignedNoLock(extension.DefaultQuotaName, pod1)) gqm.updatePodUsedNoLock(extension.DefaultQuotaName, nil, pod1) + gqm.updatePodRequestNoLock(extension.DefaultQuotaName, nil, pod1) assert.Equal(t, expectedTotalUsed, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetUsed()) + assert.Equal(t, expectedTotalNonpreemptibleRequest, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetNonPreemptibleRequest()) // case3: when build quota tree, root quota used qi1 := createQuota("1", extension.RootQuotaName, 20, 20, 10, 10) @@ -1542,12 +1591,18 @@ func TestGroupQuotaManager_UpdateRootQuotaUsed(t *testing.T) { gqm.UpdateQuota(qi3, false) gqm.updateGroupDeltaUsedNoLock("2", createResourceList(5, 5), createResourceList(0, 0)) gqm.updateGroupDeltaUsedNoLock("3", createResourceList(7, 5), createResourceList(0, 0)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(0, 0), createResourceList(5, 5)) + gqm.updateGroupDeltaRequestNoLock("3", createResourceList(0, 0), createResourceList(7, 5)) expectedTotalUsed = quotav1.Add(expectedTotalUsed, createResourceList(5, 5)) expectedTotalUsed = quotav1.Add(expectedTotalUsed, createResourceList(7, 5)) + expectedTotalNonpreemptibleRequest = quotav1.Add(expectedTotalNonpreemptibleRequest, createResourceList(5, 5)) + expectedTotalNonpreemptibleRequest = quotav1.Add(expectedTotalNonpreemptibleRequest, createResourceList(7, 5)) + rootQuotaUsed = gqm.GetQuotaInfoByName(extension.RootQuotaName).GetUsed() assert.Equal(t, expectedTotalUsed, rootQuotaUsed) + assert.Equal(t, expectedTotalNonpreemptibleRequest, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetNonPreemptibleRequest()) } func TestGroupQuotaManager_ChildRequestAndRequest_All_not_allowLent(t *testing.T) { @@ -1631,12 +1686,12 @@ func TestGroupQuotaManager_UpdateRootQuotaRequest(t *testing.T) { sysRequest := createResourceList(10, 30) expectedTotalRequest = quotav1.Add(expectedTotalRequest, sysRequest) - gqm.updateGroupDeltaRequestNoLock(extension.SystemQuotaName, sysRequest) + gqm.updateGroupDeltaRequestNoLock(extension.SystemQuotaName, sysRequest, sysRequest) assert.Equal(t, sysRequest, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetRequest()) defaultRequest := createResourceList(2, 5) expectedTotalRequest = quotav1.Add(expectedTotalRequest, defaultRequest) - gqm.updateGroupDeltaRequestNoLock(extension.DefaultQuotaName, defaultRequest) + gqm.updateGroupDeltaRequestNoLock(extension.DefaultQuotaName, defaultRequest, defaultRequest) assert.Equal(t, defaultRequest, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetRequest()) //case1: no other quota, root quota request @@ -1669,8 +1724,8 @@ func TestGroupQuotaManager_UpdateRootQuotaRequest(t *testing.T) { gqm.UpdateQuota(qi1, false) gqm.UpdateQuota(qi2, false) gqm.UpdateQuota(qi3, false) - gqm.updateGroupDeltaRequestNoLock("2", createResourceList(5, 5)) - gqm.updateGroupDeltaRequestNoLock("3", createResourceList(7, 5)) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(5, 5), createResourceList(5, 5)) + gqm.updateGroupDeltaRequestNoLock("3", createResourceList(7, 5), createResourceList(7, 5)) expectedTotalRequest = quotav1.Add(expectedTotalRequest, createResourceList(5, 5)) expectedTotalRequest = quotav1.Add(expectedTotalRequest, createResourceList(7, 5)) diff --git a/pkg/scheduler/plugins/elasticquota/core/quota_info.go b/pkg/scheduler/plugins/elasticquota/core/quota_info.go index cf8106520..12b6b8105 100644 --- a/pkg/scheduler/plugins/elasticquota/core/quota_info.go +++ b/pkg/scheduler/plugins/elasticquota/core/quota_info.go @@ -45,6 +45,8 @@ type QuotaCalculateInfo struct { NonPreemptibleUsed v1.ResourceList // All pods request Request v1.ResourceList + // All non-preemptible pods request + NonPreemptibleRequest v1.ResourceList // ChildRquest is the sum of child quota requests. // If the quota is leaf, it's the sum of pods requests ChildRequest v1.ResourceList @@ -86,17 +88,18 @@ func NewQuotaInfo(isParent, allowLentResource bool, name, parentName string) *Qu RuntimeVersion: 0, PodCache: make(map[string]*PodInfo), CalculateInfo: QuotaCalculateInfo{ - Max: v1.ResourceList{}, - AutoScaleMin: v1.ResourceList{}, - Min: v1.ResourceList{}, - Used: v1.ResourceList{}, - NonPreemptibleUsed: v1.ResourceList{}, - Request: v1.ResourceList{}, - SharedWeight: v1.ResourceList{}, - Runtime: v1.ResourceList{}, - ChildRequest: v1.ResourceList{}, - Guaranteed: v1.ResourceList{}, - Allocated: v1.ResourceList{}, + Max: v1.ResourceList{}, + AutoScaleMin: v1.ResourceList{}, + Min: v1.ResourceList{}, + Used: v1.ResourceList{}, + NonPreemptibleUsed: v1.ResourceList{}, + Request: v1.ResourceList{}, + NonPreemptibleRequest: v1.ResourceList{}, + SharedWeight: v1.ResourceList{}, + Runtime: v1.ResourceList{}, + ChildRequest: v1.ResourceList{}, + Guaranteed: v1.ResourceList{}, + Allocated: v1.ResourceList{}, }, } } @@ -116,17 +119,18 @@ func (qi *QuotaInfo) DeepCopy() *QuotaInfo { RuntimeVersion: qi.RuntimeVersion, PodCache: make(map[string]*PodInfo), CalculateInfo: QuotaCalculateInfo{ - Max: qi.CalculateInfo.Max.DeepCopy(), - AutoScaleMin: qi.CalculateInfo.AutoScaleMin.DeepCopy(), - Min: qi.CalculateInfo.Min.DeepCopy(), - Used: qi.CalculateInfo.Used.DeepCopy(), - NonPreemptibleUsed: qi.CalculateInfo.NonPreemptibleUsed.DeepCopy(), - Request: qi.CalculateInfo.Request.DeepCopy(), - SharedWeight: qi.CalculateInfo.SharedWeight.DeepCopy(), - Runtime: qi.CalculateInfo.Runtime.DeepCopy(), - ChildRequest: qi.CalculateInfo.ChildRequest.DeepCopy(), - Guaranteed: qi.CalculateInfo.Guaranteed.DeepCopy(), - Allocated: qi.CalculateInfo.Allocated.DeepCopy(), + Max: qi.CalculateInfo.Max.DeepCopy(), + AutoScaleMin: qi.CalculateInfo.AutoScaleMin.DeepCopy(), + Min: qi.CalculateInfo.Min.DeepCopy(), + Used: qi.CalculateInfo.Used.DeepCopy(), + NonPreemptibleUsed: qi.CalculateInfo.NonPreemptibleUsed.DeepCopy(), + Request: qi.CalculateInfo.Request.DeepCopy(), + NonPreemptibleRequest: qi.CalculateInfo.NonPreemptibleRequest.DeepCopy(), + SharedWeight: qi.CalculateInfo.SharedWeight.DeepCopy(), + Runtime: qi.CalculateInfo.Runtime.DeepCopy(), + ChildRequest: qi.CalculateInfo.ChildRequest.DeepCopy(), + Guaranteed: qi.CalculateInfo.Guaranteed.DeepCopy(), + Allocated: qi.CalculateInfo.Allocated.DeepCopy(), }, } for name, pod := range qi.PodCache { @@ -151,6 +155,7 @@ func (qi *QuotaInfo) GetQuotaSummary() *QuotaInfoSummary { quotaInfoSummary.Used = qi.CalculateInfo.Used.DeepCopy() quotaInfoSummary.NonPreemptibleUsed = qi.CalculateInfo.NonPreemptibleUsed.DeepCopy() quotaInfoSummary.Request = qi.CalculateInfo.Request.DeepCopy() + quotaInfoSummary.NonPreemptibleRequest = qi.CalculateInfo.NonPreemptibleRequest.DeepCopy() quotaInfoSummary.SharedWeight = qi.CalculateInfo.SharedWeight.DeepCopy() quotaInfoSummary.Runtime = qi.CalculateInfo.Runtime.DeepCopy() quotaInfoSummary.ChildRequest = qi.CalculateInfo.ChildRequest.DeepCopy() @@ -211,11 +216,15 @@ func (qi *QuotaInfo) setMinNoLock(min v1.ResourceList) { qi.CalculateInfo.Min = min.DeepCopy() } -func (qi *QuotaInfo) addRequestNonNegativeNoLock(delta v1.ResourceList) { +func (qi *QuotaInfo) addRequestNonNegativeNoLock(delta, deltaNonPreemptibleRequest v1.ResourceList) { qi.CalculateInfo.Request = quotav1.Add(qi.CalculateInfo.Request, delta) for _, resName := range quotav1.IsNegative(qi.CalculateInfo.Request) { qi.CalculateInfo.Request[resName] = createQuantity(0, resName) } + qi.CalculateInfo.NonPreemptibleRequest = quotav1.Add(qi.CalculateInfo.NonPreemptibleRequest, deltaNonPreemptibleRequest) + for _, resName := range quotav1.IsNegative(qi.CalculateInfo.NonPreemptibleRequest) { + qi.CalculateInfo.NonPreemptibleRequest[resName] = createQuantity(0, resName) + } } func (qi *QuotaInfo) addChildRequestNonNegativeNoLock(delta v1.ResourceList) { @@ -295,6 +304,12 @@ func (qi *QuotaInfo) GetNonPreemptibleUsed() v1.ResourceList { return qi.CalculateInfo.NonPreemptibleUsed.DeepCopy() } +func (qi *QuotaInfo) GetNonPreemptibleRequest() v1.ResourceList { + qi.lock.Lock() + defer qi.lock.Unlock() + return qi.CalculateInfo.NonPreemptibleRequest.DeepCopy() +} + func (qi *QuotaInfo) GetRuntime() v1.ResourceList { qi.lock.Lock() defer qi.lock.Unlock() @@ -337,6 +352,7 @@ func (qi *QuotaInfo) getMaskedRuntimeNoLock() v1.ResourceList { func (qi *QuotaInfo) clearForResetNoLock() { qi.CalculateInfo.Request = v1.ResourceList{} + qi.CalculateInfo.NonPreemptibleRequest = v1.ResourceList{} qi.CalculateInfo.Used = v1.ResourceList{} qi.CalculateInfo.NonPreemptibleUsed = v1.ResourceList{} qi.CalculateInfo.Runtime = v1.ResourceList{} diff --git a/pkg/scheduler/plugins/elasticquota/core/quota_summary.go b/pkg/scheduler/plugins/elasticquota/core/quota_summary.go index 90280a1f8..9941fc100 100644 --- a/pkg/scheduler/plugins/elasticquota/core/quota_summary.go +++ b/pkg/scheduler/plugins/elasticquota/core/quota_summary.go @@ -33,17 +33,18 @@ type QuotaInfoSummary struct { AllowLentResource bool `json:"allowLentResource"` Tree string `json:"tree"` - Max v1.ResourceList `json:"max"` - Min v1.ResourceList `json:"min"` - AutoScaleMin v1.ResourceList `json:"autoScaleMin"` - Used v1.ResourceList `json:"used"` - NonPreemptibleUsed v1.ResourceList `json:"nonPreemptibleUsed"` - Request v1.ResourceList `json:"request"` - SharedWeight v1.ResourceList `json:"sharedWeight"` - Runtime v1.ResourceList `json:"runtime"` - ChildRequest v1.ResourceList `json:"childRequest"` - Allocated v1.ResourceList `json:"allocated"` - Guaranteed v1.ResourceList `json:"guaranteed"` + Max v1.ResourceList `json:"max"` + Min v1.ResourceList `json:"min"` + AutoScaleMin v1.ResourceList `json:"autoScaleMin"` + Used v1.ResourceList `json:"used"` + NonPreemptibleUsed v1.ResourceList `json:"nonPreemptibleUsed"` + Request v1.ResourceList `json:"request"` + NonPreemptibleRequest v1.ResourceList `json:"nonPreemptibleRequest"` + SharedWeight v1.ResourceList `json:"sharedWeight"` + Runtime v1.ResourceList `json:"runtime"` + ChildRequest v1.ResourceList `json:"childRequest"` + Allocated v1.ResourceList `json:"allocated"` + Guaranteed v1.ResourceList `json:"guaranteed"` PodCache map[string]*SimplePodInfo `json:"podCache"` } diff --git a/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go index 7e8a6d305..ed6526a77 100644 --- a/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go +++ b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go @@ -45,7 +45,7 @@ func TestQuotaInfo_GetLimitRequest(t *testing.T) { assertObj.Equal(*resource.NewQuantity(1000, resource.BinarySI), quotaInfo.getLimitRequestNoLock()[corev1.ResourceMemory]) req2 := createResourceList(100, 1000) - quotaInfo.addRequestNonNegativeNoLock(req2) + quotaInfo.addRequestNonNegativeNoLock(req2, req2) assertObj.Equal(*resource.NewQuantity(2000, resource.BinarySI), quotaInfo.getLimitRequestNoLock()[corev1.ResourceMemory]) } @@ -57,7 +57,7 @@ func TestQuotaInfo_AddRequestNonNegativeNoLock(t *testing.T) { Used: createResourceList(40, 40), }, } - quotaInfo.addRequestNonNegativeNoLock(req1) + quotaInfo.addRequestNonNegativeNoLock(req1, req1) quotaInfo.addUsedNonNegativeNoLock(req1, createResourceList(0, 0)) assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(0, 0)) assert.Equal(t, quotaInfo.CalculateInfo.Used, createResourceList(0, 0)) @@ -217,7 +217,7 @@ func TestRuntimeQuotaCalculator_UpdateOneGroupMaxQuota(t *testing.T) { newMax := createResourceList(200, 9000) request := createResourceList(30, 3000) - quotaInfo.addRequestNonNegativeNoLock(request) + quotaInfo.addRequestNonNegativeNoLock(request, request) assert.Equal(t, request, quotaInfo.CalculateInfo.Request) qtw.setClusterTotalResource(max) @@ -309,7 +309,7 @@ func TestRuntimeQuotaCalculator_UpdateOneGroupRequest(t *testing.T) { request := createResourceList(int64(i*10), int64(i*1000)) quotaName := fmt.Sprintf("test-%d", i) quotaInfo := createQuotaInfoWithRes(quotaName, max, min) - quotaInfo.addRequestNonNegativeNoLock(request) + quotaInfo.addRequestNonNegativeNoLock(request, request) qtw.updateOneGroupMaxQuota(quotaInfo) qtw.updateOneGroupMinQuota(quotaInfo)