From 32a301ad8401088439f869dbf2faed9dd72676ce Mon Sep 17 00:00:00 2001 From: Abhilash Shetty Date: Sun, 16 Feb 2025 20:30:01 +0000 Subject: [PATCH 1/2] test(ci): add volume provisioning test on cordoned node Signed-off-by: Abhilash Shetty --- tests/lvm_utils.go | 1 - tests/provision_test.go | 17 ++++++++ tests/utils.go | 95 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 3 deletions(-) diff --git a/tests/lvm_utils.go b/tests/lvm_utils.go index bab213d0..85fce9d8 100644 --- a/tests/lvm_utils.go +++ b/tests/lvm_utils.go @@ -79,7 +79,6 @@ func vgEmpty(name string) bool { lvs, _, _ := execAtLocal("sudo", nil, args_lvs...) lvs_str := strings.TrimSpace(string(lvs)) lv_cnt, _ := strconv.Atoi(lvs_str) - fmt.Printf("lvs cnt is %d\n", lv_cnt) if lv_cnt != 0 { return false } else { diff --git a/tests/provision_test.go b/tests/provision_test.go index 9053455b..87834127 100644 --- a/tests/provision_test.go +++ b/tests/provision_test.go @@ -149,6 +149,22 @@ func vgPatternMatchPresentTest() { By("Deleting storage class", deleteStorageClass) } +func scheduleOnCordonedNodeTest() { + device := setupVg(20, "lvmvg") + defer cleanupVg(device, "lvmvg") + By("Creating storage class", createStorageClass) + By("Cordoning the node", cordonk8sNode) + By("Creating and verifying PVC Bound status. It should not be Bound") + createAndVerifyPVC(false) + By("Uncordon the node", uncordonk8sNode) + By("Verify the PVC gets Bound") + verifyPVC(pvcName, true) + deleteAndVerifyPVC(pvcName) + By("Verifying that PV doesnt exists after PVC deletion") + verifyPVForPVC(false, pvcName) + By("Deleting storage class", deleteStorageClass) +} + func vgPatternNoMatchPresentTest() { device := setupVg(20, "lvmvg212") device_1 := setupVg(20, "lvmvg") @@ -316,6 +332,7 @@ func schedulingTest() { By("###Running vg specified in sc not present test###", vgSpecifiedNotPresentTest) By("###Running lvmnode has vg matching vgpattern test###", vgPatternMatchPresentTest) By("###Running lvmnode doesnt have vg matching vgpattern test###", vgPatternNoMatchPresentTest) + By("###Running volume schedule on Cordoned node test###", scheduleOnCordonedNodeTest) } func capacityTest() { diff --git a/tests/utils.go b/tests/utils.go index 033f4fdc..ca5b8826 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -3,6 +3,7 @@ package tests import ( "context" "fmt" + "path/filepath" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -12,6 +13,9 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" "github.com/openebs/lvm-localpv/pkg/lvm" "github.com/openebs/lvm-localpv/tests/container" @@ -249,14 +253,12 @@ func VerifyLVMVolume(expect_ready bool, expected_vg string) { // It gets deleted, by the csi provisioner only when the owner node of cr marks is // as Failed. So incase, we do a get of cr when the cr was being handled then we expect // state to be either Pending or Failed. - fmt.Printf("checking vol object as vol is non nil, vol is %v\n", vol) gomega.Expect(vol.Status.State).To(gomega.Or(gomega.Equal("Pending"), gomega.Equal("Failed")), "While checking if lvmvolume: %s is in Pending or Failed state", pvcObj.Spec.VolumeName) } } else { gomega.Expect(err).To(gomega.BeNil(), "while fetching the lvm volume {%s}", pvcObj.Spec.VolumeName) if expected_vg != "" { - fmt.Printf("vol is %v\n", vol) gomega.Expect(vol.Spec.VolGroup).To(gomega.Equal(expected_vg), "while checking volume group of lvm volume", pvcObj.Spec.VolumeName) } else { @@ -320,6 +322,18 @@ func createAndVerifyPVC(expect_bound bool) { ) } +// Verifies state of already created pvc based on expect_bound. +func verifyPVC(pvc_name string, expect_bound bool) { + ok := false + if !expect_bound { + ok = IsPVCPendingConsistently(pvc_name) + } else { + ok = IsPVCBoundEventually(pvc_name) + } + gomega.Expect(ok).To(gomega.Equal(true), + "while checking the pvc status") +} + func createAndVerifyBlockPVC(expect_bound bool) { var ( err error @@ -819,3 +833,80 @@ func createNodeDaemonSet(ds *appsv1.DaemonSet) { gomega.BeNil(), "creating node plugin daemonset %v", nodeDaemonSet) } + +// Creates a k8s client using kubeconfig path. +func getk8sClient() (client *kubernetes.Clientset) { + kubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config") + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + gomega.Expect(err).To( + gomega.BeNil(), + "Could not created a k8s client", + ) + client, c_err := kubernetes.NewForConfig(config) + gomega.Expect(c_err).To( + gomega.BeNil(), + "Could not created a k8s client", + ) + return client +} + +// Lists k8s nodes. +func listNodes(client *kubernetes.Clientset) (nodes *corev1.NodeList) { + nodes, n_err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + gomega.Expect(n_err).To( + gomega.BeNil(), + "Could not list nodes", + ) + return nodes +} + +// Cordons all k8s nodes. +func cordonk8sNode() { + client := getk8sClient() + nodes := listNodes(client) + for _, node := range nodes.Items { + if !node.Spec.Unschedulable { + c_err := cordonNode(client, &node) + gomega.Expect(c_err).To( + gomega.BeNil(), + "Could not cordon node", + ) + } + + } +} + +// UnCordons all k8s nodes. +func uncordonk8sNode() { + client := getk8sClient() + nodes := listNodes(client) + for _, node := range nodes.Items { + if node.Spec.Unschedulable { + c_err := uncordonNode(client, &node) + gomega.Expect(c_err).To( + gomega.BeNil(), + "Could not uncordon node", + ) + } + } +} + +// Adds cordon taint to a specific node. +func cordonNode(clientset *kubernetes.Clientset, node *corev1.Node) error { + updatedNode := node.DeepCopy() + updatedNode.Spec.Unschedulable = true + + _, err := clientset.CoreV1().Nodes().Update(context.TODO(), updatedNode, metav1.UpdateOptions{}) + return err + +} + +// Removes cordon taint from a specific node. +func uncordonNode(clientset *kubernetes.Clientset, node *corev1.Node) error { + updatedNode := node.DeepCopy() + updatedNode.Spec.Unschedulable = false + + _, err := clientset.CoreV1().Nodes().Update(context.TODO(), updatedNode, metav1.UpdateOptions{}) + return err + +} From 26a721a29fb35027a25dcd0a4c0475e651143932 Mon Sep 17 00:00:00 2001 From: Abhilash Shetty Date: Tue, 25 Feb 2025 07:57:19 +0000 Subject: [PATCH 2/2] fix(scheduler): update lib-csi pkg to exclude cordoned nodes Signed-off-by: Abhilash Shetty --- go.mod | 2 +- go.sum | 4 ++-- tests/lvm_utils.go | 1 - tests/provision_test.go | 2 +- tests/utils.go | 12 ++---------- 5 files changed, 6 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index bead9cac..7c6bc94a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.27.4 github.com/openebs/google-analytics-4 v0.3.0 - github.com/openebs/lib-csi v0.8.2 + github.com/openebs/lib-csi v0.9.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/spf13/cobra v1.6.0 diff --git a/go.sum b/go.sum index 1087f15d..37055dc1 100644 --- a/go.sum +++ b/go.sum @@ -314,8 +314,8 @@ github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/openebs/google-analytics-4 v0.3.0 h1:rAv9BC476ennyGPWAy4D3vU7TlxEKiynbLUP3VgeBUc= github.com/openebs/google-analytics-4 v0.3.0/go.mod h1:lKjvRs6HAYOlOTYjtOUp35iTutoXzRL0hOeVAvLAfZI= -github.com/openebs/lib-csi v0.8.2 h1:HmoiZX3VXFPglwqnRPnRus7K58ixDWBa19OpPZGk2Ws= -github.com/openebs/lib-csi v0.8.2/go.mod h1:4yc0Q1thH+oU80z73zGELfrOw2yeLdLNIRmcrxBxsBc= +github.com/openebs/lib-csi v0.9.0 h1:Yro5CN0cT3bRYOB1IdGPKXnBcPzxZnkxd4MLREAtFME= +github.com/openebs/lib-csi v0.9.0/go.mod h1:4yc0Q1thH+oU80z73zGELfrOw2yeLdLNIRmcrxBxsBc= github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/tests/lvm_utils.go b/tests/lvm_utils.go index 85fce9d8..5b8eec1f 100644 --- a/tests/lvm_utils.go +++ b/tests/lvm_utils.go @@ -159,7 +159,6 @@ func removeVg(name string) { if current_retry < retries { vg_empty := vgEmpty(name) if vg_empty { - fmt.Printf("No lv in vg before vg remove\n") break } else { fmt.Printf("lv in vg during retry %d\n", current_retry) diff --git a/tests/provision_test.go b/tests/provision_test.go index 87834127..8723027c 100644 --- a/tests/provision_test.go +++ b/tests/provision_test.go @@ -158,7 +158,7 @@ func scheduleOnCordonedNodeTest() { createAndVerifyPVC(false) By("Uncordon the node", uncordonk8sNode) By("Verify the PVC gets Bound") - verifyPVC(pvcName, true) + verifyPVCStatus(pvcName, true) deleteAndVerifyPVC(pvcName) By("Verifying that PV doesnt exists after PVC deletion") verifyPVForPVC(false, pvcName) diff --git a/tests/utils.go b/tests/utils.go index ca5b8826..a624caf8 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -304,14 +304,7 @@ func createAndVerifyPVC(expect_bound bool) { pvcName, OpenEBSNamespace, ) - ok := false - if !expect_bound { - ok = IsPVCPendingConsistently(pvcName) - } else { - ok = IsPVCBoundEventually(pvcName) - } - gomega.Expect(ok).To(gomega.Equal(true), - "while checking the pvc status") + verifyPVCStatus(pvcName, expect_bound) pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{}) gomega.Expect(err).To( @@ -323,7 +316,7 @@ func createAndVerifyPVC(expect_bound bool) { } // Verifies state of already created pvc based on expect_bound. -func verifyPVC(pvc_name string, expect_bound bool) { +func verifyPVCStatus(pvc_name string, expect_bound bool) { ok := false if !expect_bound { ok = IsPVCPendingConsistently(pvc_name) @@ -872,7 +865,6 @@ func cordonk8sNode() { "Could not cordon node", ) } - } }