test: add volume provisioning test on cordoned node

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.27.4
 	github.com/openebs/google-analytics-4 v0.3.0
-	github.com/openebs/lib-csi v0.8.2
+	github.com/openebs/lib-csi v0.9.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.15.1
 	github.com/spf13/cobra v1.6.0

go.sum

@@ -314,8 +314,8 @@ github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
 github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
 github.com/openebs/google-analytics-4 v0.3.0 h1:rAv9BC476ennyGPWAy4D3vU7TlxEKiynbLUP3VgeBUc=
 github.com/openebs/google-analytics-4 v0.3.0/go.mod h1:lKjvRs6HAYOlOTYjtOUp35iTutoXzRL0hOeVAvLAfZI=
-github.com/openebs/lib-csi v0.8.2 h1:HmoiZX3VXFPglwqnRPnRus7K58ixDWBa19OpPZGk2Ws=
-github.com/openebs/lib-csi v0.8.2/go.mod h1:4yc0Q1thH+oU80z73zGELfrOw2yeLdLNIRmcrxBxsBc=
+github.com/openebs/lib-csi v0.9.0 h1:Yro5CN0cT3bRYOB1IdGPKXnBcPzxZnkxd4MLREAtFME=
+github.com/openebs/lib-csi v0.9.0/go.mod h1:4yc0Q1thH+oU80z73zGELfrOw2yeLdLNIRmcrxBxsBc=
 github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

tests/lvm_utils.go

@@ -79,7 +79,6 @@ func vgEmpty(name string) bool {
 	lvs, _, _ := execAtLocal("sudo", nil, args_lvs...)
 	lvs_str := strings.TrimSpace(string(lvs))
 	lv_cnt, _ := strconv.Atoi(lvs_str)
-	fmt.Printf("lvs cnt is %d\n", lv_cnt)
 	if lv_cnt != 0 {
 		return false
 	} else {
@@ -160,7 +159,6 @@ func removeVg(name string) {
 		if current_retry < retries {
 			vg_empty := vgEmpty(name)
 			if vg_empty {
-				fmt.Printf("No lv in vg before vg remove\n")
 				break
 			} else {
 				fmt.Printf("lv in vg during retry %d\n", current_retry)

tests/provision_test.go

@@ -149,6 +149,22 @@ func vgPatternMatchPresentTest() {
 	By("Deleting storage class", deleteStorageClass)
 }
 
+func scheduleOnCordonedNodeTest() {
+	device := setupVg(20, "lvmvg")
+	defer cleanupVg(device, "lvmvg")
+	By("Creating storage class", createStorageClass)
+	By("Cordoning the node", cordonk8sNode)
+	By("Creating and verifying PVC Bound status. It should not be Bound")
+	createAndVerifyPVC(false)
+	By("Uncordon the node", uncordonk8sNode)
+	By("Verify the PVC gets Bound")
+	verifyPVCStatus(pvcName, true)
+	deleteAndVerifyPVC(pvcName)
+	By("Verifying that PV doesnt exists after PVC deletion")
+	verifyPVForPVC(false, pvcName)
+	By("Deleting storage class", deleteStorageClass)
+}
+
 func vgPatternNoMatchPresentTest() {
 	device := setupVg(20, "lvmvg212")
 	device_1 := setupVg(20, "lvmvg")
@@ -316,6 +332,7 @@ func schedulingTest() {
 	By("###Running vg specified in sc not present test###", vgSpecifiedNotPresentTest)
 	By("###Running lvmnode has vg matching vgpattern test###", vgPatternMatchPresentTest)
 	By("###Running lvmnode doesnt have vg matching vgpattern test###", vgPatternNoMatchPresentTest)
+	By("###Running volume schedule on Cordoned node test###", scheduleOnCordonedNodeTest)
 }
 
 func capacityTest() {

tests/utils.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -12,6 +13,9 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/homedir"
 
 	"github.com/openebs/lvm-localpv/pkg/lvm"
 	"github.com/openebs/lvm-localpv/tests/container"
@@ -249,14 +253,12 @@ func VerifyLVMVolume(expect_ready bool, expected_vg string) {
 			// It gets deleted, by the csi provisioner only when the owner node of cr marks is
 			// as Failed. So incase, we do a get of cr when the cr was being handled then we expect
 			// state to be either Pending or Failed.
-			fmt.Printf("checking vol object as vol is non nil, vol is %v\n", vol)
 			gomega.Expect(vol.Status.State).To(gomega.Or(gomega.Equal("Pending"), gomega.Equal("Failed")),
 				"While checking if lvmvolume: %s is in Pending or Failed state", pvcObj.Spec.VolumeName)
 		}
 	} else {
 		gomega.Expect(err).To(gomega.BeNil(), "while fetching the lvm volume {%s}", pvcObj.Spec.VolumeName)
 		if expected_vg != "" {
-			fmt.Printf("vol is %v\n", vol)
 			gomega.Expect(vol.Spec.VolGroup).To(gomega.Equal(expected_vg),
 				"while checking volume group of lvm volume", pvcObj.Spec.VolumeName)
 		} else {
@@ -302,14 +304,7 @@ func createAndVerifyPVC(expect_bound bool) {
 		pvcName,
 		OpenEBSNamespace,
 	)
-	ok := false
-	if !expect_bound {
-		ok = IsPVCPendingConsistently(pvcName)
-	} else {
-		ok = IsPVCBoundEventually(pvcName)
-	}
-	gomega.Expect(ok).To(gomega.Equal(true),
-		"while checking the pvc status")
+	verifyPVCStatus(pvcName, expect_bound)
 
 	pvcObj, err = PVCClient.WithNamespace(OpenEBSNamespace).Get(pvcObj.Name, metav1.GetOptions{})
 	gomega.Expect(err).To(
@@ -320,6 +315,18 @@ func createAndVerifyPVC(expect_bound bool) {
 	)
 }
 
+// Verifies state of already created pvc based on expect_bound.
+func verifyPVCStatus(pvc_name string, expect_bound bool) {
+	ok := false
+	if !expect_bound {
+		ok = IsPVCPendingConsistently(pvc_name)
+	} else {
+		ok = IsPVCBoundEventually(pvc_name)
+	}
+	gomega.Expect(ok).To(gomega.Equal(true),
+		"while checking the pvc status")
+}
+
 func createAndVerifyBlockPVC(expect_bound bool) {
 	var (
 		err     error
@@ -819,3 +826,79 @@ func createNodeDaemonSet(ds *appsv1.DaemonSet) {
 		gomega.BeNil(),
 		"creating node plugin daemonset %v", nodeDaemonSet)
 }
+
+// Creates a k8s client using kubeconfig path.
+func getk8sClient() (client *kubernetes.Clientset) {
+	kubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config")
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	gomega.Expect(err).To(
+		gomega.BeNil(),
+		"Could not created a k8s client",
+	)
+	client, c_err := kubernetes.NewForConfig(config)
+	gomega.Expect(c_err).To(
+		gomega.BeNil(),
+		"Could not created a k8s client",
+	)
+	return client
+}
+
+// Lists k8s nodes.
+func listNodes(client *kubernetes.Clientset) (nodes *corev1.NodeList) {
+	nodes, n_err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	gomega.Expect(n_err).To(
+		gomega.BeNil(),
+		"Could not list nodes",
+	)
+	return nodes
+}
+
+// Cordons all k8s nodes.
+func cordonk8sNode() {
+	client := getk8sClient()
+	nodes := listNodes(client)
+	for _, node := range nodes.Items {
+		if !node.Spec.Unschedulable {
+			c_err := cordonNode(client, &node)
+			gomega.Expect(c_err).To(
+				gomega.BeNil(),
+				"Could not cordon node",
+			)
+		}
+	}
+}
+
+// UnCordons all k8s nodes.
+func uncordonk8sNode() {
+	client := getk8sClient()
+	nodes := listNodes(client)
+	for _, node := range nodes.Items {
+		if node.Spec.Unschedulable {
+			c_err := uncordonNode(client, &node)
+			gomega.Expect(c_err).To(
+				gomega.BeNil(),
+				"Could not uncordon node",
+			)
+		}
+	}
+}
+
+// Adds cordon taint to a specific node.
+func cordonNode(clientset *kubernetes.Clientset, node *corev1.Node) error {
+	updatedNode := node.DeepCopy()
+	updatedNode.Spec.Unschedulable = true
+
+	_, err := clientset.CoreV1().Nodes().Update(context.TODO(), updatedNode, metav1.UpdateOptions{})
+	return err
+
+}
+
+// Removes cordon taint from a specific node.
+func uncordonNode(clientset *kubernetes.Clientset, node *corev1.Node) error {
+	updatedNode := node.DeepCopy()
+	updatedNode.Spec.Unschedulable = false
+
+	_, err := clientset.CoreV1().Nodes().Update(context.TODO(), updatedNode, metav1.UpdateOptions{})
+	return err
+
+}