From f3e85602e5228e43adda3954a338af9b1d88f538 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Fri, 23 Aug 2024 14:04:08 +0300 Subject: [PATCH 1/2] test: Create functional test for different KubevirtCI cluster configurations This should run as a CI lane. It will repetitively up and down clusters with the following features in the order they are named in KSM and Swap ETCD in memory NFS CSI Rook Ceph Signed-off-by: aerosouund --- cluster-provision/k8s/check-cluster-opts.sh | 62 +++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 cluster-provision/k8s/check-cluster-opts.sh diff --git a/cluster-provision/k8s/check-cluster-opts.sh b/cluster-provision/k8s/check-cluster-opts.sh new file mode 100644 index 0000000000..e8a6c9c57a --- /dev/null +++ b/cluster-provision/k8s/check-cluster-opts.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -exuo pipefail + +make -C ../gocli container + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +provision_dir="$1" +provider="${provision_dir}" + +function cleanup() { + cd "$DIR" && cd ../.. + make cluster-down +} + +export KUBEVIRTCI_GOCLI_CONTAINER=quay.io/kubevirtci/gocli:latest +# check cluster-up +( + ksh="./cluster-up/kubectl.sh" + cd "$DIR" && cd ../.. + export KUBEVIRTCI_PROVISION_CHECK=1 + export KUBEVIRT_PROVIDER="k8s-${provider}" + export KUBEVIRT_MEMORY_SIZE=5520M + + # Test KSM and Swap + export KUBEVIRT_KSM_ON="true" + export KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS=20 + export KUBEVIRT_KSM_PAGES_TO_SCAN=10 + + export KUBEVIRT_SWAP_ON="true" + export KUBEVIRT_SWAP_SIZE_IN_GB=1 + + trap cleanup EXIT ERR SIGINT SIGTERM SIGQUIT + make cluster-up + ${ksh} get nodes + make cluster-down + + export KUBEVIRT_KSM_ON="false" + export KUBEVIRT_SWAP_ON="false" + + # Test ETCD in memory + export KUBEVIRT_WITH_ETC_IN_MEMORY="true" + export KUBEVIRT_WITH_ETC_CAPACITY="1024M" + make cluster-up + ${ksh} get nodes + make cluster-down + + # Test NFS CSI + export KUBEVIRT_DEPLOY_NFS_CSI="true" + make cluster-up + ${ksh} get nodes + make cluster-down + + export KUBEVIRT_DEPLOY_NFS_CSI="false" + + # Test rook ceph + export KUBEVIRT_STORAGE="rook-ceph-default" + make cluster-up + ${ksh} get nodes + make cluster-down +) \ No newline at end of file From 8f6e7696c52934e2d63e658033f0352c2641e528 Mon Sep 17 00:00:00 2001 From: aerosouund Date: Thu, 19 Sep 2024 16:40:41 +0300 Subject: [PATCH 2/2] test: Create ginkgo based functional testing suite Add tests for rook and nfs-csi and configure check cluster opts script to build ginkgo from vendor directory and run the tests in separate shell environments Signed-off-by: aerosouund --- .../gocli/tests/kvci_suite_test.go | 30 ++++++++++++ cluster-provision/gocli/tests/nfs_test.go | 49 +++++++++++++++++++ cluster-provision/gocli/tests/rook_test.go | 49 +++++++++++++++++++ cluster-provision/k8s/check-cluster-opts.sh | 34 +++++++------ 4 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 cluster-provision/gocli/tests/kvci_suite_test.go create mode 100644 cluster-provision/gocli/tests/nfs_test.go create mode 100644 cluster-provision/gocli/tests/rook_test.go diff --git a/cluster-provision/gocli/tests/kvci_suite_test.go b/cluster-provision/gocli/tests/kvci_suite_test.go new file mode 100644 index 0000000000..2a6a9b14c2 --- /dev/null +++ b/cluster-provision/gocli/tests/kvci_suite_test.go @@ -0,0 +1,30 @@ +package tests + +import ( + "flag" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + k8s "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/k8s" +) + +var ( + kubeconfig string + k8sClient k8s.K8sDynamicClient +) + +func TestTests(t *testing.T) { + flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file") + + RegisterFailHandler(Fail) + RunSpecs(t, "Tests Suite") +} + +var _ = BeforeEach(func() { + config, err := k8s.NewConfig(kubeconfig, 36443) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = k8s.NewDynamicClient(config) + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/cluster-provision/gocli/tests/nfs_test.go b/cluster-provision/gocli/tests/nfs_test.go new file mode 100644 index 0000000000..64ce6c6f64 --- /dev/null +++ b/cluster-provision/gocli/tests/nfs_test.go @@ -0,0 +1,49 @@ +package tests + +import ( + "fmt" + "time" + + "github.com/cenkalti/backoff/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("NFS Functional test", func() { + It("should execute nfs-csi successfully", func() { + pvc := &corev1.PersistentVolumeClaim{} + + operation := func() error { + obj, err := k8sClient.Get(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PersistentVolumeClaim", + }, "pvc-nfs-dynamic", "nfs-csi") + + if err != nil { + return err + } + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, pvc) + if err != nil { + return err + } + + if pvc.Status.Phase != "Bound" { + return fmt.Errorf("PVC didn't move to Bound phase") + } + + return nil + } + + backoffStrategy := backoff.NewExponentialBackOff() + backoffStrategy.InitialInterval = 10 * time.Second + backoffStrategy.MaxElapsedTime = 3 * time.Minute + + err := backoff.Retry(operation, backoffStrategy) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/cluster-provision/gocli/tests/rook_test.go b/cluster-provision/gocli/tests/rook_test.go new file mode 100644 index 0000000000..3e76b56df6 --- /dev/null +++ b/cluster-provision/gocli/tests/rook_test.go @@ -0,0 +1,49 @@ +package tests + +import ( + "fmt" + "time" + + "github.com/cenkalti/backoff/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("Ceph Functional test", func() { + It("should execute ceph successfully", func() { // test for something else, the pvc wont exist + blockpool := &cephv1.CephBlockPool{} + + operation := func() error { + obj, err := k8sClient.Get(schema.GroupVersionKind{ + Group: "ceph.rook.io", + Version: "v1", + Kind: "CephBlockPool", + }, "replicapool", "rook-ceph") + + if err != nil { + return err + } + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, blockpool) + if err != nil { + return err + } + + if blockpool.Status == nil || blockpool.Status.Phase != "Ready" { + return fmt.Errorf("Ceph pool block didn't move to ready status") + } + + return nil + } + + backoffStrategy := backoff.NewExponentialBackOff() + backoffStrategy.InitialInterval = 30 * time.Second + backoffStrategy.MaxElapsedTime = 6 * time.Minute + + err := backoff.Retry(operation, backoffStrategy) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/cluster-provision/k8s/check-cluster-opts.sh b/cluster-provision/k8s/check-cluster-opts.sh index e8a6c9c57a..9d23f9e532 100644 --- a/cluster-provision/k8s/check-cluster-opts.sh +++ b/cluster-provision/k8s/check-cluster-opts.sh @@ -14,15 +14,20 @@ function cleanup() { make cluster-down } +go build -o ./ginkgo cluster-provision/gocli/vendor/github.com/onsi/ginkgo/v2/ginkgo + export KUBEVIRTCI_GOCLI_CONTAINER=quay.io/kubevirtci/gocli:latest +export KUBEVIRT_PROVIDER_EXTRA_ARGS='--k8s-port 36443' +export KUBEVIRTCI_PROVISION_CHECK=1 +export KUBEVIRT_PROVIDER="k8s-${provider}" +export KUBEVIRT_MEMORY_SIZE=5520M + +ksh="./cluster-up/kubectl.sh" +cd "$DIR" && cd ../.. + +trap cleanup EXIT ERR SIGINT SIGTERM SIGQUIT # check cluster-up ( - ksh="./cluster-up/kubectl.sh" - cd "$DIR" && cd ../.. - export KUBEVIRTCI_PROVISION_CHECK=1 - export KUBEVIRT_PROVIDER="k8s-${provider}" - export KUBEVIRT_MEMORY_SIZE=5520M - # Test KSM and Swap export KUBEVIRT_KSM_ON="true" export KUBEVIRT_KSM_SLEEP_BETWEEN_SCANS_MS=20 @@ -31,32 +36,31 @@ export KUBEVIRTCI_GOCLI_CONTAINER=quay.io/kubevirtci/gocli:latest export KUBEVIRT_SWAP_ON="true" export KUBEVIRT_SWAP_SIZE_IN_GB=1 - trap cleanup EXIT ERR SIGINT SIGTERM SIGQUIT make cluster-up ${ksh} get nodes make cluster-down - - export KUBEVIRT_KSM_ON="false" - export KUBEVIRT_SWAP_ON="false" - +) +( # Test ETCD in memory export KUBEVIRT_WITH_ETC_IN_MEMORY="true" export KUBEVIRT_WITH_ETC_CAPACITY="1024M" make cluster-up ${ksh} get nodes make cluster-down - +) +( # Test NFS CSI export KUBEVIRT_DEPLOY_NFS_CSI="true" make cluster-up ${ksh} get nodes + ./ginkgo -focus="nfs-csi" cluster-provision/gocli/tests/ make cluster-down - - export KUBEVIRT_DEPLOY_NFS_CSI="false" - +) +( # Test rook ceph export KUBEVIRT_STORAGE="rook-ceph-default" make cluster-up ${ksh} get nodes + ./ginkgo -focus="rook" cluster-provision/gocli/tests/ make cluster-down ) \ No newline at end of file