From 2c09bbb91bdeda975c1e6d25ca85f1be73992cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 12 Apr 2022 16:33:12 +0200 Subject: [PATCH] ci: add multus integration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This new integration test will deploy a cluster with multus enabled. It will be comprised of two network interfaces for ceph public and cluster communications. For now, it only deploys a Ceph cluster up to the OSDs. Closes: https://github.com/rook/rook/issues/9784 Signed-off-by: Sébastien Han --- .github/workflows/canary-integration-test.yml | 45 +++++++++++++++++ .../setup-cluster-resources/action.yaml | 2 +- deploy/examples/cluster-multus-test.yaml | 49 ++++++++++++++++++ tests/scripts/github-action-helper.sh | 50 +++++++++++++++++++ tests/scripts/validate_cluster.sh | 2 +- 5 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 deploy/examples/cluster-multus-test.yaml diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 91c840d480f3..ccb3afe5ffa2 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -908,3 +908,48 @@ jobs: if: failure() && github.event_name == 'pull_request' uses: mxschmitt/action-tmate@v3 timeout-minutes: 60 + + multus-cluster-network: + runs-on: ubuntu-18.04 + if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: setup cluster resources + uses: ./.github/workflows/setup-cluster-resources + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: validate-yaml + run: tests/scripts/github-action-helper.sh validate_yaml + + - name: use local disk and create partitions for osds + run: | + tests/scripts/github-action-helper.sh use_local_disk + tests/scripts/github-action-helper.sh create_partitions_for_osds + + - name: deploy multus + run: tests/scripts/github-action-helper.sh deploy_multus + + - name: deploy multus cluster + run: tests/scripts/github-action-helper.sh deploy_multus_cluster + + - name: wait for prepare pod + run: tests/scripts/github-action-helper.sh wait_for_prepare_pod + + - name: wait for ceph to be ready + run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 + + - name: collect common logs + if: always() + uses: ./.github/workflows/collect-logs + with: + name: canary-multus + + - name: setup tmate session for debugging when event is PR + if: failure() && github.event_name == 'pull_request' + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 60 diff --git a/.github/workflows/setup-cluster-resources/action.yaml b/.github/workflows/setup-cluster-resources/action.yaml index 0dd3c6f411ab..8446235c668e 100644 --- a/.github/workflows/setup-cluster-resources/action.yaml +++ b/.github/workflows/setup-cluster-resources/action.yaml @@ -18,7 +18,7 @@ runs: with: minikube version: 'v1.24.0' kubernetes version: 'v1.23.0' - start args: --memory 6g --cpus=2 --addons ingress + start args: --memory 6g --cpus=2 --addons ingress --cni=flannel github token: ${{ inputs.github-token }} - name: install deps diff --git a/deploy/examples/cluster-multus-test.yaml b/deploy/examples/cluster-multus-test.yaml new file mode 100644 index 000000000000..3b11f8dd4a6d --- /dev/null +++ b/deploy/examples/cluster-multus-test.yaml @@ -0,0 +1,49 @@ +################################################################################################################# +# Define the settings for the rook-ceph cluster with common settings for a small test cluster. +# All nodes with available raw devices will be used for the Ceph cluster. One node is sufficient +# in this example. + +# For example, to create the cluster: +# kubectl create -f crds.yaml -f common.yaml -f operator.yaml +# kubectl create -f cluster-test.yaml +################################################################################################################# +kind: ConfigMap +apiVersion: v1 +metadata: + name: rook-config-override + namespace: rook-ceph # namespace:cluster +data: + config: | + [global] + osd_pool_default_size = 1 + mon_warn_on_pool_no_redundancy = false + bdev_flock_retry = 20 + bluefs_buffered_io = false +--- +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: my-cluster + namespace: rook-ceph # namespace:cluster +spec: + dataDirHostPath: /var/lib/rook + cephVersion: + image: quay.io/ceph/ceph:v17 + allowUnsupported: true + mon: + count: 1 + mgr: + count: 1 + dashboard: + enabled: true + network: + provider: multus + selectors: + public: public-net + cluster: cluster-net + crashCollector: + disable: true + storage: + useAllNodes: true + useAllDevices: true + #deviceFilter: diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 36b0d5436d84..6582739d79fa 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -379,6 +379,56 @@ function create_helm_tag() { docker tag "${build_image}" "rook/ceph:${helm_tag}" } +function deploy_multus() { + # download the multus daemonset, and remove mem and cpu limits that cause it to crash on minikube + curl https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset-thick-plugin.yml \ + | sed -e 's/cpu: /# cpu: /g' -e 's/memory: /# memory: /g' \ + | kubectl apply -f - + + # install whereabouts + kubectl apply \ + -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/daemonset-install.yaml \ + -f https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/master/doc/crds/ip-reconciler-job.yaml \ + -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_ippools.yaml \ + -f https://github.com/k8snetworkplumbingwg/whereabouts/raw/master/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml + + # create the rook-ceph namespace if it doesn't exist, the NAD will go in this namespace + kubectl create namespace rook-ceph || true + + # install network attachment definitions + IFACE="eth0" # the runner has eth0 so we don't need any heureustics to find the interface + kubectl apply -f - <