feat: Add e2e-test ci yaml

Signed-off-by: duanmengkk <[email protected]>

.github/workflows/ci.yml

@@ -65,4 +65,38 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: '1.20'
-      - run: make test
+      - run: make test
+  e2e:
+    name: E2e test
+    needs: build
+    runs-on: ubuntu-22.04
+    steps:
+      # Free up disk space on Ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed, if set to "true" but frees about 6 GB
+          tool-cache: false
+          # all of these default to true, but feel free to set to "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: false
+          swap-storage: false
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+      - name: Run e2e test
+        run: ./hack/rune2e.sh
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: kosmos-e2e-logs-${{ github.run_id }}
+          path: ${{ github.workspace }}/e2e-test/logs-*

Makefile

@@ -7,6 +7,7 @@ REGISTRY?="ghcr.io/kosmos-io"
 REGISTRY_USER_NAME?=""
 REGISTRY_PASSWORD?=""
 REGISTRY_SERVER_ADDRESS?=""
+KIND_IMAGE_TAG?="v1.25.3"
 
 TARGETS :=  clusterlink-controller-manager  \
 			kosmos-operator \
@@ -133,4 +134,11 @@ ifeq (, $(shell which golangci-lint))
 GOLANGLINT_BIN=$(shell go env GOPATH)/bin/golangci-lint
 else
 GOLANGLINT_BIN=$(shell which golangci-lint)
-endif
+endif
+
+image-base-kind-builder:
+	docker buildx build \
+	    -t $(REGISTRY)/node:$(KIND_IMAGE_TAG) \
+        --platform=linux/amd64,linux/arm64 \
+        --push \
+        -f cluster/images/buildx.kind.Dockerfile .

cluster/images/buildx.kind.Dockerfile

@@ -0,0 +1,2 @@
+FROM kindest/node:v1.25.3
+RUN clean-install tcpdump && clean-install iputils-ping && clean-install iptables && clean-install net-tools

deploy/clustertree-cluster-manager.yml

@@ -79,6 +79,7 @@ spec:
           command:
             - clustertree-cluster-manager
             - --multi-cluster-service=true
+            - --auto-mcs-prefix=kosmos-auto
             - --v=4
       volumes:
         - name: credentials

deploy/clusterlink-operator.yml → deploy/kosmos-operator.yml

@@ -1,13 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name:  clusterlink-operator
+  name:  kosmos-operator
   namespace: kosmos-system
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: clusterlink-operator
+  name: kosmos-operator
   namespace: kosmos-system
   labels:
     app: operator
@@ -21,7 +21,7 @@ spec:
       labels:
         app: operator
     spec:
-      serviceAccountName: clusterlink-operator
+      serviceAccountName: kosmos-operator
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -36,11 +36,11 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: operator
-        image: ghcr.io/kosmos-io/clusterlink-operator:__VERSION__
+        image: ghcr.io/kosmos-io/kosmos-operator:__VERSION__
         imagePullPolicy: IfNotPresent
         command:
-          - clusterlink-operator
-          - --controlpanelconfig=/etc/clusterlink/kubeconfig
+          - kosmos-operator
+          - --controlpanelconfig=/etc/kosmos-operator/kubeconfig
         resources:
           limits:
             memory: 200Mi
@@ -51,12 +51,10 @@ spec:
         env:
         - name: VERSION
           value: __VERSION__
-        - name: CLUSTER_NAME
-          value: __CLUSTER_NAME__
         - name: USE_PROXY
           value: "false"
         volumeMounts:
-          - mountPath: /etc/clusterlink
+          - mountPath: /etc/kosmos-operator
             name: proxy-config
             readOnly: true
       volumes:

deploy/kosmos-rbac.yml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kosmos
+rules:
+  - apiGroups: ['*']
+    resources: ['*']
+    verbs: ["*"]
+  - nonResourceURLs: ['*']
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kosmos
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kosmos
+subjects:
+  - kind: ServiceAccount
+    name: kosmos-control
+    namespace: kosmos-system
+  - kind: ServiceAccount
+    name: clusterlink-controller-manager
+    namespace: kosmos-system
+  - kind: ServiceAccount
+    name: kosmos-operator
+    namespace: kosmos-system

hack/cluster.sh

@@ -6,8 +6,7 @@ set -o pipefail
 
 CURRENT="$(dirname "${BASH_SOURCE[0]}")"
 ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-DEFAULT_NAMESPACE="kosmos-system"
-KIND_IMAGE="ghcr.io/kosmos-io/kindest/node:v1.25.3_1"
+KIND_IMAGE="ghcr.io/kosmos-io/node:v1.25.3"
 # true: when cluster is exist, reuse exist one!
 REUSE=${REUSE:-false}
 VERSION=${VERSION:-latest}
@@ -49,8 +48,6 @@ function create_cluster() {
       sed -e "s|__POD_CIDR__|$podcidr|g" -e "s|__SERVICE_CIDR__|$servicecidr|g" -e "w ${CLUSTER_DIR}/calicoconfig" "${CURRENT}/clustertemplete/calicoconfig"
     fi
 
-
-
     if [[ "$(kind get clusters | grep -c "${clustername}")" -eq 1 && "${REUSE}" = true ]]; then
       echo "cluster ${clustername} exist reuse it"
     else
@@ -111,15 +108,13 @@ function create_cluster() {
       "kubectl get nodes | awk 'NR>1 {if (\$2 != \"Ready\") exit 1; }' && [ \$(kubectl get nodes --no-headers | wc -l) -eq ${N} ]" \
       300
     echo "all node ready"
-
 }
 
 function join_cluster() {
   local host_cluster=$1
   local member_cluster=$2
-  local container_ip_port
   local kubeconfig_path="${ROOT}/environments/${member_cluster}/kubeconfig"
-  local base64_kubeconfig=$(base64 < "$kubeconfig_path")
+  local base64_kubeconfig=$(base64 -w 0 < "$kubeconfig_path")
   echo " base64 kubeconfig successfully converted: $base64_kubeconfig "
 
   local common_metadata=""
@@ -148,8 +143,6 @@ EOF
   kubectl --context="kind-${member_cluster}" -n kosmos-system delete secret controlpanel-config || true
   kubectl --context="kind-${member_cluster}" -n kosmos-system create secret generic controlpanel-config --from-file=kubeconfig="${ROOT}/environments/${host_cluster}/kubeconfig"
   kubectl --context="kind-${member_cluster}" apply -f "$ROOT"/deploy/clusterlink-datapanel-rbac.yml
-  sed -e "s|__VERSION__|$VERSION|g" -e "s|__CLUSTER_NAME__|$member_cluster|g" -e "w ${ROOT}/environments/${member_cluster}/clusterlink-operator.yml" "$ROOT"/deploy/clusterlink-operator.yml
-  kubectl --context="kind-${member_cluster}" apply -f "${ROOT}/environments/${member_cluster}/clusterlink-operator.yml"
 }
 
 function deploy_cluster() {
@@ -158,6 +151,7 @@ function deploy_cluster() {
    load_cluster_images "$clustername"
 
    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/clusterlink-namespace.yml
+   kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/kosmos-rbac.yml
    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/crds
    kubectl --context="kind-${clustername}" apply -f "$ROOT"/deploy/crds/mcs
    util::wait_for_crd clusternodes.kosmos.io clusters.kosmos.io
@@ -171,6 +165,11 @@ function deploy_cluster() {
    kubectl --context="kind-${clustername}" apply -f "${ROOT}/environments/clustertree-cluster-manager.yml"
 
    echo "cluster $clustername deploy clustertree success"
+
+   sed -e "s|__VERSION__|$VERSION|g" -e "w ${ROOT}/environments/kosmos-operator.yml" "$ROOT"/deploy/kosmos-operator.yml
+   kubectl --context="kind-${clustername}" apply -f "${ROOT}/environments/kosmos-operator.yml"
+
+   echo "cluster $clustername deploy kosmos-operator success"
 }
 
 function load_cluster_images() {
@@ -179,7 +178,7 @@ function load_cluster_images() {
     kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-network-manager:"${VERSION}"
     kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-controller-manager:"${VERSION}"
     kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-elector:"${VERSION}"
-    kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-operator:"${VERSION}"
+    kind load docker-image -n "$clustername" ghcr.io/kosmos-io/kosmos-operator:"${VERSION}"
     kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-agent:"${VERSION}"
     kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clusterlink-proxy:"${VERSION}"
     kind load docker-image -n "$clustername" ghcr.io/kosmos-io/clustertree-cluster-manager:"${VERSION}"

hack/local-down-clusterlink.sh

@@ -7,13 +7,15 @@ set -o pipefail
 HOST_CLUSTER_NAME="cluster-host-local"
 
 MEMBER1_CLUSTER_NAME="cluster-member1-local"
+MEMBER2_CLUSTER_NAME="cluster-member2-local"
 
 ROOT="$(dirname "${BASH_SOURCE[0]}")"
 source "$(dirname "${BASH_SOURCE[0]}")/cluster.sh"
 
 #cluster cluster
 delete_cluster $HOST_CLUSTER_NAME
 delete_cluster $MEMBER1_CLUSTER_NAME
+delete_cluster $MEMBER2_CLUSTER_NAME
 
 
-echo "clusterlink local down success"
+echo "clusterlink local down success"