[Release-1.29] E2E GH Action Test (#5908)

* Support local builds of RKE2 in E2E dnscache test

Signed-off-by: Derek Nola <[email protected]>

* Combine integration and e2e test into single github action

Signed-off-by: Derek Nola <[email protected]>

---------

Signed-off-by: Derek Nola <[email protected]>

.github/workflows/integration.yaml → .github/workflows/test-suite.yaml

@@ -1,24 +1,19 @@
-name: Integration Test Coverage
+name: Test Suite
 on: 
   push:
     paths-ignore:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/**"
-      - "!tests/integration**"
       - ".github/**"
-      - "!.github/workflows/integration.yaml"
+      - "!.github/workflows/test-suite.yaml"
   pull_request:
     paths-ignore:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/**"
-      - "!tests/integration**"
-      - "!tests/e2e**"
       - ".github/**"
-      - "!.github/workflows/integration.yaml"
+      - "!.github/workflows/test-suite.yaml"
   workflow_dispatch: {}
 
 permissions:
@@ -52,16 +47,17 @@ jobs:
       run: sudo apt-get install -y libarchive-tools
     - name: Build RKE2 Binary and Runtime Image
       run: |
-        GOCOVER=true make build-binary
+        GOCOVER=true make package-bundle
         make package-image-runtime
-        cp ./bin/rke2 ./build/images/rke2-binary
+        cp ./bin/rke2 ./build/images/rke2
+        cp ./dist/artifacts/rke2.linux-amd64.tar.gz ./build/images/rke2.linux-amd64.tar.gz
     # Can only upload from a single path, so we need to copy the binary to the image directory
     - name: Upload RKE2 Binary and Runtime Image
       uses: actions/upload-artifact@v4
       with:
         name: rke2-test-artifacts
         path: ./build/images/*
-  test:
+  itest:
     needs: build
     name: Integration Tests
     runs-on: ubuntu-22.04
@@ -94,7 +90,7 @@ jobs:
         path: ./build/images
     - name: Setup Binary
       run: |
-        mv ./build/images/rke2-binary ./bin/rke2
+        mv ./build/images/rke2 ./bin/rke2
         chmod +x ./bin/rke2
     - name: Run Integration Tests
       run: | 
@@ -113,9 +109,68 @@ jobs:
       if: ${{ failure() }}
       run: cat ./tests/integration/${{ matrix.itest }}/r2log.txt
     - name: On Failure, Launch Debug Session
-      uses: dereknola/action-upterm@main
+      uses: dereknola/action-upterm@v1.1
       if: ${{ failure() }}
       with:
         ## If no one connects after 5 minutes, shut down server.
         wait-timeout-minutes: 5
         limit-access-to-actor: true
+
+  e2e:
+    name: "E2E Tests"
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        etest: [dnscache]
+      max-parallel: 3
+    steps:
+    - name: "Checkout"
+      uses: actions/checkout@v4
+      with: {fetch-depth: 1}
+
+    - name: Set up vagrant and libvirt
+      uses: ./.github/actions/vagrant-setup
+    - name: "Vagrant Cache"
+      uses: actions/cache@v4
+      with:
+        path: |
+            ~/.vagrant.d/boxes
+        key: vagrant-box-ubuntu-2004
+    - name: "Vagrant Plugin(s)"
+      run: vagrant plugin install vagrant-rke2 vagrant-reload vagrant-scp
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with: 
+        go-version-file: 'go.mod'
+        cache: false
+    - name: Install Kubectl
+      run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+    - name: Download RKE2 Binary and Runtime Image
+      uses: actions/download-artifact@v4
+      with:
+        name: rke2-test-artifacts
+        path: ./artifacts
+    - name: Place the artifacts back and compute sha256sum
+      run: |
+        mkdir -p ./dist/artifacts
+        mkdir -p ./build/images
+        mv ./artifacts/rke2-* ./build/images/
+        mv ./artifacts/rke2.linux-amd64.tar.gz ./dist/artifacts/rke2.linux-amd64.tar.gz
+        sha256sum ./dist/artifacts/rke2.linux-amd64.tar.gz > ./dist/artifacts/sha256sum-amd64.txt
+    - name: Run ${{ matrix.etest }} Test
+      run: | 
+        cd tests/e2e/${{ matrix.etest }}
+        go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
+    - name: On Failure, Launch Debug Session
+      uses: dereknola/[email protected]
+      if: ${{ failure() }}
+      with:
+        ## If no one connects after 5 minutes, shut down server.
+        wait-timeout-minutes: 5
+        limit-access-to-actor: true

tests/e2e/dnscache/Vagrantfile

@@ -35,7 +35,6 @@ def provision(vm, roles, role_num, node_num)
   vm.provision "IPv6 Setup", type: "shell", path: scripts_location + "/ipv6.sh", args: [node_ip4, node_ip6, node_ip6_gw, CNI, vm.box]
 
   install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
-  vm.provision "Ping Check", type: "shell", inline: "ping -4 -c 2 rke2.io"
 
   if roles.include?("server")
     vm.provision :rke2, run: 'once' do |rke2|

tests/e2e/dnscache/dnscache_test.go

@@ -18,6 +18,7 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
+var local = flag.Bool("local", false, "deploy a locally built RKE2")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+rke2r1 or nil for latest commit from master
@@ -40,7 +41,11 @@ var _ = Describe("Verify dnscache Configuration", Ordered, func() {
 
 	It("Starts up with no issues", func() {
 		var err error
-		serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
+		if *local {
+			serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
+		} else {
+			serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
+		}
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 		fmt.Println("CLUSTER CONFIG")
 		fmt.Println("OS:", *nodeOS)
@@ -98,12 +103,11 @@ var _ = Describe("Verify dnscache Configuration", Ordered, func() {
 		}, "120s", "5s").Should(ContainSubstring("2"))
 	})
 
-
 	It("Verifies nodecache is working", func() {
 		cmd := "dig +retries=0 @169.254.20.10 www.kubernetes.io"
 		for _, nodeName := range serverNodeNames {
 			Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("status: NOERROR"), "failed cmd: "+cmd)
-	}
+		}
 	})
 })
 

tests/e2e/testutils.go

@@ -128,6 +128,103 @@ func CreateCluster(nodeOS string, serverCount int, agentCount int) ([]string, []
 	return serverNodeNames, agentNodeNames, nil
 }
 
+func scpRKE2Artifacts(nodeNames []string) error {
+	binary := []string{
+		"dist/artifacts/rke2.linux-amd64.tar.gz",
+		"dist/artifacts/sha256sum-amd64.txt",
+	}
+	images := []string{
+		"build/images/rke2-images.linux-amd64.tar.zst",
+		"build/images/rke2-runtime.tar",
+	}
+
+	// vagrant scp doesn't allow coping multiple files at once
+	// nor does it allow copying as sudo, so we have to copy each file individually
+	// to /tmp/ and then move them to the correct location
+	for _, node := range nodeNames {
+		for _, artifact := range append(binary, images...) {
+			cmd := fmt.Sprintf(`vagrant scp ../../../%s %s:/tmp/`, artifact, node)
+			if _, err := RunCommand(cmd); err != nil {
+				return err
+			}
+		}
+		if _, err := RunCmdOnNode("mkdir -p /var/lib/rancher/rke2/agent/images", node); err != nil {
+			return err
+		}
+		for _, image := range images {
+			cmd := fmt.Sprintf("mv /tmp/%s /var/lib/rancher/rke2/agent/images/", filepath.Base(image))
+			if _, err := RunCmdOnNode(cmd, node); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// CreateLocalCluster creates a cluster using the locally built RKE2 bundled binary and images.
+// Run at a minimum "make package-bundle" and "make package-image-runtime" first
+// The vagrant-scp plugin must be installed for this function to work.
+func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) {
+
+	serverNodeNames, agentNodeNames, nodeEnvs := genNodeEnvs(nodeOS, serverCount, agentCount)
+
+	var testOptions string
+	var cmd string
+
+	for _, env := range os.Environ() {
+		if strings.HasPrefix(env, "E2E_") {
+			testOptions += " " + env
+		}
+	}
+	testOptions += " E2E_RELEASE_VERSION=skip"
+
+	// Provision the first server node. In GitHub Actions, this also imports the VM image into libvirt, which
+	// takes time and can cause the next vagrant up to fail if it is not given enough time to complete.
+	cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0])
+	fmt.Println(cmd)
+	if _, err := RunCommand(cmd); err != nil {
+		return nil, nil, newNodeError(cmd, serverNodeNames[0], err)
+	}
+
+	// Bring up the rest of the nodes in parallel
+	errg, _ := errgroup.WithContext(context.Background())
+	for _, node := range append(serverNodeNames[1:], agentNodeNames...) {
+		cmd := fmt.Sprintf(`%s %s vagrant up --no-provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
+		errg.Go(func() error {
+			if _, err := RunCommand(cmd); err != nil {
+				return newNodeError(cmd, node, err)
+			}
+			return nil
+		})
+		// libVirt/Virtualbox needs some time between provisioning nodes
+		time.Sleep(10 * time.Second)
+	}
+	if err := errg.Wait(); err != nil {
+		return nil, nil, err
+	}
+
+	if err := scpRKE2Artifacts(append(serverNodeNames, agentNodeNames...)); err != nil {
+		return nil, nil, err
+	}
+	// Install RKE2 on all nodes in parallel
+	errg, _ = errgroup.WithContext(context.Background())
+	for _, node := range append(serverNodeNames, agentNodeNames...) {
+		cmd = fmt.Sprintf(`%s %s vagrant provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
+		errg.Go(func() error {
+			if _, err := RunCommand(cmd); err != nil {
+				return newNodeError(cmd, node, err)
+			}
+			return nil
+		})
+		// RKE2 needs some time between joining nodes to avoid learner issues
+		time.Sleep(20 * time.Second)
+	}
+	if err := errg.Wait(); err != nil {
+		return nil, nil, err
+	}
+	return serverNodeNames, agentNodeNames, nil
+}
+
 func DeployWorkload(workload string, kubeconfig string) (string, error) {
 	resourceDir := "../resource_files"
 	files, err := os.ReadDir(resourceDir)
@@ -148,7 +245,7 @@ func DeployWorkload(workload string, kubeconfig string) (string, error) {
 // RestartCluster restarts the rke2 service on each server-agent given
 func RestartCluster(nodeNames []string) error {
 	for _, nodeName := range nodeNames {
-		const cmd = "sudo systemctl restart rke2-*"
+		const cmd = "systemctl restart rke2-*"
 		if _, err := RunCmdOnNode(cmd, nodeName); err != nil {
 			return err
 		}
@@ -239,11 +336,11 @@ func GetVagrantLog(cErr error) string {
 }
 
 func GenKubeConfigFile(serverName string) (string, error) {
-	cmd := fmt.Sprintf("vagrant ssh %s -c \"sudo cat /etc/rancher/rke2/rke2.yaml\"", serverName)
-	kubeConfig, err := RunCommand(cmd)
+	kubeConfig, err := RunCmdOnNode("cat /etc/rancher/rke2/rke2.yaml", serverName)
 	if err != nil {
 		return "", err
 	}
+
 	nodeIP, err := FetchNodeExternalIP(serverName)
 	if err != nil {
 		return "", err
@@ -316,11 +413,15 @@ func ParsePods(kubeconfig string, print bool) ([]Pod, error) {
 	return pods, nil
 }
 
-// RunCmdOnNode executes a command from within the given node
+// RunCmdOnNode executes a command from within the given node as sudo
 func RunCmdOnNode(cmd string, nodename string) (string, error) {
 	communicator := "ssh"
-	runcmd := "vagrant " + communicator + " -c \"" + cmd + "\" " + nodename
-	return RunCommand(runcmd)
+	runcmd := "vagrant " + communicator + " -c \"sudo " + cmd + "\" " + nodename
+	out, err := RunCommand(runcmd)
+	if err != nil {
+		return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err)
+	}
+	return out, nil
 }
 
 // RunCmdOnWindowsNode executes a command from within the given windows node
@@ -339,7 +440,7 @@ func RunCommand(cmd string) (string, error) {
 // StartCluster starts the rke2 service on each node given
 func StartCluster(nodeNames []string) error {
 	for _, nodeName := range nodeNames {
-		cmd := "sudo systemctl start rke2"
+		cmd := "systemctl start rke2"
 		if strings.Contains(nodeName, "server") {
 			cmd += "-server"
 		}
@@ -356,7 +457,7 @@ func StartCluster(nodeNames []string) error {
 // StopCluster starts the rke2 service on each node given
 func StopCluster(nodeNames []string) error {
 	for _, nodeName := range nodeNames {
-		cmd := "sudo systemctl stop rke2*"
+		cmd := "systemctl stop rke2*"
 		if _, err := RunCmdOnNode(cmd, nodeName); err != nil {
 			return err
 		}

tests/e2e/vagrantdefaults.rb

@@ -25,4 +25,11 @@ def getInstallType(vm, version, branch)
 def cisPrep(vm)
   vm.provision "shell", inline: "useradd -r -c 'etcd user' -s /sbin/nologin -M etcd -U"
   vm.provision "shell", inline: "printf 'vm.panic_on_oom=0\nvm.overcommit_memory=1\nkernel.panic=10\nkernel.panic_on_oops=1' > /etc/sysctl.d/60-rke2-cis.conf; systemctl restart systemd-sysctl"
+end
+
+def loadManifests(vm, files)
+  vm.provision "Load extra manifests", type: "shell", inline: "mkdir -p /var/lib/rancher/rke2/server/manifests"
+  files.each do |file|
+    vm.provision "file", source: file, destination: "/var/lib/rancher/rke2/server/manifests/#{File.basename(file)}"
+  end
 end