From 48a33d531e65e8051ad7653121cfdb2b0ed1ab87 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 8 Oct 2024 20:11:36 -0300 Subject: [PATCH 1/6] workflows: add AWS e2e tests Created a callable workflow for running the AWS e2e tests. This initial implementation has support for testing mkosi or packer based images, being default the later. The cluster_type has only support to "onprem" cluster, and the workflow will create a kcli-based kubeadm one. Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/e2e_aws.yaml | 174 +++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 .github/workflows/e2e_aws.yaml diff --git a/.github/workflows/e2e_aws.yaml b/.github/workflows/e2e_aws.yaml new file mode 100644 index 000000000..8452fb00f --- /dev/null +++ b/.github/workflows/e2e_aws.yaml @@ -0,0 +1,174 @@ +# (C) Copyright Confidential Containers Contributors 2025. +# SPDX-License-Identifier: Apache-2.0 +# +# Run aws e2e tests. +name: (Callable) aws e2e tests + +on: + workflow_call: + inputs: + podvm_image: + required: true + type: string + caa_image: + required: true + type: string + git_ref: + default: 'main' + description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. + required: false + type: string + oras: + description: Whether the podvm_image is oras published + default: false + required: false + type: boolean + cluster_type: + description: Specify the cluster type. Accepted values are "onprem" or "eks". + default: onprem + required: false + type: string + container_runtime: + default: 'containerd' + description: Name of the container runtime. Either containerd or crio. + required: false + type: string +env: + CLOUD_PROVIDER: aws + DEBIAN_FRONTEND: noninteractive + +jobs: + # Check the org/repository has AWS secrets (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY). On absence of + # secrets it should skip the execution of the test job. + aws-credentials: + runs-on: ubuntu-22.04 + outputs: + has_secrets: ${{ steps.check_secrets.outputs.has_secrets }} + steps: + - name: Check secrets + id: check_secrets + run: | + if [[ -n "${{ secrets.AWS_ACCESS_KEY_ID }}" && -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then + echo "has_secrets=true" >> "$GITHUB_OUTPUT" + else + echo "has_secrets=false" >> "$GITHUB_OUTPUT" + fi + + test-aws: + needs: aws-credentials + if: needs.aws-credentials.outputs.has_secrets == 'true' + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: src/cloud-api-adaptor + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - name: Rebase the code + if: github.event_name == 'pull_request_target' + working-directory: ./ + run: | + ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch + + - name: Read properties from versions.yaml + run: | + sudo snap install yq + go_version="$(yq '.tools.golang' versions.yaml)" + [ -n "$go_version" ] + echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV" + echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV" + + - name: Setup Golang version ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: oras-project/setup-oras@v1 + with: + version: ${{ env.ORAS_VERSION }} + + - name: Extract qcow2 from ${{ inputs.podvm_image }} + if: ${{ !inputs.oras }} + run: | + qcow2=$(echo "${{ inputs.podvm_image }}" | sed -e "s#.*/\(.*\):.*#\1.qcow2#") + ./hack/download-image.sh "${{ inputs.podvm_image }}" . -o "${qcow2}" + echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV" + # Clean up docker images to make space + docker system prune -a -f + working-directory: src/cloud-api-adaptor/podvm + + - name: Use oras to get qcow2 from ${{ inputs.podvm_image }} + if: ${{ inputs.oras }} + run: | + oras pull ${{ inputs.podvm_image }} + tar xvJpf podvm.tar.xz + qcow2=$(find ./*.qcow2) + echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV" + working-directory: src/cloud-api-adaptor/podvm + + - name: Install kustomize + run: | + command -v kustomize >/dev/null || \ + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \ + sudo bash -s /usr/local/bin + + - name: Update kustomization configuration + run: | + cd "install/overlays/aws" + kustomize edit set image "cloud-api-adaptor=${{ inputs.caa_image }}" + # Print for debugging + echo "::group::aws kustomization" + cat kustomization.yaml + echo "::endgroup::" + + - name: Config aws + run: | + cat <>aws.properties + CAA_IMAGE="${{ inputs.caa_image }}" + disablecvm="true" + cluster_type="${{ inputs.cluster_type }}" + ssh_kp_name="caa-e2e-test" + EOF + # For debugging + echo "::group::aws.properties" + cat aws.properties + echo "::endgroup::" + + # Note: aws cli is already installed on github's runner image. + - name: Config aws CLI + run: | + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} --profile default + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} --profile default + aws configure set region us-east-1 --profile default + + - name: Create on-prem cluster + if: inputs.cluster_type == 'onprem' + run: | + # Let's use kcli to build a kubeadm cluster for us + echo "::group::Configure libvirt" + ./libvirt/config_libvirt.sh + # Add the kcli install directory to PATH for later steps + echo "${HOME}/.local/bin" >> "$GITHUB_PATH" + echo "::endgroup::" + export CONTAINER_RUNTIME=${{ inputs.container_runtime }} + echo "::group::Create cluster with $CONTAINER_RUNTIME" + ./libvirt/kcli_cluster.sh create + echo "KUBECONFIG=$HOME/.kcli/clusters/peer-pods/auth/kubeconfig" >> "$GITHUB_ENV" + echo "::endgroup::" + + - name: run tests + id: runTests + run: | + export CLOUD_PROVIDER=aws + export DEPLOY_KBS=false + export TEST_PROVISION="yes" + export TEST_TEARDOWN="yes" + export TEST_PROVISION_FILE="$PWD/aws.properties" + export TEST_PODVM_IMAGE="${{ env.PODVM_QCOW2 }}" + export TEST_E2E_TIMEOUT="90m" + + make test-e2e From 5ccfb3f9968c59ebdd09c34f05ea47a615bfd2b9 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 28 Jan 2025 16:48:01 -0300 Subject: [PATCH 2/6] workflows: enable aws e2e test in e2e_run_all The new created e2e_aws is called by the e2e_run_all, so AWS e2e tests will run on nightly. At this point it won't be triggered by pull request. It's testing the packer based podvm images. Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/e2e_run_all.yaml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index fd06a0a73..fb04d779d 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -122,7 +122,7 @@ jobs: outputs: matrix: ${{ steps.matrix.outputs.matrix }} env: - PROVIDERS: "docker libvirt" + PROVIDERS: "aws docker libvirt" steps: - name: Checkout Code uses: actions/checkout@v4 @@ -174,6 +174,28 @@ jobs: id: matrix run: | echo "matrix=$(jq -c . < ./libvirt/e2e_matrix_libvirt.json)" >> "$GITHUB_OUTPUT" + # Run AWS e2e tests + aws: + name: aws + if: | + github.event_name == 'workflow_dispatch' + needs: [podvm, image, prep_install] + strategy: + fail-fast: false + matrix: + os: + - ubuntu + provider: + - generic + arch: + - amd64 + uses: ./.github/workflows/e2e_aws.yaml + with: + caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }} + podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }} + git_ref: ${{ inputs.git_ref }} + oras: false + secrets: inherit # Run libvirt e2e tests if pull request labeled 'test_e2e_libvirt' libvirt: From fc6d7eb1dbbae3d6e8650a718c8940c44839d690 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Wed, 29 Jan 2025 17:08:52 -0300 Subject: [PATCH 3/6] test/provisioner: ensure AWS resources are name tagged Tagging with "Name" all the AWS resources created to help on tracking and removal of them all, mainly when running on CI. In order to tag images I had to bump github.com/aws/aws-sdk-go-v2/service/ec2 which cascated to updating other modules. Signed-off-by: Wainer dos Santos Moschetta --- src/cloud-api-adaptor/go.mod | 26 ++--- src/cloud-api-adaptor/go.sum | 49 ++++----- .../test/provisioner/aws/provision_common.go | 99 +++++++------------ .../test/provisioner/aws/utils.go | 24 +++++ 4 files changed, 96 insertions(+), 102 deletions(-) create mode 100644 src/cloud-api-adaptor/test/provisioner/aws/utils.go diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod index 06d110661..6a48cf4af 100644 --- a/src/cloud-api-adaptor/go.mod +++ b/src/cloud-api-adaptor/go.mod @@ -10,10 +10,10 @@ require ( github.com/IBM/go-sdk-core/v5 v5.13.1 github.com/IBM/platform-services-go-sdk v0.36.0 // indirect github.com/IBM/vpc-go-sdk v0.35.0 - github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2 v1.34.0 github.com/aws/aws-sdk-go-v2/config v1.15.11 github.com/aws/aws-sdk-go-v2/credentials v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.202.0 github.com/containerd/containerd v1.7.16 github.com/containerd/ttrpc v1.2.3 github.com/coreos/go-iptables v0.6.0 @@ -44,8 +44,8 @@ require ( github.com/IBM/ibm-cos-sdk-go v1.9.4 github.com/avast/retry-go/v4 v4.5.1 github.com/aws/aws-sdk-go-v2/service/eks v1.29.5 - github.com/aws/aws-sdk-go-v2/service/iam v1.22.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 + github.com/aws/aws-sdk-go-v2/service/iam v1.38.8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.75.0 github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers v0.12.0 github.com/confidential-containers/cloud-api-adaptor/src/peerpod-ctrl v0.12.0 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f @@ -85,19 +85,19 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.12.0-rc.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 // indirect - github.com/aws/smithy-go v1.17.0 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/containerd/fifo v1.1.0 // indirect diff --git a/src/cloud-api-adaptor/go.sum b/src/cloud-api-adaptor/go.sum index 05fcd56fe..420741a35 100644 --- a/src/cloud-api-adaptor/go.sum +++ b/src/cloud-api-adaptor/go.sum @@ -82,10 +82,11 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= -github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU= +github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= @@ -93,40 +94,42 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Ob github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0 h1:Yq39vbwQX+Xw+Ubcsg/ElwO+TWAxAIAdrREtpjGnCHw= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.117.0/go.mod h1:0FhI2Rzcv5BNM3dNnbcCx2qa2naFZoAidJi11cQgzL0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.202.0 h1:/kB9Uf7fgpYNLvwhAW0YiDSg7xQyxB6MbEYoC0yXtjs= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.202.0/go.mod h1:cRD0Fhzj0YD+uAh16NChQAv9/BB0S9x3YK9hLx1jb/k= github.com/aws/aws-sdk-go-v2/service/eks v1.29.5 h1:6eSpTHOsDixcFIvPdiAAVdyCru3k2jIVRPdIQfGzfc8= github.com/aws/aws-sdk-go-v2/service/eks v1.29.5/go.mod h1:TwqefcyPlF31NTF+fH34tJ2VwMMR6c74IbiiUgA6kVY= -github.com/aws/aws-sdk-go-v2/service/iam v1.22.5 h1:qGv+oW4uV1T3kbE9uSYEfdZbo38OqxgRxxfStfDr4BU= -github.com/aws/aws-sdk-go-v2/service/iam v1.22.5/go.mod h1:8lyPrjQczmx72ac9s82zTjf9xLqs7uuFMG9TVEZ07XU= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.8 h1:+PjS9gfr15U+MaUafN89dWxhbsvVrJg2D1umkc8R4uA= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.8/go.mod h1:V7xF4f2fgf9GSVxTqeYQz7bNu8AITVsgqP6otlHzjPs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.75.0 h1:UPQJDyqUXICUt60X4PwbiEf+2QQ4VfXUhDk8OEiGtik= +github.com/aws/aws-sdk-go-v2/service/s3 v1.75.0/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I= github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= -github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= diff --git a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go index 15c1ec6e9..23f8193c7 100644 --- a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go @@ -35,6 +35,7 @@ const ( EksCniAddonVersion = "v1.12.5-eksbuild.2" EksVersion = "1.26" AwsCredentialsFile = "aws-cred.env" + ResourcesBaseName = "caa-e2e-test" ) var AWSProps = &AWSProvisioner{} @@ -338,7 +339,7 @@ func NewVpc(client *ec2.Client, properties map[string]string) *Vpc { } return &Vpc{ - BaseName: "caa-e2e-test", + BaseName: ResourcesBaseName, CidrBlock: cidrBlock, Client: client, ID: properties["aws_vpc_id"], @@ -353,18 +354,8 @@ func NewVpc(client *ec2.Client, properties map[string]string) *Vpc { // createVpc creates the VPC func (v *Vpc) createVpc() error { vpc, err := v.Client.CreateVpc(context.TODO(), &ec2.CreateVpcInput{ - CidrBlock: aws.String(v.CidrBlock), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeVpc, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-vpc"), - }, - }, - }, - }, + CidrBlock: aws.String(v.CidrBlock), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-vpc", ec2types.ResourceTypeVpc), }) if err != nil { return err @@ -378,19 +369,9 @@ func (v *Vpc) createVpc() error { func (v *Vpc) createSubnet() error { subnet, err := v.Client.CreateSubnet(context.TODO(), &ec2.CreateSubnetInput{ - VpcId: aws.String(v.ID), - CidrBlock: aws.String("10.0.0.0/25"), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeSubnet, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-subnet"), - }, - }, - }, - }, + VpcId: aws.String(v.ID), + CidrBlock: aws.String("10.0.0.0/25"), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-subnet", ec2types.ResourceTypeSubnet), }) if err != nil { @@ -437,20 +418,10 @@ func (v *Vpc) createSecondarySubnet() error { subnet, err := v.Client.CreateSubnet(context.TODO(), &ec2.CreateSubnetInput{ - AvailabilityZone: aws.String(secondarySubnetAz), - VpcId: aws.String(v.ID), - CidrBlock: aws.String("10.0.0.128/25"), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeSubnet, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-subnet-2"), - }, - }, - }, - }, + AvailabilityZone: aws.String(secondarySubnetAz), + VpcId: aws.String(v.ID), + CidrBlock: aws.String("10.0.0.128/25"), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-subnet-2", ec2types.ResourceTypeSubnet), }) if err != nil { @@ -477,17 +448,7 @@ func (v *Vpc) setupVpcNetworking() error { if igwOutput, err = v.Client.CreateInternetGateway(context.TODO(), &ec2.CreateInternetGatewayInput{ - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeInternetGateway, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-igw"), - }, - }, - }, - }, + TagSpecifications: defaultTagSpecifications(v.BaseName+"-igw", ec2types.ResourceTypeInternetGateway), }); err != nil { return err } @@ -503,18 +464,8 @@ func (v *Vpc) setupVpcNetworking() error { if rtOutput, err = v.Client.CreateRouteTable(context.TODO(), &ec2.CreateRouteTableInput{ - VpcId: aws.String(v.ID), - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeRouteTable, - Tags: []ec2types.Tag{ - { - Key: aws.String("Name"), - Value: aws.String(v.BaseName + "-rtb"), - }, - }, - }, - }, + VpcId: aws.String(v.ID), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-rtb", ec2types.ResourceTypeRouteTable), }); err != nil { return err } @@ -544,9 +495,10 @@ func (v *Vpc) setupVpcNetworking() error { func (v *Vpc) setupSecurityGroup() error { if sgOutput, err := v.Client.CreateSecurityGroup(context.TODO(), &ec2.CreateSecurityGroupInput{ - Description: aws.String("cloud-api-adaptor e2e tests"), - GroupName: aws.String(v.BaseName + "-sg"), - VpcId: aws.String(v.ID), + Description: aws.String("cloud-api-adaptor e2e tests"), + GroupName: aws.String(v.BaseName + "-sg"), + VpcId: aws.String(v.ID), + TagSpecifications: defaultTagSpecifications(v.BaseName+"-sg", ec2types.ResourceTypeSecurityGroup), }); err != nil { return err } else { @@ -873,6 +825,7 @@ func (i *AMIImage) importEBSSnapshot(bucket *S3Bucket) error { S3Key: aws.String(bucket.Key), }, }, + TagSpecifications: defaultTagSpecifications(ResourcesBaseName+"-snap", ec2types.ResourceTypeImportSnapshotTask), }) if err != nil { return err @@ -897,6 +850,19 @@ func (i *AMIImage) importEBSSnapshot(bucket *S3Bucket) error { taskDetail := describeTasks.ImportSnapshotTasks[0].SnapshotTaskDetail i.EBSSnapshotId = *taskDetail.SnapshotId + // Let's warn but ignore any tagging error + if _, err = i.Client.CreateTags(context.TODO(), &ec2.CreateTagsInput{ + Resources: []string{i.EBSSnapshotId}, + Tags: []ec2types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(ResourcesBaseName + "-snap"), + }, + }, + }); err != nil { + log.Warnf("Failed to tag EBS snapshot %s: %v", i.EBSSnapshotId, err) + } + return nil } @@ -921,6 +887,7 @@ func (i *AMIImage) registerImage(imageName string) error { EnaSupport: aws.Bool(true), RootDeviceName: aws.String(i.RootDeviceName), VirtualizationType: aws.String("hvm"), + TagSpecifications: defaultTagSpecifications(ResourcesBaseName+"-img", ec2types.ResourceTypeImage), }) if err != nil { return err diff --git a/src/cloud-api-adaptor/test/provisioner/aws/utils.go b/src/cloud-api-adaptor/test/provisioner/aws/utils.go new file mode 100644 index 000000000..d0f3e3c23 --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/aws/utils.go @@ -0,0 +1,24 @@ +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +// defaultTagSpecifications returns a tag specifications array with only one element and one "Name" tag. +func defaultTagSpecifications(name string, resourceType ec2types.ResourceType) []ec2types.TagSpecification { + return []ec2types.TagSpecification{ + { + ResourceType: resourceType, + Tags: []ec2types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(name), + }, + }, + }, + } +} From b564731d77c93de3f6f44fb4c7e1bf3cd01201f5 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Wed, 29 Jan 2025 17:26:35 -0300 Subject: [PATCH 4/6] test/provision: add timestamp to created AWS AMIs So to generate unique names to avoid clashing published images. Signed-off-by: Wainer dos Santos Moschetta --- .../test/provisioner/aws/provision_common.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go index 23f8193c7..ffa03278a 100644 --- a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "text/template" "time" @@ -321,7 +322,8 @@ func (a *AWSProvisioner) UploadPodvm(imagePath string, ctx context.Context, cfg return err } - imageName := strings.Replace(filepath.Base(imagePath), ".qcow2", ".raw", 1) + imageNameSuffix := "-" + strconv.FormatInt(time.Now().Unix(), 10) + imageName := strings.Replace(filepath.Base(imagePath), ".qcow2", imageNameSuffix, 1) log.Infof("Register image with name: %s", imageName) err = a.Image.registerImage(imageName) if err != nil { From 57f0d7743c170650e2f3b5150d8b176da514cf99 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Thu, 30 Jan 2025 21:12:08 -0300 Subject: [PATCH 5/6] test: add function to deregister AWS AMI Image So that on VPC teardown (if enabled) the created AMI will be deleted along with its corresponding EBS snapshot. Signed-off-by: Wainer dos Santos Moschetta --- .../test/provisioner/aws/provision_common.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go index ffa03278a..a8f9f3a0d 100644 --- a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go @@ -255,6 +255,10 @@ func (a *AWSProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) err } } + if a.Image.ID != "" || a.Image.EBSSnapshotId != "" { + a.Image.deregisterImage() + } + return nil } @@ -900,6 +904,34 @@ func (i *AMIImage) registerImage(imageName string) error { return nil } +// deregisterImage Deregisters an AMI image. The associated EBS snapshot is deleted too. +func (i *AMIImage) deregisterImage() error { + var err error + + if i.ID != "" { + log.Infof("Deregister AMI ID: %s", i.ID) + _, err = i.Client.DeregisterImage(context.TODO(), &ec2.DeregisterImageInput{ + ImageId: aws.String(i.ID), + }) + if err != nil { + log.Errorf("Failed to deregister AMI: %s", err) + } + } + + // Removing the EBS snapshot + if i.EBSSnapshotId != "" { + log.Infof("Delete Snapshot ID: %s", i.EBSSnapshotId) + i.Client.DeleteSnapshot(context.TODO(), &ec2.DeleteSnapshotInput{ + SnapshotId: aws.String(i.EBSSnapshotId), + }) + if err != nil { + log.Errorf("Failed to delete snapshot: %s", err) + } + } + + return err +} + // uploadLargeFileWithCli Uploads large files (>5GB) using the AWS CLI func (b *S3Bucket) uploadLargeFileWithCli(filepath string) error { file, err := os.Open(filepath) From e88f53f2f91fa20227b2b331b24c445beb5603a6 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 31 Jan 2025 14:53:44 -0300 Subject: [PATCH 6/6] test: add function to delete AWS bucket key Delete the key from bucket that contains the raw disk image. Signed-off-by: Wainer dos Santos Moschetta --- .../test/provisioner/aws/provision_common.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go index a8f9f3a0d..5bbbc5fde 100644 --- a/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/aws/provision_common.go @@ -259,6 +259,11 @@ func (a *AWSProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) err a.Image.deregisterImage() } + if a.Bucket.Key != "" { + log.Infof("Delete key %s from bucket: %s", a.Bucket.Key, a.Bucket.Name) + a.Bucket.deleteKey() + } + return nil } @@ -964,6 +969,17 @@ func (b *S3Bucket) uploadLargeFileWithCli(filepath string) error { return nil } +func (b *S3Bucket) deleteKey() error { + if _, err := b.Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(b.Name), + Key: aws.String(b.Key), + }); err != nil { + return err + } + + return nil +} + // ConvertQcow2ToRaw Converts an qcow2 image to raw. Requires `qemu-img` installed. func ConvertQcow2ToRaw(qcow2 string, raw string) error { cmd := exec.Command("qemu-img", "convert", "-O", "raw", qcow2, raw)