test/workflow: Update test commands and related workflows (#1733)

* update test commands and related workflows

* Add comment and clean up test output

* add parallel input for test

* syntax fix

* syntax fix

* makefile and workflow fix

* address intermittent failures and add run long test param

* decrease default value of parallel execution to 5

.github/workflows/integration_tests.yml

@@ -1,7 +1,20 @@
 name: Integration Tests
 
 on:
-  workflow_dispatch: null
+  workflow_dispatch:
+    inputs:
+      parallel_value:
+        description: 'Number of parallel test executions for every user'
+        required: false
+      run_long_tests:
+        description: 'Choose whether to run long-duration tests that may take additional time. Set to "true" to enable long tests e.g. database related cases. Default is "false"'
+        required: false
+        default: 'false'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
+
   push:
     branches:
       - main
@@ -37,24 +50,24 @@ jobs:
       - name: Install go-junit-report
         run: go install github.com/jstemmer/go-junit-report/v2@latest
 
-      # Note: test tags are not in alphabetical order, it was divided to optimize test execution time
+      # Note: test suites are not in alphabetical order, it was divided to optimize test execution time
       - name: Set Test Scope for each Account
         run: |
           case "${{ matrix.user }}" in 
             "USER_1")
-              echo "TEST_TAGS=acceptance,backup,domain,domainrecord,domains,domainzonefile,helper,instance,provider" >> $GITHUB_ENV
+              echo "TEST_SUITE=acceptance,backup,domain,domainrecord,domains,domainzonefile,helper,instance,provider" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_1 }}" >> $GITHUB_ENV
               ;;
             "USER_2")
-              echo "TEST_TAGS=databasemysqlv2,firewall,firewalldevice,firewalls,image,images,instancenetworking,instancesharedips,instancetype,instancetypes,ipv6range,ipv6ranges,kernel,kernels,nb,nbconfig,nbconfigs,nbnode,nbs,sshkey,sshkeys,vlan,volume,volumes,vpc,vpcs,vpcsubnets" >> $GITHUB_ENV
+              echo "TEST_SUITE=databasemysqlv2,firewall,firewalldevice,firewalls,image,images,instancenetworking,instancesharedips,instancetype,instancetypes,ipv6range,ipv6ranges,kernel,kernels,nb,nbconfig,nbconfigs,nbnode,nbs,sshkey,sshkeys,vlan,volume,volumes,vpc,vpcs,vpcsubnets" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_2 }}" >> $GITHUB_ENV
               ;;
             "USER_3")
-              echo "TEST_TAGS=databasepostgresqlv2,instanceconfig,instancedisk,instanceip,networkingip,objcluster,objkey,profile,rdns,region,regions,stackscript,stackscripts" >> $GITHUB_ENV
+              echo "TEST_SUITE=databasepostgresqlv2,instanceconfig,instancedisk,instanceip,networkingip,objcluster,objkey,profile,rdns,region,regions,stackscript,stackscripts" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_3 }}" >> $GITHUB_ENV
               ;;
             "USER_4")
-              echo "TEST_TAGS=lke,lkeclusters,lkenodepool,lkeversions,obj,objbucket,placementgroup,placementgroups,placementgorupassignment,token,user,users" >> $GITHUB_ENV
+              echo "TEST_SUITE=lke,lkeclusters,lkenodepool,lkeversions,obj,objbucket,placementgroup,placementgroups,placementgorupassignment,token,user,users" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_4 }}" >> $GITHUB_ENV
               ;;
           esac
@@ -64,7 +77,7 @@ jobs:
 
       - name: Run Integration Tests
         run: |
-          make TEST_TAGS="${{ env.TEST_TAGS }}" int-test | go-junit-report -set-exit-code -iocopy -out $REPORT_FILENAME
+          make TEST_SUITE="${{ env.TEST_SUITE }}" PARALLEL="${{ github.event.inputs.parallel_value || '5' }}" test-int | go-junit-report -set-exit-code -iocopy -out $REPORT_FILENAME
         env:
           LINODE_TOKEN: ${{ env.LINODE_TOKEN }}
 
@@ -152,7 +165,7 @@ jobs:
           python-version: '3.x'
 
       - name: Install Python dependencies
-        run: pip3 install requests wheel boto3
+        run: pip3 install requests wheel boto3==1.35.99
 
       - name: Download test report
         uses: actions/download-artifact@v4

.github/workflows/integration_tests_pr.yml

@@ -2,20 +2,24 @@ on:
   pull_request:
   workflow_dispatch:
     inputs:
-      module:
-        description: 'The module to be tested.'
-        required: true
-        default: 'linode/...'
+      test_suite:
+        description: "Specify  test suite to run from inside 'linode/' directory. Examples: 'account', 'domain', etc. If not provided, all suites are executed"
+        required: false
+        default: 'integration'
       sha:
-        description: 'The hash value of the commit.'
+        description: 'Specify commit hash to test. This value is mandatory to ensure the tests run against a specific commit'
         required: true
       pull_request_number:
-        description: 'The number of the PR.'
+        description: 'Specify pull request number associated with the commit. Optional, but recommended when providing a commit hash (sha)'
         required: false
       run_long_tests:
-        description: 'Setting this value will skip long running tests (e.g. Database related cases): (true|false)'
+        description: 'Choose whether to run long-duration tests that may take additional time. Set to "true" to enable long tests e.g. database related cases. Default is "false"'
         required: false
-        default: false
+        default: 'false'
+        type: choice
+        options:
+          - 'true'
+          - 'false'
 
 
 name: Integration tests on PR
@@ -33,7 +37,7 @@ jobs:
       - uses: actions/github-script@v7
         id: disallowed-character-check
         env:
-          text: ${{ inputs.module }}
+          text: ${{ inputs.test_suite }}
         with:
           result-encoding: string
           script: |
@@ -55,7 +59,7 @@ jobs:
 
       - run: make deps
 
-      - run: make PKG_NAME="${{ inputs.module }}" int-test
+      - run: make TEST_SUITE="${{ inputs.test_suite }}" test-int
         if: ${{ steps.disallowed-character-check.outputs.result == 'pass' }}
         env:
           LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}

.github/workflows/nightly_smoke_tests.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Run smoke tests
         id: smoke_tests
         run: |
-          make smoke-test
+          make test-smoke
         env:
           LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}
 

.github/workflows/pull_request.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Lint
         run: make lint
       - name: Unit tests
-        run: make unit-test
+        run: make test-unit
       - name: Vet
         run: make vet
       - name: Tidy

.github/workflows/unit-tests.yml

@@ -22,4 +22,4 @@ jobs:
         run: make deps
 
       - name: Run unit tests
-        run: make unit-test
+        run: make test-unit

Makefile

@@ -1,15 +1,4 @@
-COUNT?=1
-PARALLEL?=10
-PKG_NAME=linode/...
-TIMEOUT?=240m
-RUN_LONG_TESTS?=false
-SWEEP?="tf_test,tf-test"
-TEST_TAGS="integration"
-
-MARKDOWNLINT_IMG := 06kellyjac/markdownlint-cli
-MARKDOWNLINT_TAG := 0.28.1
-
-IP_ENV_FILE := /tmp/linode/ip_vars.env
+SHELL := /bin/bash
 
 .PHONY: default
 default: build
@@ -57,55 +46,57 @@ vet:
 	golangci-lint run --disable-all --enable govet ./...
 
 .PHONY: test
-test: fmt-check smoke-test unit-test int-test
-
-.PHONY: unit-test
-unit-test: fmt-check
-	go test -v --tags=unit ./$(PKG_NAME)
-
-.PHONY: int-test
-int-test: fmt-check generate-ip-env-fw-e2e include-env
-	TF_ACC=1 \
-	LINODE_API_VERSION="v4beta" \
-	RUN_LONG_TESTS=$(RUN_LONG_TESTS) \
-	TF_VAR_ipv4_addr=${PUBLIC_IPV4} \
-	TF_VAR_ipv6_addr=${PUBLIC_IPV6} \
-	go test --tags="$(TEST_TAGS)" -v ./$(PKG_NAME) -count $(COUNT) -timeout $(TIMEOUT) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" -parallel=$(PARALLEL) $(ARGS)
+test: fmt-check test-unit test-smoke test-int
 
-.PHONY: include-env
-include-env: $(IP_ENV_FILE)
--include $(IP_ENV_FILE)
+.PHONY: test-unit
+test-unit: fmt-check
+	go test -v --tags=unit ./$(if $(PKG_NAME),linode/$(PKG_NAME),linode/...)
 
-generate-ip-env-fw-e2e: $(IP_ENV_FILE)
+IP_ENV_FILE = /tmp/linode/ip_vars.env
+SUBMODULE_DIR = e2e_scripts
+E2E_SCRIPT = ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh
 
-SUBMODULE_DIR := e2e_scripts
+# Generate IP env file
+.PHONY: generate-ip-env
+generate-ip-env: $(IP_ENV_FILE)
 
 $(IP_ENV_FILE):
-	# Generate env file for E2E cloud firewall
 	@if [ ! -d $(SUBMODULE_DIR) ]; then \
 		echo "Submodule directory $(SUBMODULE_DIR) does not exist. Updating submodules..."; \
 		git submodule update --init --recursive; \
-	else \
-		echo "Submodule directory $(SUBMODULE_DIR) already exists. Skipping update."; \
 	fi
-	. ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh
+	$(E2E_SCRIPT)
+
+# TEST_SUITE: Optional, specify a test suite (e.g. domain), Default to run everything if not set
+# TEST_ARGS: Optional, additional arguments for go test (e.g. -o -json, etc)
+# PKG_NAME: Recommended usage with TEST_CASE argument for faster execution e.g. make PKG_NAME="volume" TEST_CASE="TestAccResourceVolume_basic" test-int
+# TEST_CASE: Optional, specify a test case (e.g. 'TestAccResourceVolume_basic')
+
+# Integration Test
+.PHONY: test-int
+test-int: fmt-check generate-ip-env
+	\
+	TF_VAR_ipv4_addr=$(shell grep PUBLIC_IPV4 $(IP_ENV_FILE) | cut -d '=' -f2 | tr -d '[:space:]') \
+	TF_VAR_ipv6_addr=$(shell grep PUBLIC_IPV6 $(IP_ENV_FILE) | cut -d '=' -f2 | tr -d '[:space:]') \
+	TF_ACC=1 \
+	LINODE_API_VERSION="v4beta" \
+	RUN_LONG_TESTS=$(if $(RUN_LONG_TESTS),$(RUN_LONG_TESTS),false) \
+	bash -c 'set -o pipefail && go test --tags=$(if $(TEST_SUITE),$(TEST_SUITE),"integration") -v ./$(if $(PKG_NAME),linode/$(PKG_NAME),linode/...) \
+	-count $(if $(COUNT),$(COUNT),1) -timeout $(if $(TIMEOUT),$(TIMEOUT),240m) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" -parallel $(if $(PARALLEL),$(PARALLEL),10) $(if $(TEST_CASE),-run $(TEST_CASE)) $(if $(TEST_ARGS),$(TEST_ARGS)) | sed -e "/testing: warning: no tests to run/,+1d" -e "/\[no test files\]/d" -e "/\[no tests to run\]/d"'
 
-.PHONY: smoke-test
-smoke-test: fmt-check generate-ip-env-fw-e2e include-env
+.PHONY: test-smoke
+test-smoke: fmt-check generate-ip-env
+	\
 	TF_ACC=1 \
 	LINODE_API_VERSION="v4beta" \
 	RUN_LONG_TESTS=$(RUN_LONG_TESTS) \
-	TF_VAR_ipv4_addr=${PUBLIC_IPV4} \
-	TF_VAR_ipv6_addr=${PUBLIC_IPV6} \
+	TF_VAR_ipv4_addr=$(shell grep PUBLIC_IPV4 $(IP_ENV_FILE) | cut -d '=' -f2 | tr -d '[:space:]') \
+	TF_VAR_ipv6_addr=$(shell grep PUBLIC_IPV6 $(IP_ENV_FILE) | cut -d '=' -f2 | tr -d '[:space:]') \
 	bash -c 'set -o pipefail && go test -v ./linode/... -run TestSmokeTests -tags=integration \
-		-count $(COUNT) \
-		-timeout $(TIMEOUT) \
-		-parallel=$(PARALLEL) \
-		-ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" \
-		| sed -e "/testing: warning: no tests to run/,+1d" -e "/\[no test files\]/d" -e "/\[no tests to run\]/d"; \
-	exit_status=$$?; \
-	exit $$exit_status'
+	-count $(if $(COUNT),$(COUNT),1) -timeout $(if $(TIMEOUT),$(TIMEOUT),240m) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" -parallel $(if $(PARALLEL),$(PARALLEL),10) $(if $(TEST_ARGS),$(TEST_ARGS)) | sed -e "/testing: warning: no tests to run/,+1d" -e "/\[no test files\]/d" -e "/\[no tests to run\]/d"'
 
+MARKDOWNLINT_IMG := 06kellyjac/markdownlint-cli
+MARKDOWNLINT_TAG := 0.28.1
 
 .PHONY: docs-check
 docs-check:
@@ -116,8 +107,10 @@ docs-check:
 		--config .markdownlint.yml \
 		docs
 
+SWEEP?="tf_test,tf-test"
+
 .PHONY: sweep
 sweep:
 	# sweep cleans the test infra from your account
 	@echo "WARNING: This will destroy infrastructure. Use only in development accounts."
-	go test -v ./$(PKG_NAME) -sweep=$(SWEEP) $(ARGS)
+	go test -v ./$(if $(PKG_NAME),linode/$(PKG_NAME),linode/...) -sweep=$(SWEEP) $(TEST_ARGS)

README.md

@@ -54,24 +54,25 @@ make
 
 ### Testing the provider
 
-In order to run the full suite of Acceptance tests, run `make int-test`. Acceptance testing will require the `LINODE_TOKEN` variable to be populated with a Linode APIv4 Token.  See [Linode Provider documentation](https://www.terraform.io/docs/providers/linode/index.html) for more details.
+In order to run the full suite of Acceptance tests, run `make test-int`. Acceptance testing will require the `LINODE_TOKEN` variable to be populated with a Linode APIv4 Token.  See [Linode Provider documentation](https://www.terraform.io/docs/providers/linode/index.html) for more details.
 
 *Note:* Acceptance tests create real resources, and often cost money to run.
 
 ```sh
-make int-test
+make test-int
 ```
 
-Use the following command template to execute specific Acceptance test
+Use the following command template to execute specific Acceptance test, 
 
 ```shell
-make ARGS="-run TestAccResourceVolume_basic" int-test
+# PKG_NAME is the directory in linode/ that contains the corresponding TEST_CASE
+make PKG_NAME="volume" TEST_CASE="TestAccResourceVolume_basic" test-int
 ```
 
 Use the following command template to execute particular Acceptance tests within a specific package
 
 ```shell
-make TEST_TAGS="volume" int-test
+make TEST_SUITE="volume" test-int
 ```
 
 There are a number of useful flags and variables to aid in debugging.