From 7d781d597e9c80b35325245426d11d3164afb2fe Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-1@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:30:15 -0700 Subject: [PATCH] test: Refactor smoke tests and add improve nightly workflow (#1566) * refactor smoke tests and update Makefile and nightly_smoke_tests.yml * clean up output on smoke tests and add slack notifications in nightly_smoke_tests.yml * gha test 1 * gha test 2 * Fix makefile smoke-test command * add option to input sha value in workflow file * set env for nightly_smoke_tests.yml * add conditions to cron in nightly_smoke_tests.yml * fix syntax for slack message * setting pipefail status before piping output * set to bash terminal instead of sh * set to bash terminal instead of sh * Change slack notification * resolve gha issues * set slack webhook env * Change slack message * Change slack message * Fix slack payload * Fix slack payload # 2 * Fix slack payload # 3 * Fix slack payload # 3 * revert slack gha package * add submodule checks before running integration and smoke tests * addressing pr comment to remove global smoke test map variable for simplicity * fix syntax in slack message * Change slack notification * Add repo name in payload * Add repo name in slack payload * bump slackgha version and add slack notification to integration workflow * fix typo in workflow * add condition to send test results and notification only on main repository --- .github/workflows/integration_tests.yml | 70 +++++++++++++++++- .github/workflows/nightly_smoke_tests.yml | 84 ++++++++++++++++++++-- Makefile | 24 ++++++- linode/childaccount/datasource_test.go | 13 ++++ linode/databasepostgresql/resource_test.go | 13 ++++ linode/domain/datasource_test.go | 13 ++++ linode/domain/resource_test.go | 13 ++++ linode/firewall/framework_resource_test.go | 13 ++++ linode/firewalldevice/resource_test.go | 13 ++++ linode/instance/resource_test.go | 13 ++++ linode/instancedisk/resource_test.go | 13 ++++ linode/lke/framework_resource_test.go | 13 ++++ linode/nb/resource_test.go | 13 ++++ linode/objbucket/resource_test.go | 14 ++++ linode/regions/datasource_test.go | 13 ++++ linode/stackscript/resource_test.go | 13 ++++ linode/stackscripts/datasource_test.go | 13 ++++ linode/volume/resource_test.go | 13 ++++ linode/vpcsubnets/datasource_test.go | 13 ++++ 19 files changed, 377 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 2260b408b..c7777eb90 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -81,7 +81,7 @@ jobs: process-upload-report: runs-on: ubuntu-latest needs: [integration_tests] - if: always() # Run even if previous job fails + if: always() && github.repository == 'linode/terraform-provider-linode' # Run even if integration tests fail and only on main repository steps: - name: Checkout code @@ -145,4 +145,70 @@ jobs: run: | cd e2e_scripts/cloud_security_scripts/lke_calico_rules/ && ./lke_calico_rules_e2e.sh env: - LINODE_TOKEN: ${{ secrets.LINODE_TOKEN_USER_4 }} \ No newline at end of file + LINODE_TOKEN: ${{ secrets.LINODE_TOKEN_USER_4 }} + + notify-slack: + runs-on: ubuntu-latest + needs: [integration_tests] + if: always() && github.repository == 'linode/terraform-provider-linode' # Run even if integration tests fail and only on main repository + + steps: + - name: Notify Slack + uses: slackapi/slack-github-action@v1.27.0 + with: + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Build Result:*\n${{ steps.integration_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}" + }, + { + "type": "mrkdwn", + "text": "*Branch:*\n`${{ github.ref_name }}`" + } + ] + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" + }, + { + "type": "mrkdwn", + "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>" + } + ] + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`" + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/nightly_smoke_tests.yml b/.github/workflows/nightly_smoke_tests.yml index 076f3e2d0..21f12d1cc 100644 --- a/.github/workflows/nightly_smoke_tests.yml +++ b/.github/workflows/nightly_smoke_tests.yml @@ -4,25 +4,99 @@ on: schedule: - cron: "0 0 * * *" workflow_dispatch: + inputs: + sha: + description: 'Commit SHA to test' + required: false + default: '' + type: string jobs: smoke_tests: + if: github.repository == 'linode/terraform-provider-linode' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 with: - ref: dev + fetch-depth: 0 + submodules: 'recursive' + ref: ${{ github.event.inputs.sha || github.ref }} - - name: Set up go + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 'stable' + go-version: '1.x' - - run: go version + - name: Install Dependencies + run: | + make deps - name: Run smoke tests - run: make smoke-test + id: smoke_tests + run: | + make smoke-test env: LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }} + + - name: Notify Slack + if: always() + uses: slackapi/slack-github-action@v1.27.0 + with: + channel-id: ${{ secrets.SLACK_CHANNEL_ID }} + payload: | + { + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Build Result:*\n${{ steps.smoke_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}" + }, + { + "type": "mrkdwn", + "text": "*Branch:*\n`${{ github.ref_name }}`" + } + ] + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" + }, + { + "type": "mrkdwn", + "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>" + } + ] + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`" + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/Makefile b/Makefile index 7e3c93403..9e325c27f 100644 --- a/Makefile +++ b/Makefile @@ -78,16 +78,34 @@ include-env: $(IP_ENV_FILE) generate-ip-env-fw-e2e: $(IP_ENV_FILE) +SUBMODULE_DIR := e2e_scripts + $(IP_ENV_FILE): # Generate env file for E2E cloud firewall - . ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh || touch $(IP_ENV_FILE) + @if [ ! -d $(SUBMODULE_DIR) ]; then \ + echo "Submodule directory $(SUBMODULE_DIR) does not exist. Updating submodules..."; \ + git submodule update --init --recursive; \ + else \ + echo "Submodule directory $(SUBMODULE_DIR) already exists. Skipping update."; \ + fi + . ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh .PHONY: smoke-test -smoke-test: fmt-check +smoke-test: fmt-check generate-ip-env-fw-e2e include-env TF_ACC=1 \ LINODE_API_VERSION="v4beta" \ RUN_LONG_TESTS=$(RUN_LONG_TESTS) \ - go test -v -run smoke ./linode/... -count $(COUNT) -timeout $(TIMEOUT) -parallel=$(PARALLEL) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" + TF_VAR_ipv4_addr=${PUBLIC_IPV4} \ + TF_VAR_ipv6_addr=${PUBLIC_IPV6} \ + bash -c 'set -o pipefail && go test -v ./linode/... -run TestSmokeTests -tags=integration \ + -count $(COUNT) \ + -timeout $(TIMEOUT) \ + -parallel=$(PARALLEL) \ + -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" \ + | sed -e "/testing: warning: no tests to run/,+1d" -e "/\[no test files\]/d" -e "/\[no tests to run\]/d"; \ + exit_status=$$?; \ + exit $$exit_status' + .PHONY: docs-check docs-check: diff --git a/linode/childaccount/datasource_test.go b/linode/childaccount/datasource_test.go index b800ff486..3f0dac7c9 100644 --- a/linode/childaccount/datasource_test.go +++ b/linode/childaccount/datasource_test.go @@ -10,6 +10,19 @@ import ( "github.com/linode/terraform-provider-linode/v2/linode/childaccount/tmpl" ) +func TestSmokeTests_childaccount(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccDataSourceChildAccount_basic_smoke", TestAccDataSourceChildAccount_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccDataSourceChildAccount_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/databasepostgresql/resource_test.go b/linode/databasepostgresql/resource_test.go index afa66a1ec..518acd7d4 100644 --- a/linode/databasepostgresql/resource_test.go +++ b/linode/databasepostgresql/resource_test.go @@ -75,6 +75,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_databasepostgresql(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceDatabasePostgres_basic_smoke", TestAccResourceDatabasePostgres_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceDatabasePostgres_basic_smoke(t *testing.T) { acceptance.LongRunningTest(t) diff --git a/linode/domain/datasource_test.go b/linode/domain/datasource_test.go index 8a7f81f0f..2b54315d6 100644 --- a/linode/domain/datasource_test.go +++ b/linode/domain/datasource_test.go @@ -12,6 +12,19 @@ import ( "github.com/linode/terraform-provider-linode/v2/linode/domain/tmpl" ) +func TestSmokeTests_domain_datasource(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccDataSourceDomain_basic_smoke", TestAccDataSourceDomain_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccDataSourceDomain_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/domain/resource_test.go b/linode/domain/resource_test.go index b39a74d02..059939c67 100644 --- a/linode/domain/resource_test.go +++ b/linode/domain/resource_test.go @@ -48,6 +48,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_domain_resource(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceDomain_basic_smoke", TestAccResourceDomain_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceDomain_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/firewall/framework_resource_test.go b/linode/firewall/framework_resource_test.go index 26f126205..6db229064 100644 --- a/linode/firewall/framework_resource_test.go +++ b/linode/firewall/framework_resource_test.go @@ -57,6 +57,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_firewall(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccLinodeFirewall_basic", TestAccLinodeFirewall_basic}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccLinodeFirewall_basic(t *testing.T) { t.Parallel() diff --git a/linode/firewalldevice/resource_test.go b/linode/firewalldevice/resource_test.go index e595db960..0ac2dacea 100644 --- a/linode/firewalldevice/resource_test.go +++ b/linode/firewalldevice/resource_test.go @@ -29,6 +29,19 @@ func init() { testRegion = region } +func TestSmokeTests_firewalldevice(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceFirewallDevice_basic_smoke", TestAccResourceFirewallDevice_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceFirewallDevice_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/instance/resource_test.go b/linode/instance/resource_test.go index 1fff21aae..db0d77e16 100644 --- a/linode/instance/resource_test.go +++ b/linode/instance/resource_test.go @@ -64,6 +64,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_instance(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceInstance_basic_smoke", TestAccResourceInstance_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceInstance_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/instancedisk/resource_test.go b/linode/instancedisk/resource_test.go index 3755e8bd4..7d79b5688 100644 --- a/linode/instancedisk/resource_test.go +++ b/linode/instancedisk/resource_test.go @@ -29,6 +29,19 @@ func init() { testRegion = region } +func TestSmokeTests_instancedisk(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceInstanceDisk_basic_smoke", TestAccResourceInstanceDisk_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceInstanceDisk_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/lke/framework_resource_test.go b/linode/lke/framework_resource_test.go index 4dbcc2852..72218b1b5 100644 --- a/linode/lke/framework_resource_test.go +++ b/linode/lke/framework_resource_test.go @@ -164,6 +164,19 @@ func waitForAllNodesReady(t *testing.T, cluster *linodego.LKECluster, pollInterv } } +func TestSmokeTests_lke(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceLKECluster_basic_smoke", TestAccResourceLKECluster_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceLKECluster_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/nb/resource_test.go b/linode/nb/resource_test.go index ed8443c2f..1c2554e71 100644 --- a/linode/nb/resource_test.go +++ b/linode/nb/resource_test.go @@ -62,6 +62,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_nb(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceNodeBalancer_basic_smoke", TestAccResourceNodeBalancer_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceNodeBalancer_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/objbucket/resource_test.go b/linode/objbucket/resource_test.go index 9277bf8f3..95c22d4f3 100644 --- a/linode/objbucket/resource_test.go +++ b/linode/objbucket/resource_test.go @@ -171,6 +171,20 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_objbucket(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceBucket_basic_legacy_smoke", TestAccResourceBucket_basic_legacy_smoke}, + {"TestAccResourceBucket_basic_smoke", TestAccResourceBucket_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceBucket_basic_legacy_smoke(t *testing.T) { t.Parallel() diff --git a/linode/regions/datasource_test.go b/linode/regions/datasource_test.go index 36faa22a3..789809879 100644 --- a/linode/regions/datasource_test.go +++ b/linode/regions/datasource_test.go @@ -13,6 +13,19 @@ import ( "github.com/linode/terraform-provider-linode/v2/linode/regions/tmpl" ) +func TestSmokeTests_firewall(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccDataSourceRegions_basic_smoke", TestAccDataSourceRegions_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccDataSourceRegions_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/stackscript/resource_test.go b/linode/stackscript/resource_test.go index b22e4a3ea..09e077741 100644 --- a/linode/stackscript/resource_test.go +++ b/linode/stackscript/resource_test.go @@ -48,6 +48,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_stackscript(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceStackscript_basic_smoke", TestAccResourceStackscript_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceStackscript_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/stackscripts/datasource_test.go b/linode/stackscripts/datasource_test.go index f5e894787..ba10ca9a4 100644 --- a/linode/stackscripts/datasource_test.go +++ b/linode/stackscripts/datasource_test.go @@ -17,6 +17,19 @@ var basicStackScript = `#!/bin/bash echo "Hello, $NAME!" ` +func TestSmokeTests_stackscripts(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccDataSourceStackscripts_basic_smoke", TestAccDataSourceStackscripts_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccDataSourceStackscripts_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/volume/resource_test.go b/linode/volume/resource_test.go index c117f4f6b..4cf2e4094 100644 --- a/linode/volume/resource_test.go +++ b/linode/volume/resource_test.go @@ -56,6 +56,19 @@ func sweep(prefix string) error { return nil } +func TestSmokeTests_volume(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccResourceVolume_basic_smoke", TestAccResourceVolume_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccResourceVolume_basic_smoke(t *testing.T) { t.Parallel() diff --git a/linode/vpcsubnets/datasource_test.go b/linode/vpcsubnets/datasource_test.go index 8d1b6f249..91c01e632 100644 --- a/linode/vpcsubnets/datasource_test.go +++ b/linode/vpcsubnets/datasource_test.go @@ -13,6 +13,19 @@ import ( "github.com/linode/terraform-provider-linode/v2/linode/vpcsubnets/tmpl" ) +func TestSmokeTests_vpcsubnets(t *testing.T) { + tests := []struct { + name string + test func(*testing.T) + }{ + {"TestAccDataSourceVPCSubnets_basic_smoke", TestAccDataSourceVPCSubnets_basic_smoke}, + } + + for _, tt := range tests { + t.Run(tt.name, tt.test) + } +} + func TestAccDataSourceVPCSubnets_basic_smoke(t *testing.T) { t.Parallel()