test: Migrate all e2e test scripts to git submodule (#1489)

* gha test 1

* update submodule

* delete duplicate scripts and update test scope in matrix users

* moved generate ip env script and update submodule and pr workflow

.github/workflows/integration_tests.yml

@@ -46,15 +46,15 @@ jobs:
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_1 }}" >> $GITHUB_ENV
               ;;
             "USER_2")
-              echo "TEST_TAGS=firewall,firewalldevice,firewalls,image,images,instancenetworking,instancesharedips,instancetype,instancetypes,ipv6range,ipv6ranges,kernel,kernels,nb,nbconfig,nbconfigs,nbnode,nbs,sshkey,sshkeys" >> $GITHUB_ENV
+              echo "TEST_TAGS=firewall,firewalldevice,firewalls,image,images,instancenetworking,instancesharedips,instancetype,instancetypes,ipv6range,ipv6ranges,kernel,kernels,nb,nbconfig,nbconfigs,nbnode,nbs,sshkey,sshkeys,vlan,volume,volumes,vpc,vpcs" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_2 }}" >> $GITHUB_ENV
               ;;
             "USER_3")
               echo "TEST_TAGS=instanceconfig,instancedisk,instanceip,networkingip,objcluster,objkey,profile,rdns,region,regions,stackscript,stackscripts" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_3 }}" >> $GITHUB_ENV
               ;;
             "USER_4")
-              echo "TEST_TAGS=lke,lkeclusters,lkenodepool,lkeversions,obj,objbucket,token,user,users,vlan,volume,volumes,vpc,vpcs,placementgroup,placementgroups,placementgorupassignment" >> $GITHUB_ENV
+              echo "TEST_TAGS=lke,lkeclusters,lkenodepool,lkeversions,obj,objbucket,placementgroup,placementgroups,placementgorupassignment,token,user,users" >> $GITHUB_ENV
               echo "LINODE_TOKEN=${{ secrets.LINODE_TOKEN_USER_4 }}" >> $GITHUB_ENV
               ;;
           esac
@@ -105,17 +105,17 @@ jobs:
         run: rsync -av test-report-USER_*/* .
 
       - name: Merge Test Reports
-        run: python tod_scripts/merge_terraform_results.py
+        run: python e2e_scripts/tod_scripts/xml_to_obj_storage/terraform_tests/merge_terraform_results.py
 
       - name: Add Information to XML Report
         run: |
           filename=$(ls | grep -E '^[0-9]{12}_terraform_merged_report.xml$') 
-          python tod_scripts/add_to_xml_test_report.py --branch_name "${GITHUB_REF#refs/*/}" --gha_run_id "${GITHUB_RUN_ID}" --gha_run_number "${GITHUB_RUN_NUMBER}" --xmlfile "${filename}"
+          python e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py --branch_name "${GITHUB_REF#refs/*/}" --gha_run_id "${GITHUB_RUN_ID}" --gha_run_number "${GITHUB_RUN_NUMBER}" --xmlfile "${filename}"
 
       - name: Upload Test Results
         run: |
           filename=$(ls | grep -E '^[0-9]{12}_terraform_merged_report.xml$') 
-          python3 tod_scripts/test_report_upload_script.py "${filename}"
+          python e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}"
         env:
           LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }}
           LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}
@@ -143,6 +143,6 @@ jobs:
 
       - name: Apply Calico Rules to LKE # Only Running against Matrix USER 4 which includes LKE test suite
         run: |
-          cd scripts && ./lke_calico_rules_e2e.sh
+          cd e2e_scripts/cloud_security_scripts/lke_calico_rules/ && ./lke_calico_rules_e2e.sh
         env:
           LINODE_TOKEN: ${{ secrets.LINODE_TOKEN_USER_4 }}

.github/workflows/integration_tests_pr.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Apply Calico Rules to LKE
         if: always()
         run: |
-          cd scripts && ./lke_calico_rules_e2e.sh
+          cd e2e_scripts/cloud_security_scripts/lke_calico_rules/ && ./lke_calico_rules_e2e.sh
         env:
           LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}
 

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "tod_scripts"]
-	path = tod_scripts
-	url = https://github.com/linode/TOD-test-report-uploader.git
+[submodule "e2e_scripts"]
+	path = e2e_scripts
+	url = https://github.com/linode/dx-e2e-test-scripts

Makefile

@@ -70,7 +70,7 @@ int-test: fmt-check generate-ip-env-fw-e2e include-env
 	RUN_LONG_TESTS=$(RUN_LONG_TESTS) \
 	TF_VAR_ipv4_addr=${PUBLIC_IPV4} \
 	TF_VAR_ipv6_addr=${PUBLIC_IPV6} \
-	go test --tags="$(TEST_TAGS)" -v ./$(PKG_NAME) -count $(COUNT) -timeout $(TIMEOUT) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" $(ARGS) | grep -v "\[no test files\]"
+	go test --tags="$(TEST_TAGS)" -v ./$(PKG_NAME) -count $(COUNT) -timeout $(TIMEOUT) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" -parallel=$(PARALLEL) $(ARGS) | grep -v "\[no test files\]"
 
 .PHONY: include-env
 include-env: $(IP_ENV_FILE)
@@ -80,7 +80,7 @@ generate-ip-env-fw-e2e: $(IP_ENV_FILE)
 
 $(IP_ENV_FILE):
 	# Generate env file for E2E cloud firewall
-	. ./scripts/generate_ip_env_fw_e2e.sh
+	. ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh || touch $(IP_ENV_FILE)
 
 .PHONY: smoke-test
 smoke-test: fmt-check