diff --git a/apiclient/rancher_api/api.py b/apiclient/rancher_api/api.py index 8bfb38aa9..afee9aeb1 100644 --- a/apiclient/rancher_api/api.py +++ b/apiclient/rancher_api/api.py @@ -86,8 +86,8 @@ def authenticate(self, user, passwd, **kwargs): except AssertionError: pass # TODO: Log authenticate error else: - token = "Bearer %s" % r.json()['token'] - self.session.headers.update(Authorization=token) + self.token = r.json()['token'] + self.session.headers.update(Authorization=f"Bearer {self.token}") self._version = None return r.json() diff --git a/apiclient/rancher_api/managers.py b/apiclient/rancher_api/managers.py index 1f780ed7e..5826fd6e2 100644 --- a/apiclient/rancher_api/managers.py +++ b/apiclient/rancher_api/managers.py @@ -325,7 +325,12 @@ def create(self, name, kubeconfig, cluster_id="", *, raw=False): def get(self, name="", *, raw=False): if name == "": return self._get(self.PATH_fmt.format(uid=""), raw=raw) - return self._get(self.PATH_fmt.format(uid=f"/{name}"), raw=raw) + + code, data = self._get(self.PATH_fmt.format(uid=f"/{name}"), raw=raw) + if 404 == code: + code, data = self._get(self.PATH_fmt.format(uid=f"?name={name}"), raw=raw) + + return code, data def delete(self, name, *, raw=False): return self._delete(self.PATH_fmt.format(uid=f"/{name}"), raw=raw) diff --git a/config.yml b/config.yml index 1b54fc5ec..bcf919084 100644 --- a/config.yml +++ b/config.yml @@ -25,6 +25,7 @@ image-cache-url: '' # script location for terraform related test cases terraform-scripts-location: 'terraform_test_artifacts' +terraform-provider-rancher: '3.1.1' # Backup Target S3 s3-endpoint: '' @@ -38,12 +39,11 @@ nfs-endpoint: '' nfs-mount-dir: 'nfsshare' # Rancher Cluster -# rancher-endpoint: 'https://127.0.0.1' -# External Rancher admin user password -# rancher-admin-password: 'rancher_password' +rancher-endpoint: 'https://127.0.0.1' +rancher-admin-password: 'rancher_password' # Kubernetes version for RKE -RKE1-version: 'v1.24.11-rancher2-1' -RKE2-version: 'v1.24.11+rke2r1' +RKE1-version: 'v1.26.11-rancher2-1' +RKE2-version: 'v1.26.11+rke2r1' # Wait time for polling Rancher cluster status. rancher-cluster-wait-timeout: 7200 diff --git a/harvester_e2e_tests/conftest.py b/harvester_e2e_tests/conftest.py index 75d5a1d22..6cdcd3499 100644 --- a/harvester_e2e_tests/conftest.py +++ b/harvester_e2e_tests/conftest.py @@ -249,6 +249,12 @@ def pytest_addoption(parser): default=config_data.get('terraform-provider-harvester'), help=('Version of Terraform Harvester Provider') ) + parser.addoption( + '--terraform-provider-rancher', + action='store', + default=config_data.get('terraform-provider-rancher'), + help=('Version of Terraform Rancher Provider') + ) def pytest_configure(config): diff --git a/harvester_e2e_tests/fixtures/terraform_rancher.py b/harvester_e2e_tests/fixtures/terraform_rancher.py new file mode 100644 index 000000000..fa6e4e51e --- /dev/null +++ b/harvester_e2e_tests/fixtures/terraform_rancher.py @@ -0,0 +1,347 @@ +import re +import json +import requests +from pathlib import Path +from datetime import datetime +from subprocess import run, PIPE +from dataclasses import dataclass, field + +import pytest +from pkg_resources import parse_version + + +TF_PROVIDER = ''' +terraform { + required_providers { + rancher2 = { + source = "%(provider_source)s" + version = "%(provider_version)s" + } + } +} + +provider "rancher2" { + api_url = "%(rancher_endpoint)s" + token_key = "%(rancher_token)s" + insecure = true +} + +data "rancher2_cluster_v2" "%(harvester_name)s" { + name = "%(harvester_name)s" +} + +''' + +# rancher2_machine_config_v2 +TF_MACHINE_CONFIG = ''' +resource "rancher2_machine_config_v2" "%(name)s" { + generate_name = "%(name)s" + %(harvester_config)s +} + +''' +TF_HARVESTER_CONFIG = ''' + harvester_config { + vm_namespace = "default" + cpu_count = "2" + memory_size = "4" + ssh_user = "%(ssh_user)s" + disk_info = %(disk_info)s + network_info = %(network_info)s + user_data = %(user_data)s + } +''' +TF_DISK_INFO = '''<= parse_version(cls.support_to) + + @classmethod + def for_version(cls, version): + for c in sorted(cls._sub_classes.get(cls, []), + reverse=True, key=lambda x: parse_version(x.support_to).release): + if c.is_support(version): + return c + return cls + + def __init_subclass__(cls): + for parent in cls.__mro__: + if issubclass(parent, BaseTerraformResource): + cls._sub_classes.setdefault(parent, []).append(cls) + + def __init__(self, converter): + self.executor = Path(converter).resolve() + + def convert_to_hcl(self, json_spec, raw=False): + rv = run(f"echo {json.dumps(json_spec)!r} | {self.executor!s}", + shell=True, stdout=PIPE, stderr=PIPE) + if raw: + return rv + if rv.stderr: + raise TypeError(rv.stderr, rv.stdout, rv.returncode) + out = rv.stdout.decode() + out = re.sub(r'"resource"', "resource", out) # resource should not quote + out = re.sub(r"\"(.+?)\" =", r"\1 =", out) # property should not quote + out = re.sub(r'"(data\.\S+?)"', r"\1", out) # data should not quote + out = re.sub(r"(.[^ ]+) = {", r"\1 {", out) # block should not have `=` + return out + + def make_resource(self, resource_type, resource_name, *, convert=True, **properties): + rv = dict(resource={resource_type: {resource_name: properties}}) + if convert: + return ResourceContext(resource_type, resource_name, self.convert_to_hcl(rv), rv) + return rv + + +class TerraformResource(BaseTerraformResource): + ''' https://github.com/rancher/terraform-provider-rancher2/tree/v3.1.1/docs/resources + ''' + support_to = "3.1.1" + + def cloud_credential(self, name, harvester_name, *, convert=True, **properties): + harvester_credential_config = { + "cluster_id": f"data.rancher2_cluster_v2.{harvester_name}.cluster_v1_id", + "cluster_type": "imported", + "kubeconfig_content": f"data.rancher2_cluster_v2.{harvester_name}.kube_config" + } + return self.make_resource("rancher2_cloud_credential", name, + name=name, + harvester_credential_config=harvester_credential_config, + convert=convert, **properties) + + def machine_config(self, rke2_cluster, image, network): + hcl_str = TF_MACHINE_CONFIG % { + "name": rke2_cluster["name"], + "harvester_config": TF_HARVESTER_CONFIG % { + "ssh_user": image["ssh_user"], + "disk_info": TF_DISK_INFO % {"image_name": image["id"]}, + "network_info": TF_NETWORK_INFO % {"network_name": network["id"]}, + "user_data": TF_USER_DATA + } + } + + return ResourceContext("rancher2_machine_config_v2", rke2_cluster["name"], hcl_str, "") + + def cluster_config(self, rke2_cluster, harvester, cloud_credential): + machine_pools = TF_MACHINE_POOLS % { + "cloud_credential_name": cloud_credential["name"], + "machine_config_name": rke2_cluster["name"] + } + rke_config = TF_RKE_CONFIG % { + "machine_pools": machine_pools, + "harvester_name": harvester["name"] + } + hcl_str = TF_CLUSTER_CONFIG % { + "name": rke2_cluster["name"], + "rke2_version": rke2_cluster["k8s_version"], + "rke_config": rke_config + } + + return ResourceContext("rancher2_cluster_v2", rke2_cluster["name"], hcl_str, "") diff --git a/harvester_e2e_tests/integration/test_z_terraform_rancher.py b/harvester_e2e_tests/integration/test_z_terraform_rancher.py index 104a5e576..370759ece 100644 --- a/harvester_e2e_tests/integration/test_z_terraform_rancher.py +++ b/harvester_e2e_tests/integration/test_z_terraform_rancher.py @@ -2,17 +2,12 @@ import pytest -# TODO: Drop after debug # -########################## -import pdb -from pprint import pprint - pytest_plugins = [ "harvester_e2e_tests.fixtures.api_client", 'harvester_e2e_tests.fixtures.rancher_api_client', "harvester_e2e_tests.fixtures.images", - "harvester_e2e_tests.fixtures.terraform" + "harvester_e2e_tests.fixtures.terraform_rancher" ] @@ -65,55 +60,84 @@ def vlan_network(request, api_client): ) yield { - "id": data['metadata']['name'] + "id": f"default/{data['metadata']['name']}" } api_client.networks.delete(network_name) @pytest.fixture(scope='module') -def harvester_cluster(api_client, rancher_api_client, unique_name, wait_timeout): +def harvester(api_client, rancher_api_client, unique_name, wait_timeout): """ Rancher creates Harvester entry (Import Existing) """ - rc, data = rancher_api_client.mgmt_clusters.create_harvester(unique_name) + name = f"hvst-{unique_name}" + + rc, data = rancher_api_client.mgmt_clusters.create_harvester(name) assert 201 == rc, (rc, data) endtime = datetime.now() + timedelta(seconds=wait_timeout) while endtime > datetime.now(): - rc, data = rancher_api_client.mgmt_clusters.get(unique_name) + rc, data = rancher_api_client.mgmt_clusters.get(name) if data.get('status', {}).get('clusterName'): break else: raise AssertionError( - f"Fail to get MgmtCluster with clusterName {unique_name}\n" + f"Fail to get MgmtCluster with clusterName {name}\n" f"rc: {rc}, data:\n{data}" ) yield { - "name": unique_name, - "id": data['status']['clusterName'] + "name": name, + "id": data['status']['clusterName'], + "kubeconfig": api_client.generate_kubeconfig() } - rancher_api_client.mgmt_clusters.delete(unique_name) + rancher_api_client.mgmt_clusters.delete(name) updates = dict(value="") api_client.settings.update("cluster-registration-url", updates) -# Tests +@pytest.fixture(scope="module") +def rancher(rancher_api_client): + access_key, secret_key = rancher_api_client.token.split(":") + yield { + "endpoint": rancher_api_client.endpoint, + "token": rancher_api_client.token, + "access_key": access_key, + "secret_key": secret_key + } + + +@pytest.fixture(scope='module') +def rke2_cluster(unique_name, k8s_version): + return { + "name": f"rke2-{unique_name}", + "id": "", + "k8s_version": k8s_version + } + + +@pytest.fixture(scope="module") +def cloud_credential(unique_name): + return { + "name": f"cc-{unique_name}" + } + + @pytest.mark.p0 @pytest.mark.terraform @pytest.mark.rancher @pytest.mark.dependency(name="import_harvester") -def test_import_harvester(api_client, rancher_api_client, harvester_cluster, wait_timeout): +def test_import_harvester(api_client, rancher_api_client, harvester, wait_timeout): # Get cluster registration URL in Rancher's Virtualization Management endtime = datetime.now() + timedelta(seconds=wait_timeout) while endtime > datetime.now(): - rc, data = rancher_api_client.cluster_registration_tokens.get(harvester_cluster['id']) + rc, data = rancher_api_client.cluster_registration_tokens.get(harvester['id']) if 200 == rc and data.get('manifestUrl'): break else: raise AssertionError( - f"Fail to registration URL for the imported harvester {harvester_cluster['name']}\n" + f"Fail to registration URL for the imported harvester {harvester['name']}\n" f"rc: {rc}, data:\n{data}" ) @@ -128,7 +152,7 @@ def test_import_harvester(api_client, rancher_api_client, harvester_cluster, wai # Check Cluster becomes `active` in Rancher's Virtualization Management endtime = datetime.now() + timedelta(seconds=wait_timeout) while endtime > datetime.now(): - rc, data = rancher_api_client.mgmt_clusters.get(harvester_cluster['name']) + rc, data = rancher_api_client.mgmt_clusters.get(harvester['name']) cluster_state = data['metadata']['state'] if "active" == cluster_state['name'] and "Ready" in cluster_state['message']: break @@ -142,9 +166,94 @@ def test_import_harvester(api_client, rancher_api_client, harvester_cluster, wai @pytest.mark.p0 @pytest.mark.terraform @pytest.mark.rancher -class TestRKE2: - # @pytest.mark.dependency(depends=["import_harvester"], name="create_rke2") - def test_create_rke2(self, ubuntu_image, vlan_network): - - assert True +@pytest.mark.dependency(name="create_cloud_credential", depends=["import_harvester"]) +def test_create_cloud_credential(rancher_api_client, tf_resource, tf_rancher, + harvester, cloud_credential): + spec = tf_resource.cloud_credential(cloud_credential["name"], harvester["name"]) + tf_rancher.save_as(spec.ctx, "cloud_credential") + + out, err, code = tf_rancher.apply_resource(spec.type, spec.name) + assert not err and 0 == code + + code, data = rancher_api_client.cloud_credentials.get(spec.name) + assert 200 == code, ( + f"Failed to get cloud credential {spec.name}: {code}, {data}" + ) + + +@pytest.mark.p0 +@pytest.mark.terraform +@pytest.mark.rancher +@pytest.mark.dependency(name="create_machine_config", depends=["create_cloud_credential"]) +def test_create_machine_config(tf_resource, tf_rancher, rke2_cluster, ubuntu_image, vlan_network): + spec = tf_resource.machine_config(rke2_cluster, ubuntu_image, vlan_network) + tf_rancher.save_as(spec.ctx, "machine_config") + + out, err, code = tf_rancher.apply_resource(spec.type, spec.name) + assert not err and 0 == code + + +@pytest.mark.p0 +@pytest.mark.terraform +@pytest.mark.rancher +@pytest.mark.dependency(name="create_rke2_cluster", depends=["create_machine_config"]) +def test_create_rke2_cluster(tf_resource, tf_rancher, rke2_cluster, rancher_api_client, + harvester, cloud_credential): + spec = tf_resource.cluster_config(rke2_cluster, harvester, cloud_credential) + tf_rancher.save_as(spec.ctx, "rke2_cluster") + + out, err, code = tf_rancher.apply_resource(spec.type, spec.name) + assert not err and 0 == code + + rc, data = rancher_api_client.mgmt_clusters.get(rke2_cluster['name']) + cluster_state = data.get("metadata", {}).get("state", {}) + assert "active" == cluster_state['name'] and \ + "Ready" in cluster_state['message'] + + # check deployments + rke2_cluster['id'] = data["status"]["clusterName"] + for deployment in ["harvester-cloud-provider", "harvester-csi-driver-controllers"]: + rc, data = rancher_api_client.cluster_deployments.get( + rke2_cluster['id'], "kube-system", deployment + ) + cluster_state = data.get("metadata", {}).get("state", {}) + assert 200 == rc and \ + "active" == cluster_state["name"] + + +@pytest.mark.p0 +@pytest.mark.terraform +@pytest.mark.rancher +@pytest.mark.dependency(name="delete_rke2_cluster", depends=["create_rke2_cluster"]) +def test_delete_rke2_cluster(tf_resource, tf_rancher, rke2_cluster, rancher_api_client, + harvester, cloud_credential): + spec = tf_resource.cluster_config(rke2_cluster, harvester, cloud_credential) + + out, err, code = tf_rancher.destroy_resource(spec.type, spec.name) + assert not err and 0 == code + + rc, data = rancher_api_client.mgmt_clusters.get(rke2_cluster['name']) + assert 404 == rc + + +@pytest.mark.p0 +@pytest.mark.terraform +@pytest.mark.rancher +@pytest.mark.dependency(name="delete_machine_config", depends=["create_machine_config"]) +def test_delete_machine_config(tf_resource, tf_rancher, rke2_cluster, ubuntu_image, vlan_network): + spec = tf_resource.machine_config(rke2_cluster, ubuntu_image, vlan_network) + + out, err, code = tf_rancher.destroy_resource(spec.type, spec.name) + assert not err and 0 == code + + +@pytest.mark.p0 +@pytest.mark.terraform +@pytest.mark.rancher +@pytest.mark.dependency(name="delete_cloud_credential", depends=["create_cloud_credential"]) +def test_delete_cloud_credential(rancher_api_client, tf_resource, tf_rancher, + harvester, cloud_credential): + spec = tf_resource.cloud_credential(cloud_credential["name"], harvester["name"]) + out, err, code = tf_rancher.destroy_resource(spec.type, spec.name) + assert not err and 0 == code