diff --git a/RELEASING.md b/RELEASING.md index 454628c4..6a2decbb 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,10 +3,16 @@ Releases are made through the [GitHub UI](https://github.com/microsoft/planetary-computer-tasks/releases/new). Create a new tag for your release, using the format `..`. -You also need to update the `pc-test-gha-tags-release` Federated Identity Credential on the `PC Test GitHub Actions Deployment` App Regestration to match the new tag. +You also need to update the `pc-test-gha-tags-release` Federated Identity Credential on the `PC Test GitHub Actions Deployment` App Registration to match the new tag. ```azurecli az ad app federated-credential update --federated-credential-id "pc-test-gha-tags-release" --id "$CLIENT_ID" --parameters '{"issuer": "https://token.actions.githubusercontent.com", "subject": "repo:microsoft/planetary-computer-tasks:ref:refs/tags/$TAG", "description": "Federated credential for Github Actions to deploy to Azure from microsoft/planetary-computer-tasks with any tag", "audiences": ["api://AzureADTokenExchange"]}' ``` where `$TAG` is something like `2024.6.1`. + +This identity has been granted the necessary Azure RBAC permissions to do the deployment. + +```azurecli +az role assignment create --role "Key Vault Secrets Officer" --assignee "$CLIENT_ID" --scope "$KEYVAULT_ID" +``` \ No newline at end of file diff --git a/deployment/bin/deploy b/deployment/bin/deploy index 22a3c485..74543629 100755 --- a/deployment/bin/deploy +++ b/deployment/bin/deploy @@ -223,36 +223,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then --timeout 2m0s \ --debug - echo "==================" - echo "===== KEDA =======" - echo "==================" - - helm upgrade --install keda helm/vendored/keda-2.14.2.tgz \ - -n keda \ - --create-namespace \ - -f helm/keda-values.yaml \ - --wait \ - --timeout 2m0s \ - --debug - - # TODO: Figure out how to apply to set this with helm - echo "Adding KEDA secret" - pushd ${TERRAFORM_DIR} - SA_CONNECTION_STRING=$(tf_output sa_connection_string) - popd - # pipe into kubectl apply to ensure create or update works. - kubectl -n pc create secret generic secrets-storage-queue-connection-string \ - --from-literal="ConnectionString=$SA_CONNECTION_STRING" \ - --dry-run=client -o yaml \ - | kubectl apply -f - - echo "Creating KEDA trigger auth" - # This namespace must match where argo runs its workflows - kubectl -n pc apply -f helm/keda-trigger-authentication.yaml - - echo "Adding KEDA roles and rolebindings" - # This namespace must match where argo runs its workflows - kubectl -n pc apply -f helm/argo-workflow-keda-rbac-prod.yaml - echo "====================" echo "== PCTasksIngress ==" echo "====================" diff --git a/deployment/helm/argo-workflow-keda-rbac-dev.yaml b/deployment/helm/argo-workflow-keda-rbac-dev.yaml deleted file mode 100644 index 21402b69..00000000 --- a/deployment/helm/argo-workflow-keda-rbac-dev.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# This is identical to argo-workflow-keda-rbac-prod, -# with a different namespace. -# Roles and Role Bindings for letting pods managed by Argo Workflows -# create Deployments. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: streaming-manager-role - namespace: argo # must match where argo spawns stuff -rules: - - apiGroups: - - "apps" - - "keda.sh" - resources: - - deployments - - scaledobjects - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: streaming-manager-rolebinding - namespace: argo # must match where argo spawns stuff -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: streaming-manager-role -subjects: - - namespace: argo - kind: ServiceAccount - name: default diff --git a/deployment/helm/argo-workflow-keda-rbac-prod.yaml b/deployment/helm/argo-workflow-keda-rbac-prod.yaml deleted file mode 100644 index c96551b9..00000000 --- a/deployment/helm/argo-workflow-keda-rbac-prod.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# This is identical to argo-workflow-keda-rbac-dev, -# with a different namespace. -# Roles and Role Bindings for letting pods managed by Argo Workflows -# create Deployments. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: streaming-manager-role - namespace: pc # must match where argo spawns stuff -rules: - - apiGroups: - - "apps" - - "keda.sh" - resources: - - pods - - deployments - - scaledobjects - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: streaming-manager-rolebinding - namespace: pc # must match where argo spawns stuff -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: streaming-manager-role -subjects: - - namespace: pc - kind: ServiceAccount - name: default diff --git a/deployment/helm/keda-trigger-authentication.yaml b/deployment/helm/keda-trigger-authentication.yaml deleted file mode 100644 index dcfab53a..00000000 --- a/deployment/helm/keda-trigger-authentication.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# A Kubernetes Secret for authentication with the storage queues. -apiVersion: keda.sh/v1alpha1 -kind: TriggerAuthentication -metadata: - # This must match pctasks.run.workflow.kubernetes.KEDA_QUEUE_CONNECTION_STRING_AUTH_NAME - # Must be in the same namespace as the KEDA Deployments. - name: queue-connection-string-auth -spec: - secretTargetRef: - - parameter: connection - # This Kubernetes Secret is created with terraform - # The name must match the value in terraform/resources/aks.tf - name: secrets-storage-queue-connection-string - key: ConnectionString diff --git a/deployment/helm/keda-values.yaml b/deployment/helm/keda-values.yaml deleted file mode 100644 index 57bcbf21..00000000 --- a/deployment/helm/keda-values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -global: - registry: pccomponentstest.azurecr.io -image: - keda: - registry: pccomponentstest.azurecr.io - repository: kedacore/keda - tag: 2.14.0 - metricsApiServer: - registry: pccomponentstest.azurecr.io - repository: kedacore/keda-metrics-apiserver - tag: 2.14.0 - webhooks: - registry: pccomponentstest.azurecr.io - repository: kedacore/keda-admission-webhooks - tag: 2.14.0 diff --git a/deployment/terraform/resources/aks.tf b/deployment/terraform/resources/aks.tf index c23f7207..fb2aee8d 100644 --- a/deployment/terraform/resources/aks.tf +++ b/deployment/terraform/resources/aks.tf @@ -175,14 +175,10 @@ resource "azurerm_federated_identity_credential" "workflows" { timeouts {} } -resource "azurerm_key_vault_access_policy" "example" { - key_vault_id = data.azurerm_key_vault.pctasks.id - tenant_id = azurerm_user_assigned_identity.workflows.tenant_id - object_id = azurerm_user_assigned_identity.workflows.principal_id - - secret_permissions = [ - "Get" - ] +resource "azurerm_role_assignment" "workflows-secrets-user" { + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_user_assigned_identity.workflows.principal_id + scope = data.azurerm_key_vault.pctasks.id } # When you enable the key vault secrets provider block in an AKS cluster, diff --git a/deployment/terraform/resources/apim.tf b/deployment/terraform/resources/apim.tf index 79d42963..2687664e 100644 --- a/deployment/terraform/resources/apim.tf +++ b/deployment/terraform/resources/apim.tf @@ -18,18 +18,10 @@ resource "azurerm_api_management" "pctasks" { } } -resource "azurerm_key_vault_access_policy" "apim" { - key_vault_id = data.azurerm_key_vault.deploy_secrets.id - tenant_id = azurerm_api_management.pctasks.identity[0].tenant_id - object_id = azurerm_api_management.pctasks.identity[0].principal_id - - depends_on = [ - azurerm_api_management.pctasks, - ] - - secret_permissions = [ - "Get", "List" - ] +resource "azurerm_role_assignment" "apim-secrets-user" { + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_api_management.pctasks.identity[0].principal_id + scope = data.azurerm_key_vault.deploy_secrets.id } resource "azurerm_api_management_named_value" "pctasks_access_key" { diff --git a/deployment/terraform/resources/keyvault.tf b/deployment/terraform/resources/keyvault.tf index 379c4958..cbe2d8a0 100644 --- a/deployment/terraform/resources/keyvault.tf +++ b/deployment/terraform/resources/keyvault.tf @@ -3,14 +3,10 @@ data "azurerm_key_vault" "pctasks" { resource_group_name = var.pctasks_task_kv_resource_group_name } -resource "azurerm_key_vault_access_policy" "function_app" { - key_vault_id = data.azurerm_key_vault.pctasks.id - tenant_id = azurerm_linux_function_app.pctasks.identity.0.tenant_id - object_id = azurerm_linux_function_app.pctasks.identity.0.principal_id - - secret_permissions = [ - "Get", "List" - ] +resource "azurerm_role_assignment" "functions-secrets-user" { + role_definition_name = "Key Vault Secrets User" + principal_id = azurerm_linux_function_app.pctasks.identity.0.principal_id + scope = data.azurerm_key_vault.pctasks.id } # Store database information as a secret diff --git a/docs/development/deploying.md b/docs/development/deploying.md index 265ccfdc..524d5c9e 100644 --- a/docs/development/deploying.md +++ b/docs/development/deploying.md @@ -47,26 +47,6 @@ az acr import -n pccomponentstest --source quay.io/argoproj/argoexec:v3.5.7 -t a The image and tag values are specified in the `argo-values.yaml` file and used during installs. -#### KEDA - -The chart can be brought into the `deployment/helm/vendored` directory by running the following command: - -```console -cd deployment/helm/vendored -helm repo add kedacore -helm pull kedacore/keda --version 2.14.2 -``` - -The images can be imported into your ACR by running the following command: - -```console -az acr import -n pccomponentstest --source ghcr.io/kedacore/keda-admission-webhooks:2.14.0 -t kedacore/keda-admission-webhooks:2.14.0 --subscription "Planetary Computer Test" -az acr import -n pccomponentstest --source ghcr.io/kedacore/keda-metrics-apiserver:2.14.0 -t kedacore/keda-metrics-apiserver:2.14.0 --subscription "Planetary Computer Test" -az acr import -n pccomponentstest --source ghcr.io/kedacore/keda:2.14.0 -t kedacore/keda:2.14.0 --subscription "Planetary Computer Test" -``` - -The image and tag values are specified in the `keda-values.yaml` file and used during installs. - ### Deployment Service principal You'll need a service principal that has sufficient permissions to deploy Azure resources, including creating resource groups and assigning IAM roles. diff --git a/scripts/cluster b/scripts/cluster index 53fb2c1d..88de3663 100755 --- a/scripts/cluster +++ b/scripts/cluster @@ -113,16 +113,6 @@ EOF # E0303 15:59:48.181644 27243 memcache.go:255] couldn't get resource list for external.metrics.k8s.io/v1beta1: Got empty response for: external.metrics.k8s.io/v1beta1 # https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/146 - echo "===== KEDA =======" - helm repo add kedacore https://kedacore.github.io/charts - - helm upgrade --install keda kedacore/keda \ - -n keda \ - --create-namespace \ - --wait \ - --timeout 2m0s - kubectl -n argo apply -f deployment/helm/argo-workflow-keda-rbac-dev.yaml - kubectl -n argo apply -f deployment/helm/keda-trigger-authentication.yaml kubectl -n argo create secret generic queue-connection-string-auth \ --from-literal=ConnectionString='AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;' diff --git a/tests/test_run_workflow_kubernetes.py b/tests/test_run_workflow_kubernetes.py deleted file mode 100644 index e0066bbe..00000000 --- a/tests/test_run_workflow_kubernetes.py +++ /dev/null @@ -1,242 +0,0 @@ -""" -Integration test for the Kubernetes Workflow Runner. - -These tests rely on the Kind cluster and the test process having -access to the Kubernetes API. -""" -import base64 -import os - -import kubernetes -import pytest - -import pctasks.core.models.task -from pctasks.core.constants import ( - AZURITE_HOST_ENV_VAR, - AZURITE_PORT_ENV_VAR, - AZURITE_STORAGE_ACCOUNT_ENV_VAR, -) -from pctasks.core.cosmos.settings import CosmosDBSettings -from pctasks.core.models.config import BlobConfig -from pctasks.core.models.workflow import ( - JobDefinition, - Workflow, - WorkflowDefinition, - WorkflowSubmitMessage, -) -from pctasks.dev.k8s import get_streaming_task_definition -from pctasks.run.models import ( - PreparedTaskData, - PreparedTaskSubmitMessage, - TaskSubmitMessage, -) -from pctasks.run.settings import RunSettings, WorkflowExecutorConfig -from pctasks.run.workflow.executor.streaming import StreamingWorkflowExecutor - -TEST_NAMESPACE = "pctasks-test" - - -@pytest.fixture(scope="session") -def run_settings(): - """Run settings, using azurite for storage.""" - key = ( - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq" - "/K1SZFPTOtr/KBHBeksoGMGw==" - ) - - host = os.environ.get(AZURITE_HOST_ENV_VAR, "localhost") - blob_port = int(os.environ.get(AZURITE_PORT_ENV_VAR, "10000")) - - return RunSettings( - notification_queue={ - "account_url": "queue://devstoreaccount1/notifications", - "connection_string": "connstr", - "queue_name": "notifications", - "sas_token": "sas", - }, - tables_account_url=f"http://{host}:10001", - tables_account_name="devstoreaccount1", - tables_account_key="devstoreaccount1", - blob_account_url=f"http://{host}:{blob_port}", - blob_account_name="devstoreaccount1", - blob_account_key=key, - keyvault_url="https://devstoreaccount1.vault.azure.net/", - task_runner_type="local", - workflow_runner_type="local", - local_dev_endpoints_url="http://localhost:7071", - streaming_taskio_sp_client_id="test-client-id", - streaming_taskio_sp_client_secret="test-client-secret", - streaming_taskio_sp_tenant_id="test-tenant-id", - streaming_task_namespace=TEST_NAMESPACE, - local_secrets=True, - ) - - -@pytest.fixture(scope="session") -def namespace(): - """ - Setup the namespace for Kubernetes & KEDA Tests. - - This creates - - - A namespace called `pctasks-test` - - A Kubernetes secret with the Account Key for an azurite storage queue - - A KEDA TriggerAuthentication object - - The fixture is scoped to the session. The namespace is deleted as the - session closes. - """ - from kubernetes import client, config - - config.load_config() - - v1 = client.CoreV1Api() - objects = client.CustomObjectsApi() - ns = client.V1Namespace(metadata=client.V1ObjectMeta(name=TEST_NAMESPACE)) - try: - v1.create_namespace(ns) - except kubernetes.client.rest.ApiException as e: - if e.status != 409: - raise RuntimeError( - "Namespace pctasks-test already exists. Manually delete the " - "namespace to run this test." - ) from e - - connstr = base64.b64encode( - ( - "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" - "AccountKey=" - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq" - "/K1SZFPTOtr/KBHBeksoGMGw==;" - "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" - "QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" - "TableEndpoint=http://127.0.0.1.10002/devstoreaccount1;" - ).encode() - ).decode() - - v1.create_namespaced_secret( - namespace=ns.metadata.name, - body=client.V1Secret( - data={"ConnectionString": connstr}, - metadata=client.V1ObjectMeta( - name="secrets-storage-queue-connection-string" - ), - ), - ) - body = { - "apiVersion": "keda.sh/v1alpha1", - "kind": "TriggerAuthentication", - "metadata": {"name": "queue-connection-string-auth"}, - "spec": { - "secretTargetRef": [ - { - "parameter": "connection", - "name": "secrets-storage-queue-connection-string", - "key": "ConnectionString", - } - ] - }, - } - objects.create_namespaced_custom_object( - body=body, - namespace=ns.metadata.name, - group="keda.sh", - version="v1alpha1", - plural="triggerauthentications", - ) - - yield ns - - # this seems to return before the namespace is deleted - v1.delete_namespace(name=ns.metadata.name) - - -def test_submit_task(namespace, run_settings): - task_definition = get_streaming_task_definition() - prepared_task = PreparedTaskSubmitMessage( - task_submit_message=TaskSubmitMessage( - dataset_id="test-dataset-id", - run_id="test-run", - job_id="job-id", - partition_id="0", - definition=task_definition, - ), - task_run_message=pctasks.core.models.task.TaskRunMessage( - args={}, - config=pctasks.core.models.task.TaskRunConfig( - image="image", - run_id="test-run", - job_id="job-id", - partition_id="0", - task_id="task-id", - task=task_definition.task, - status_blob_config=BlobConfig( - uri="blob://devstoreaccount1/taskio/status" - ), - output_blob_config=BlobConfig( - uri="blob://devstoreaccount1/taskio/output" - ), - log_blob_config=BlobConfig(uri="blob://devstoreaccount1/taskio/log"), - ), - ), - task_input_blob_config=BlobConfig(uri="blob://devstoreaccount1/taskio/input"), - task_data=PreparedTaskData( - image="image", - runner_info={}, - ), - ) - # What do we want to assert here? What do we want to actually test? - # We're only testing submit here. We don't care whether the task actually runs. - pctasks.run.workflow.kubernetes.submit_task(prepared_task, run_settings) - - -@pytest.mark.parametrize( - ["queue_url", "expected"], - [ - ( - "http://127.0.0.1:10001/devstoreaccount1/test-queue", - "devstoreaccount1-test-queue", - ), - ( - "http://azurite:10001/devstoreaccount1/test-queue", - "devstoreaccount1-test-queue", - ), - ("https://goeseuwest.blob.core.windows.net/goes-glm", "goeseuwest-goes-glm"), - ], -) -def test_get_name_prefix(queue_url, expected): - result = pctasks.run.workflow.kubernetes.get_name_prefix(queue_url) - assert result == expected - - -# via kubernetes Python library -# https://github.com/kubernetes-client/python/issues/2024 -@pytest.mark.filterwarnings("ignore:HTTPResponse.getheaders:DeprecationWarning") -def test_execute_workflow(namespace, run_settings, monkeypatch): - task_definition = get_streaming_task_definition() - for k, v in { - AZURITE_STORAGE_ACCOUNT_ENV_VAR: "devstoreaccount1", - AZURITE_HOST_ENV_VAR: "127.0.0.1", - AZURITE_PORT_ENV_VAR: "10000", - }.items(): - if k not in os.environ: - monkeypatch.setenv(k, v) - - settings = WorkflowExecutorConfig( - run_settings=run_settings, - cosmosdb_settings=CosmosDBSettings(), - ) - executor = StreamingWorkflowExecutor(settings) - definition = WorkflowDefinition( - workflow_id="test", - name="test", - dataset_id="test", - jobs={"test": JobDefinition(tasks=[task_definition])}, - is_streaming=True, - ) - message = WorkflowSubmitMessage( - run_id="test", workflow=Workflow(id="test", definition=definition) - ) - - # Here's the test. - executor.execute_workflow(message)