diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d58225e2df..917b44576c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -458,7 +458,7 @@ jobs: context: ./data-orchestration/hm-prefect/workflows/calculate file: ./data-orchestration/hm-prefect/workflows/calculate/Dockerfile push: true - tags: ghcr.io/hongbo-miao/hm-prefect-calculate:latest + tags: ghcr.io/hongbo-miao/prefect-calculate:latest cache-from: type=gha cache-to: type=gha,mode=max attempt_limit: 3 @@ -494,7 +494,7 @@ jobs: context: ./data-orchestration/hm-prefect/workflows/daft-analysis file: ./data-orchestration/hm-prefect/workflows/daft-analysis/Dockerfile push: true - tags: ghcr.io/hongbo-miao/hm-prefect-daft-analysis:latest + tags: ghcr.io/hongbo-miao/prefect-daft-analysis:latest cache-from: type=gha cache-to: type=gha,mode=max attempt_limit: 3 @@ -530,7 +530,7 @@ jobs: context: ./data-orchestration/hm-prefect/workflows/greet file: ./data-orchestration/hm-prefect/workflows/greet/Dockerfile push: true - tags: ghcr.io/hongbo-miao/hm-prefect-greet:latest + tags: ghcr.io/hongbo-miao/prefect-greet:latest cache-from: type=gha cache-to: type=gha,mode=max attempt_limit: 3 @@ -566,7 +566,7 @@ jobs: context: ./data-orchestration/hm-prefect/workflows/print-platform file: ./data-orchestration/hm-prefect/workflows/print-platform/Dockerfile push: true - tags: ghcr.io/hongbo-miao/hm-prefect-print-platform:latest + tags: ghcr.io/hongbo-miao/prefect-print-platform:latest cache-from: type=gha cache-to: type=gha,mode=max attempt_limit: 3 diff --git a/.github/workflows/docker-prune.yml b/.github/workflows/docker-prune.yml index 169b603583..eab20dcaee 100644 --- a/.github/workflows/docker-prune.yml +++ b/.github/workflows/docker-prune.yml @@ -29,16 +29,16 @@ jobs: hm-kafka-opa-kafka-connect, hm-keydb, hm-opal-client, - hm-prefect-calculate, - hm-prefect-daft-analysis, - hm-prefect-greet, - hm-prefect-print-platform, hm-rasa-action-server, hm-spark-find-retired-people-python, hm-spark-find-retired-people-scala, hm-spark-ingest-from-s3-to-kafka, hm-traefik, hm-web, + prefect-calculate, + prefect-daft-analysis, + prefect-greet, + prefect-print-platform, ] steps: - name: Prune dangling Docker images diff --git a/cloud-infrastructure/terraform/environments/production/harbor/main.tf b/cloud-infrastructure/terraform/environments/production/harbor/main.tf index 46a00b3ffc..36860a4b75 100644 --- a/cloud-infrastructure/terraform/environments/production/harbor/main.tf +++ b/cloud-infrastructure/terraform/environments/production/harbor/main.tf @@ -3,33 +3,52 @@ module "harbor_config_system" { project_creation_restriction = "adminonly" } +# Config +data "aws_secretsmanager_secret" "hm_harbor_google_client_secret" { + name = "${var.environment}-hm-harbor-google-client" +} +data "aws_secretsmanager_secret_version" "hm_harbor_google_client_secret_version" { + secret_id = data.aws_secretsmanager_secret.hm_harbor_google_client_secret.id +} +module "harbor_config_google_auth" { + source = "../../../modules/harbor/hm_harbor_config_google_auth" + primary_auth_mode = true + oidc_client_id = "xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com" + oidc_client_secret = jsondecode(data.aws_secretsmanager_secret_version.hm_harbor_google_client_secret_version.secret_string)["secret"] +} + +# Registries module "harbor_registry_docker_hub" { source = "../../../modules/harbor/hm_harbor_registry" provider_name = "docker-hub" name = "docker-hub" endpoint_url = "https://hub.docker.com" } + +# Projects module "harbor_project_docker_hub_proxy_cache" { source = "../../../modules/harbor/hm_harbor_project" name = "docker-hub-proxy-cache" public = true registry_id = module.harbor_registry_docker_hub.id } -module "harbor_project_engineering" { +module "harbor_project_hm" { source = "../../../modules/harbor/hm_harbor_project" - name = "engineering" + name = "hm" public = false } -data "aws_secretsmanager_secret" "hm_harbor_google_client_secret" { - name = "${var.environment}-hm-harbor-google-client" +# Robots +data "aws_secretsmanager_secret" "hm_harbor_hm_kubernetes_robot_secret" { + provider = aws.production + name = "${var.environment}-hm-harbor-hm-kubernetes-robot" } -data "aws_secretsmanager_secret_version" "hm_harbor_google_client_secret_version" { - secret_id = data.aws_secretsmanager_secret.hm_harbor_google_client_secret.id +data "aws_secretsmanager_secret_version" "hm_harbor_hm_kubernetes_robot_secret_version" { + provider = aws.production + secret_id = data.aws_secretsmanager_secret.hm_harbor_hm_kubernetes_robot_secret.id } -module "harbor_config_google_auth" { - source = "../../../modules/harbor/hm_harbor_config_google_auth" - primary_auth_mode = true - oidc_client_id = "xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com" - oidc_client_secret = jsondecode(data.aws_secretsmanager_secret_version.hm_harbor_google_client_secret_version.secret_string)["secret"] +module "harbor_robot_account_hm_kubernetes_robot" { + source = "../../../modules/harbor/hm_harbor_robot_account" + name = "hm-kubernetes-robot" + secret = jsondecode(data.aws_secretsmanager_secret_version.hm_harbor_hm_kubernetes_robot_secret_version.secret_string)["secret"] } diff --git a/cloud-infrastructure/terraform/environments/production/harbor/providers.tf b/cloud-infrastructure/terraform/environments/production/harbor/providers.tf index 8c46e0b880..215be0fad9 100644 --- a/cloud-infrastructure/terraform/environments/production/harbor/providers.tf +++ b/cloud-infrastructure/terraform/environments/production/harbor/providers.tf @@ -1,3 +1,8 @@ provider "harbor" { url = "https://harbor.hongbomiao.com" } + +provider "aws" { + alias = "production" + region = "us-west-2" +} diff --git a/cloud-infrastructure/terraform/modules/harbor/hm_harbor_robot_account/main.tf b/cloud-infrastructure/terraform/modules/harbor/hm_harbor_robot_account/main.tf new file mode 100644 index 0000000000..9c229ff0e7 --- /dev/null +++ b/cloud-infrastructure/terraform/modules/harbor/hm_harbor_robot_account/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + harbor = { + source = "goharbor/harbor" + } + } +} + +# https://registry.terraform.io/providers/goharbor/harbor/latest/docs/resources/robot_account +resource "harbor_robot_account" "main" { + name = var.name + secret = var.secret + level = "system" + permissions { + access { + action = "pull" + resource = "repository" + } + kind = "project" + namespace = "*" + } +} diff --git a/cloud-infrastructure/terraform/modules/harbor/hm_harbor_robot_account/variables.tf b/cloud-infrastructure/terraform/modules/harbor/hm_harbor_robot_account/variables.tf new file mode 100644 index 0000000000..f568a7873f --- /dev/null +++ b/cloud-infrastructure/terraform/modules/harbor/hm_harbor_robot_account/variables.tf @@ -0,0 +1,7 @@ +variable "name" { + type = string +} +variable "secret" { + type = string + sensitive = true +} diff --git a/data-orchestration/hm-prefect/workflows/calculate/justfile b/data-orchestration/hm-prefect/workflows/calculate/justfile index b1c5608bb4..8b091baeef 100644 --- a/data-orchestration/hm-prefect/workflows/calculate/justfile +++ b/data-orchestration/hm-prefect/workflows/calculate/justfile @@ -17,12 +17,12 @@ test-coverage: uv run poe test-coverage publish-development-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-calculate:development . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-calculate:development + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-calculate:development . && \ + docker push harbor.hongbomiao.com/hm/prefect-calculate:development publish-production-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-calculate:latest . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-calculate:latest + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-calculate:latest . && \ + docker push harbor.hongbomiao.com/hm/prefect-calculate:latest deploy-development: uv run poe deploy --environment=development @@ -31,7 +31,7 @@ deploy-production: uv run poe deploy --environment=production run-development: - uv run poe run -- calculate/hm-development-calculate + uv run poe run -- hm-calculate/development-hm-calculate run-production: - uv run poe run -- calculate/hm-production-calculate + uv run poe run -- hm-calculate/production-hm-calculate diff --git a/data-orchestration/hm-prefect/workflows/calculate/src/workflow_deploy/utils/create_deployment.py b/data-orchestration/hm-prefect/workflows/calculate/src/workflow_deploy/utils/create_deployment.py index 84d52d62bd..a47ef54c82 100644 --- a/data-orchestration/hm-prefect/workflows/calculate/src/workflow_deploy/utils/create_deployment.py +++ b/data-orchestration/hm-prefect/workflows/calculate/src/workflow_deploy/utils/create_deployment.py @@ -11,7 +11,7 @@ async def create_deployment( docker_image_name: str, deployment: dict[str, Any], ) -> None: - deployment_name = f"hm-{environment}-{config.BASE_WORKFLOW_NAME}" + deployment_name = f"{environment}-hm-{config.BASE_WORKFLOW_NAME}" await workflow.deploy( name=deployment_name, work_pool_name=deployment["work_pool_name"], diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/justfile b/data-orchestration/hm-prefect/workflows/daft-analysis/justfile index 59d12b5786..1142178bc5 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/justfile +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/justfile @@ -17,12 +17,12 @@ test-coverage: uv run poe test-coverage publish-development-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-daft-analysis:development . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-daft-analysis:development + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-daft-analysis:development . && \ + docker push harbor.hongbomiao.com/hm/prefect-daft-analysis:development publish-production-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-daft-analysis:latest . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-daft-analysis:latest + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-daft-analysis:latest . && \ + docker push harbor.hongbomiao.com/hm/prefect-daft-analysis:latest deploy-development: uv run poe deploy --environment=development @@ -31,7 +31,7 @@ deploy-production: uv run poe deploy --environment=production run-development: - uv run poe run -- daft-analysis/hm-development-daft-analysis + uv run poe run -- hm-daft-analysis/development-hm-daft-analysis run-production: - uv run poe run -- daft-analysis/hm-production-daft-analysis + uv run poe run -- hm-daft-analysis/production-hm-daft-analysis diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py b/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py index 28972b05ae..b233bbc704 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py @@ -1,18 +1,33 @@ import daft +import ray from prefect import flow, get_run_logger @flow -def daft_analysis() -> None: +def hm_daft_analysis() -> None: logger = get_run_logger() + + ray_address = "ray://hm-ray-cluster-head-svc.production-hm-ray-cluster:10001" + ray.init( + ray_address, + runtime_env={ + "pip": [ + "getdaft==0.4.6", + ], + }, + ) + daft.context.set_runner_ray(ray_address) + df = daft.from_pydict( { - "a": [1, 2, 3, 4, 5], - "b": [True, True, False, False, False], + "a": [3, 2, 5, 6, 1, 4], + "b": [True, False, False, True, True, False], }, ) + df = df.where(df["b"]).sort(df["a"]) + df = df.collect() logger.info(f"{df = }") if __name__ == "__main__": - daft_analysis() + hm_daft_analysis() diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/src/workflow_deploy/utils/create_deployment.py b/data-orchestration/hm-prefect/workflows/daft-analysis/src/workflow_deploy/utils/create_deployment.py index 602ed04a56..476dcc923c 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/src/workflow_deploy/utils/create_deployment.py +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/src/workflow_deploy/utils/create_deployment.py @@ -11,7 +11,7 @@ async def create_deployment( docker_image_name: str, deployment: dict[str, Any], ) -> None: - deployment_name = f"hm-{environment}-{config.BASE_WORKFLOW_NAME}" + deployment_name = f"{environment}-hm-{config.BASE_WORKFLOW_NAME}" await workflow.deploy( name=deployment_name, work_pool_name=deployment["work_pool_name"], diff --git a/data-orchestration/hm-prefect/workflows/greet/justfile b/data-orchestration/hm-prefect/workflows/greet/justfile index de9276f57d..cf311dbcf1 100644 --- a/data-orchestration/hm-prefect/workflows/greet/justfile +++ b/data-orchestration/hm-prefect/workflows/greet/justfile @@ -17,12 +17,12 @@ test-coverage: uv run poe test-coverage publish-development-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-greet:development . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-greet:development + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-greet:development . && \ + docker push harbor.hongbomiao.com/hm/prefect-greet:development publish-production-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-greet:latest . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-greet:latest + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-greet:latest . && \ + docker push harbor.hongbomiao.com/hm/prefect-greet:latest deploy-development: uv run poe deploy --environment=development @@ -31,7 +31,7 @@ deploy-production: uv run poe deploy --environment=production run-development: - uv run poe run -- greet/hm-development-greet + uv run poe run -- hm-greet/development-hm-greet run-production: - uv run poe run -- greet/hm-production-greet + uv run poe run -- hm-greet/production-hm-greet diff --git a/data-orchestration/hm-prefect/workflows/greet/src/workflow_deploy/utils/create_deployment.py b/data-orchestration/hm-prefect/workflows/greet/src/workflow_deploy/utils/create_deployment.py index 84d52d62bd..a47ef54c82 100644 --- a/data-orchestration/hm-prefect/workflows/greet/src/workflow_deploy/utils/create_deployment.py +++ b/data-orchestration/hm-prefect/workflows/greet/src/workflow_deploy/utils/create_deployment.py @@ -11,7 +11,7 @@ async def create_deployment( docker_image_name: str, deployment: dict[str, Any], ) -> None: - deployment_name = f"hm-{environment}-{config.BASE_WORKFLOW_NAME}" + deployment_name = f"{environment}-hm-{config.BASE_WORKFLOW_NAME}" await workflow.deploy( name=deployment_name, work_pool_name=deployment["work_pool_name"], diff --git a/data-orchestration/hm-prefect/workflows/print-platform/justfile b/data-orchestration/hm-prefect/workflows/print-platform/justfile index fefe25f894..b4ef25a928 100644 --- a/data-orchestration/hm-prefect/workflows/print-platform/justfile +++ b/data-orchestration/hm-prefect/workflows/print-platform/justfile @@ -17,12 +17,12 @@ test-coverage: uv run poe test-coverage publish-development-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-print-platform:development . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-print-platform:development + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-print-platform:development . && \ + docker push harbor.hongbomiao.com/hm/prefect-print-platform:development publish-production-docker-image: - docker build --platform=linux/amd64 --file=Dockerfile --tag=ghcr.io/hongbo-miao/hm-prefect-print-platform:latest . && \ - docker push ghcr.io/hongbo-miao/hm-prefect-print-platform:latest + docker build --platform=linux/amd64 --file=Dockerfile --tag=harbor.hongbomiao.com/hm/prefect-print-platform:latest . && \ + docker push harbor.hongbomiao.com/hm/prefect-print-platform:latest deploy-development: uv run poe deploy --environment=development @@ -31,7 +31,7 @@ deploy-production: uv run poe deploy --environment=production run-development: - uv run poe run -- print-platform/hm-development-print-platform + uv run poe run -- hm-print-platform/development-hm-print-platform run-production: - uv run poe run -- print-platform/hm-production-print-platform + uv run poe run -- hm-print-platform/production-hm-print-platform diff --git a/data-orchestration/hm-prefect/workflows/print-platform/src/workflow_deploy/utils/create_deployment.py b/data-orchestration/hm-prefect/workflows/print-platform/src/workflow_deploy/utils/create_deployment.py index 84d52d62bd..a47ef54c82 100644 --- a/data-orchestration/hm-prefect/workflows/print-platform/src/workflow_deploy/utils/create_deployment.py +++ b/data-orchestration/hm-prefect/workflows/print-platform/src/workflow_deploy/utils/create_deployment.py @@ -11,7 +11,7 @@ async def create_deployment( docker_image_name: str, deployment: dict[str, Any], ) -> None: - deployment_name = f"hm-{environment}-{config.BASE_WORKFLOW_NAME}" + deployment_name = f"{environment}-hm-{config.BASE_WORKFLOW_NAME}" await workflow.deploy( name=deployment_name, work_pool_name=deployment["work_pool_name"], diff --git a/kubernetes/argo-cd/applications/production-hm/airbyte/justfile b/kubernetes/argo-cd/applications/production-hm/airbyte/justfile index 688bd5898d..c10f2504c0 100644 --- a/kubernetes/argo-cd/applications/production-hm/airbyte/justfile +++ b/kubernetes/argo-cd/applications/production-hm/airbyte/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-airbyte-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-airbyte-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-airbyte-secret.yaml argo-cd-app-create: argocd app create production-hm-airbyte --file=argo-cd-manifests/hm-airbyte-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/clickhouse/justfile b/kubernetes/argo-cd/applications/production-hm/clickhouse/justfile index d416f088b2..9c34942604 100644 --- a/kubernetes/argo-cd/applications/production-hm/clickhouse/justfile +++ b/kubernetes/argo-cd/applications/production-hm/clickhouse/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-clickhouse-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-clickhouse-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-clickhouse-secret.yaml argo-cd-app-create: argocd app create production-hm-clickhouse --file=argo-cd-manifests/hm-clickhouse-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/grafana/justfile b/kubernetes/argo-cd/applications/production-hm/grafana/justfile index 3fb6f12858..46228dd4ba 100644 --- a/kubernetes/argo-cd/applications/production-hm/grafana/justfile +++ b/kubernetes/argo-cd/applications/production-hm/grafana/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-grafana-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-grafana-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-grafana-secret.yaml argo-cd-app-create: argocd app create production-hm-grafana --file=argo-cd-manifests/hm-grafana-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/harbor/justfile b/kubernetes/argo-cd/applications/production-hm/harbor/justfile index 96f194cff7..21f7fa7545 100644 --- a/kubernetes/argo-cd/applications/production-hm/harbor/justfile +++ b/kubernetes/argo-cd/applications/production-hm/harbor/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-harbor-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-harbor-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-harbor-secret.yaml argo-cd-app-create: argocd app create production-hm-harbor --file=argo-cd-manifests/hm-harbor-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/mlflow/justfile b/kubernetes/argo-cd/applications/production-hm/mlflow/justfile index a2d7086d79..157bb52583 100644 --- a/kubernetes/argo-cd/applications/production-hm/mlflow/justfile +++ b/kubernetes/argo-cd/applications/production-hm/mlflow/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-mlflow-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-mlflow-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-mlflow-secret.yaml argo-cd-app-create: argocd app create production-hm-mlflow --file=argo-cd-manifests/hm-mlflow-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/odoo/justfile b/kubernetes/argo-cd/applications/production-hm/odoo/justfile index 09a1949dd8..3b597eb2dd 100644 --- a/kubernetes/argo-cd/applications/production-hm/odoo/justfile +++ b/kubernetes/argo-cd/applications/production-hm/odoo/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-odoo-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-odoo-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-odoo-secret.yaml argo-cd-app-create: argocd app create production-hm-odoo --file=argo-cd-manifests/hm-odoo-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/open-webui-pipelines/justfile b/kubernetes/argo-cd/applications/production-hm/open-webui-pipelines/justfile index 49043848af..579fa70f1f 100644 --- a/kubernetes/argo-cd/applications/production-hm/open-webui-pipelines/justfile +++ b/kubernetes/argo-cd/applications/production-hm/open-webui-pipelines/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-open-webui-pipelines-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-open-webui-pipelines-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-open-webui-pipelines-secret.yaml argo-cd-app-create: argocd app create production-hm-open-webui-pipelines --file=argo-cd-manifests/hm-open-webui-pipelines-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/open-webui/justfile b/kubernetes/argo-cd/applications/production-hm/open-webui/justfile index a30075df5b..b070bfeca1 100644 --- a/kubernetes/argo-cd/applications/production-hm/open-webui/justfile +++ b/kubernetes/argo-cd/applications/production-hm/open-webui/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-open-webui-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-open-webui-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-open-webui-secret.yaml argo-cd-app-create: argocd app create production-hm-open-webui --file=argo-cd-manifests/hm-open-webui-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/prefect-server/justfile b/kubernetes/argo-cd/applications/production-hm/prefect-server/justfile index a94c12993b..168d39329b 100644 --- a/kubernetes/argo-cd/applications/production-hm/prefect-server/justfile +++ b/kubernetes/argo-cd/applications/production-hm/prefect-server/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-prefect-server-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-prefect-server-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-prefect-server-secret.yaml argo-cd-app-create: argocd app create production-hm-prefect-server --file=argo-cd-manifests/hm-prefect-server-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-harbor-secret.yaml b/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-harbor-secret.yaml new file mode 100644 index 0000000000..1ea16ec2ec --- /dev/null +++ b/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-harbor-secret.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: hm-harbor-secret + namespace: production-hm-prefect-worker + annotations: + # https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks + argocd.argoproj.io/hook: PreSync + labels: + app.kubernetes.io/name: hm-harbor-secret + app.kubernetes.io/part-of: production-hm-prefect-worker +spec: + encryptedData: + .dockerconfigjson: AgAXYcpIY+1Rgv/3hUETkDtvPbIzHfOu+c2vUAhv7y5ZnV/vwxC/s8novk6K4s4DpAjM9Oqkf3yyfbtj95eU0kEWVcWkJMoSO2D9ddVSByiTbJmevVkKjJVdxXeMK2BrJlc53x7JQRUyx0vIhaePtEs1p1Mq/q2GYZ7J6M0mhdD/tv/KBDvYj19bqpMb4LHE2z7dhZKHdT3Kf7s5dLLdqdPCHYiL5S8U5Z5+lLB3YGLfKmZ//OfXiytvxsBRJaV6RvDrgRKHEVXsdY81GNt9gCQ1SYwxeKaL/Hh8NFgLoVH9tvmvqbD6XKpqsm9UdUEgDrnmgmjQTgEw6OTXbX/RYUbfc9Qrm7JL8yypXMNVzWC3bviABXaoUYHRQPzlT+LgklDfurJ92GBHNIwlNMthSn8GRgPMMMTTFEcKL3ZSwa2u8AKmbQwQDurpdnffvMUESfB8WslZWyp2JOTHuU/0HgAMDhGKyPjLpJWQPHCKQaIHG1OiRv7TnSJyYZDYOlkXAmMNDEt7ici0dE1PtFbhzUQsn9vvd86tKAh59CCGTNxuVlXpgHr3MuAY4VdOvkyh2xjCYucwSwd1iASpzglshcyZCf4iDIsihLH6eBLfwA5eFLJyWyv/ZsChijgegdEHCho1SmoWObwcxJbcjGL6jSaURDhUXXWuS85xFYlVND/XOmbNeILPtFfCFEL6GK42em9LbgvwOLLtzlWnbgBP1c6YZtU9HTiz0o5P9wNRuRTH0O/IfSjxCxmvItgLHRevhlxLWKUH+h1Kon7w27SOoGuBeTJpyP8UcLPCy+uPdlRP+B7l8cxX5SKrYt03fSepxCg8xtAhfFmpyR8pOvZx9Vj4MiVMbKUDFAwi4+VGkQCx1d0= + template: + type: kubernetes.io/dockerconfigjson + metadata: + annotations: + # https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks + argocd.argoproj.io/hook: PreSync + labels: + app.kubernetes.io/name: hm-harbor-secret + app.kubernetes.io/part-of: production-hm-prefect-worker + name: hm-harbor-secret + namespace: production-hm-prefect-worker diff --git a/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml b/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml index fdda68d15c..d85ce02091 100644 --- a/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml +++ b/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml @@ -67,6 +67,11 @@ spec: "name": "prefect-job" } ], + "imagePullSecrets": [ + { + "name": "hm-harbor-secret" + } + ], "parallelism": 1, "restartPolicy": "Never", "serviceAccountName": "{{ service_account_name }}" diff --git a/kubernetes/argo-cd/applications/production-hm/prefect-worker/justfile b/kubernetes/argo-cd/applications/production-hm/prefect-worker/justfile index 21e28a11d8..61fcbb6287 100644 --- a/kubernetes/argo-cd/applications/production-hm/prefect-worker/justfile +++ b/kubernetes/argo-cd/applications/production-hm/prefect-worker/justfile @@ -1,10 +1,18 @@ -sealed-secrets-seal: +sealed-secrets-seal-hm-prefect-worker-secret: cat secrets/hm-prefect-worker-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-prefect-worker-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-prefect-worker-secret.yaml + +sealed-secrets-seal-hm-harbor-secret: + cat secrets/hm-harbor-secret.unsealed.yaml | \ + kubeseal \ + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-harbor-secret.yaml argo-cd-app-create: argocd app create production-hm-prefect-worker --file=argo-cd-manifests/hm-prefect-worker-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/prefect-worker/secrets/hm-harbor-secret.unsealed.yaml.template b/kubernetes/argo-cd/applications/production-hm/prefect-worker/secrets/hm-harbor-secret.unsealed.yaml.template new file mode 100644 index 0000000000..dd73baeefc --- /dev/null +++ b/kubernetes/argo-cd/applications/production-hm/prefect-worker/secrets/hm-harbor-secret.unsealed.yaml.template @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: hm-harbor-secret + namespace: production-hm-prefect-worker + annotations: + # https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks + argocd.argoproj.io/hook: PreSync + labels: + app.kubernetes.io/name: hm-harbor-secret + app.kubernetes.io/part-of: production-hm-prefect-worker +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + { + "auths": { + "harbor.hongobomiao.com": { + "username": "robot$hm-kubernetes-robot", + "password": "xxx" + } + } + } diff --git a/kubernetes/argo-cd/applications/production-hm/ray-cluster-valkey/justfile b/kubernetes/argo-cd/applications/production-hm/ray-cluster-valkey/justfile index 699d44bde0..c0b2f4dc78 100644 --- a/kubernetes/argo-cd/applications/production-hm/ray-cluster-valkey/justfile +++ b/kubernetes/argo-cd/applications/production-hm/ray-cluster-valkey/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-ray-cluster-valkey-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-ray-cluster-valkey-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-ray-cluster-valkey-secret.yaml argo-cd-app-create: argocd app create production-hm-ray-cluster-valkey --file=manifests/hm-ray-cluster-valkey-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/ray-cluster/justfile b/kubernetes/argo-cd/applications/production-hm/ray-cluster/justfile index 5f2045e4ac..58ed33e151 100644 --- a/kubernetes/argo-cd/applications/production-hm/ray-cluster/justfile +++ b/kubernetes/argo-cd/applications/production-hm/ray-cluster/justfile @@ -1,10 +1,10 @@ sealed-secrets-seal: cat secrets/hm-ray-cluster-secret.unsealed.yaml | \ kubeseal \ - --controller-namespace=production-hm-sealed-secrets \ - --controller-name=hm-sealed-secrets \ - --format=yaml \ - > kubernetes-manifests/hm-ray-cluster-secret.yaml + --controller-namespace=production-hm-sealed-secrets \ + --controller-name=hm-sealed-secrets \ + --format=yaml \ + > kubernetes-manifests/hm-ray-cluster-secret.yaml argo-cd-app-create: argocd app create production-hm-ray-cluster --file=argo-cd-manifests/hm-ray-cluster-application.yaml diff --git a/kubernetes/argo-cd/applications/production-hm/ray-cluster/kubernetes-manifests/hm-ray-cluster.yaml b/kubernetes/argo-cd/applications/production-hm/ray-cluster/kubernetes-manifests/hm-ray-cluster.yaml index 5b81da25c0..942a12d291 100644 --- a/kubernetes/argo-cd/applications/production-hm/ray-cluster/kubernetes-manifests/hm-ray-cluster.yaml +++ b/kubernetes/argo-cd/applications/production-hm/ray-cluster/kubernetes-manifests/hm-ray-cluster.yaml @@ -374,7 +374,7 @@ spec: rayStartParams: {} template: spec: - priorityClassName: high + priorityClassName: medium serviceAccountName: hm-ray-cluster-service-account restartPolicy: Never containers: @@ -427,7 +427,7 @@ spec: rayStartParams: {} template: spec: - priorityClassName: high + priorityClassName: medium serviceAccountName: hm-ray-cluster-service-account restartPolicy: Never containers: @@ -480,7 +480,7 @@ spec: rayStartParams: {} template: spec: - priorityClassName: high + priorityClassName: medium serviceAccountName: hm-ray-cluster-service-account restartPolicy: Never containers: