From 5bf3f1fb1d64a1aff1140fa04a9c2a54995f93ff Mon Sep 17 00:00:00 2001 From: Hongbo Miao <3375461+hongbo-miao@users.noreply.github.com> Date: Sat, 8 Mar 2025 01:02:06 -0800 Subject: [PATCH] feat(daft): read delta table --- .github/workflows/build.yml | 8 +- .../production/aws/general/main.tf | 2 +- .../production/aws/kubernetes/main.tf | 41 ++- .../main.tf | 2 +- .../hm_prefect_worker_iam_role/main.tf | 99 ++++++ .../hm_prefect_worker_iam_role/variables.tf | 24 ++ .../hm_ray_cluster_iam_role/main.tf | 39 ++- .../hm_ray_cluster_iam_role/variables.tf | 5 +- cloud-platform/aws/aws-glue/justfile | 6 +- .../workflows/calculate/.prefectignore | 2 +- .../workflows/calculate/src/deploy.py | 6 +- .../workflows/calculate/src/main.py | 4 +- .../workflows/daft-analysis/.prefectignore | 2 +- .../workflows/daft-analysis/pyproject.toml | 2 +- .../workflows/daft-analysis/src/deploy.py | 6 +- .../workflows/daft-analysis/src/main.py | 19 +- .../workflows/daft-analysis/uv.lock | 301 +++++++++++++++++- .../hm-prefect/workflows/greet/.prefectignore | 2 +- .../hm-prefect/workflows/greet/src/deploy.py | 6 +- .../hm-prefect/workflows/greet/src/main.py | 4 +- .../workflows/print-platform/.prefectignore | 2 +- .../workflows/print-platform/src/deploy.py | 6 +- .../workflows/print-platform/src/main.py | 4 +- .../read-delta-lake-by-trino-matlab/main.m | 32 +- .../read-delta-lake-by-trino/src/main.py | 32 +- .../hm-prefect-worker-application.yaml | 2 +- trino/queries/delta_lake_motor_data.sql | 66 ++-- 27 files changed, 597 insertions(+), 127 deletions(-) create mode 100644 cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/main.tf create mode 100644 cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/variables.tf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 917b44576c..d4bd566598 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -429,7 +429,7 @@ jobs: attempt_delay: 2000 build-hm-prefect-calculate: - name: Build hm-prefect-calculate + name: Build prefect-calculate needs: detect-changes if: ${{ needs.detect-changes.outputs.hm-prefect-calculate == 'true' }} runs-on: ubuntu-24.04 @@ -465,7 +465,7 @@ jobs: attempt_delay: 2000 build-hm-prefect-daft-analysis: - name: Build hm-prefect-daft-analysis + name: Build prefect-daft-analysis needs: detect-changes if: ${{ needs.detect-changes.outputs.hm-prefect-daft-analysis == 'true' }} runs-on: ubuntu-24.04 @@ -501,7 +501,7 @@ jobs: attempt_delay: 2000 build-hm-prefect-greet: - name: Build hm-prefect-greet + name: Build prefect-greet needs: detect-changes if: ${{ needs.detect-changes.outputs.hm-prefect-greet == 'true' }} runs-on: ubuntu-24.04 @@ -537,7 +537,7 @@ jobs: attempt_delay: 2000 build-hm-prefect-print-platform: - name: Build hm-prefect-print-platform + name: Build prefect-print-platform needs: detect-changes if: ${{ needs.detect-changes.outputs.hm-prefect-print-platform == 'true' }} runs-on: ubuntu-24.04 diff --git a/cloud-infrastructure/terraform/environments/production/aws/general/main.tf b/cloud-infrastructure/terraform/environments/production/aws/general/main.tf index 32ca9e0fec..4f164d1912 100644 --- a/cloud-infrastructure/terraform/environments/production/aws/general/main.tf +++ b/cloud-infrastructure/terraform/environments/production/aws/general/main.tf @@ -455,7 +455,7 @@ module "hm_glue_crawler_motor_data" { source = "../../../../modules/aws/hm_aws_glue_crawler" aws_glue_crawler_name = "hm-delta-lake-crawler-iot" aws_glue_crawler_delta_tables = ["s3://hm-production-bucket/delta-tables/motor_data/"] - aws_glue_database = "production_hm_delta_db" + aws_glue_database = "production_motor_db" schedule = "cron(40 9 * * ? *)" # Every day at 9:40 UTC https://crontab.cronhub.io/ iam_role_arn = "arn:aws:iam::272394222652:role/service-role/AWSGlueServiceRole-hm" environment = var.environment diff --git a/cloud-infrastructure/terraform/environments/production/aws/kubernetes/main.tf b/cloud-infrastructure/terraform/environments/production/aws/kubernetes/main.tf index 23b80379f5..ff1df182fd 100644 --- a/cloud-infrastructure/terraform/environments/production/aws/kubernetes/main.tf +++ b/cloud-infrastructure/terraform/environments/production/aws/kubernetes/main.tf @@ -612,7 +612,8 @@ module "ray_cluster_iam_role" { ray_cluster_namespace = "${var.environment}-hm-ray-cluster" amazon_eks_cluster_oidc_provider = module.amazon_eks_cluster.oidc_provider amazon_eks_cluster_oidc_provider_arn = module.amazon_eks_cluster.oidc_provider_arn - s3_bucket_name = module.amazon_s3_bucket_hm_mlflow.name + mlflow_s3_bucket_name = module.amazon_s3_bucket_hm_mlflow.name + iot_data_s3_bucket_name = "iot-data-bucket" environment = var.environment team = var.team } @@ -628,7 +629,7 @@ module "kubernetes_namespace_hm_ray_cluster" { ] } # Ray Cluster Valkey - Kubernetes namespace -module "hm_kubernetes_namespace_hm_ray_cluster_valkey" { +module "kubernetes_namespace_hm_ray_cluster_valkey" { source = "../../../../modules/kubernetes/hm_kubernetes_namespace" kubernetes_namespace = "${var.environment}-hm-ray-cluster-valkey" labels = { @@ -680,7 +681,7 @@ module "s3_bucket_hm_mimir_ruler" { environment = var.environment team = var.team } -module "hm_mimir_iam_role" { +module "mimir_iam_role" { providers = { aws = aws.production } source = "../../../../modules/kubernetes/hm_mimir_iam_role" mimir_service_account_name = "hm-mimir" @@ -733,7 +734,7 @@ module "s3_bucket_hm_loki_ruler" { environment = var.environment team = var.team } -module "hm_loki_iam_role" { +module "loki_iam_role" { providers = { aws = aws.production } source = "../../../../modules/kubernetes/hm_loki_iam_role" loki_service_account_name = "hm-loki" @@ -791,7 +792,7 @@ module "s3_bucket_hm_tempo_trace" { environment = var.environment team = var.team } -module "hm_tempo_iam_role" { +module "tempo_iam_role" { providers = { aws = aws.production } source = "../../../../modules/kubernetes/hm_tempo_iam_role" tempo_service_account_name = "hm-tempo" @@ -896,7 +897,7 @@ data "aws_secretsmanager_secret_version" "hm_prefect_postgres_secret_version" { provider = aws.production secret_id = data.aws_secretsmanager_secret.hm_prefect_postgres_secret.id } -module "hm_prefect_postgres_security_group" { +module "prefect_postgres_security_group" { providers = { aws = aws.production } source = "../../../../modules/aws/hm_amazon_rds_security_group" amazon_ec2_security_group_name = "${local.prefect_postgres_name}-security-group" @@ -905,7 +906,7 @@ module "hm_prefect_postgres_security_group" { environment = var.environment team = var.team } -module "hm_prefect_postgres_subnet_group" { +module "prefect_postgres_subnet_group" { providers = { aws = aws.production } source = "../../../../modules/aws/hm_amazon_rds_subnet_group" subnet_group_name = "${local.prefect_postgres_name}-subnet-group" @@ -913,7 +914,7 @@ module "hm_prefect_postgres_subnet_group" { environment = var.environment team = var.team } -module "hm_prefect_postgres_parameter_group" { +module "prefect_postgres_parameter_group" { providers = { aws = aws.production } source = "../../../../modules/aws/hm_amazon_rds_parameter_group" family = "postgres17" @@ -921,7 +922,7 @@ module "hm_prefect_postgres_parameter_group" { environment = var.environment team = var.team } -module "hm_prefect_postgres_instance" { +module "prefect_postgres_instance" { providers = { aws = aws.production } source = "../../../../modules/aws/hm_amazon_rds_instance" amazon_rds_name = local.prefect_postgres_name @@ -940,7 +941,7 @@ module "hm_prefect_postgres_instance" { team = var.team } # Prefect Server - Kubernetes namespace -module "hm_kubernetes_namespace_hm_prefect_server" { +module "kubernetes_namespace_hm_prefect_server" { source = "../../../../modules/kubernetes/hm_kubernetes_namespace" kubernetes_namespace = "${var.environment}-hm-prefect-server" labels = { @@ -950,8 +951,24 @@ module "hm_kubernetes_namespace_hm_prefect_server" { module.hm_amazon_eks_cluster ] } +# Prefect Worker - IAM role +module "prefect_worker_iam_role" { + providers = { aws = aws.production } + source = "../../../../modules/kubernetes/hm_prefect_worker_iam_role" + prefect_worker_service_account_name = "hm-prefect-worker" + prefect_worker_namespace = "${var.environment}-hm-prefect-worker" + amazon_eks_cluster_oidc_provider = module.hm_amazon_eks_cluster.oidc_provider + amazon_eks_cluster_oidc_provider_arn = module.hm_amazon_eks_cluster.oidc_provider_arn + iot_data_s3_bucket_name = "iot-data-bucket" + aws_glue_database_names = [ + "${var.environment}_battery_db", + "${var.environment}_motor_db" + ] + environment = var.environment + team = var.team +} # Prefect Worker - Kubernetes namespace -module "hm_kubernetes_namespace_hm_prefect_worker" { +module "kubernetes_namespace_hm_prefect_worker" { source = "../../../../modules/kubernetes/hm_kubernetes_namespace" kubernetes_namespace = "${var.environment}-hm-prefect-worker" labels = { @@ -1090,7 +1107,7 @@ module "kubernetes_namespace_hm_kafbat_ui" { # LiteLLM # LiteLLM - IAM role -module "hm_litellm_iam_role" { +module "litellm_iam_role" { providers = { aws = aws.production } source = "../../../../modules/kubernetes/hm_litellm_iam_role" litellm_service_account_name = "hm-litellm-service-account" diff --git a/cloud-infrastructure/terraform/modules/kubernetes/hm_amazon_s3_csi_driver_mountpoint_iam_role/main.tf b/cloud-infrastructure/terraform/modules/kubernetes/hm_amazon_s3_csi_driver_mountpoint_iam_role/main.tf index 19e8848e2b..20d2d97414 100644 --- a/cloud-infrastructure/terraform/modules/kubernetes/hm_amazon_s3_csi_driver_mountpoint_iam_role/main.tf +++ b/cloud-infrastructure/terraform/modules/kubernetes/hm_amazon_s3_csi_driver_mountpoint_iam_role/main.tf @@ -70,7 +70,7 @@ resource "aws_iam_role_policy" "eks_cluster_s3_policy" { } # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy resource "aws_iam_role_policy" "iot_data_s3_policy" { - name = "${local.aws_iam_role_name_prefix}IoTDataS3Policy-${var.amazon_eks_cluster_name}" + name = "${local.aws_iam_role_name_prefix}IotDataS3Policy-${var.amazon_eks_cluster_name}" role = aws_iam_role.s3_csi_driver_mountpoint_role.name policy = jsonencode({ Version = "2012-10-17" diff --git a/cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/main.tf b/cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/main.tf new file mode 100644 index 0000000000..dd30163378 --- /dev/null +++ b/cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/main.tf @@ -0,0 +1,99 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } +} + +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +locals { + aws_iam_role_name_prefix = "PrefectWorkerRole" +} +# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role +resource "aws_iam_role" "prefect_worker_iam_role" { + name = "${local.aws_iam_role_name_prefix}-${var.prefect_worker_service_account_name}" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Federated = var.amazon_eks_cluster_oidc_provider_arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${var.amazon_eks_cluster_oidc_provider}:aud" = "sts.amazonaws.com", + "${var.amazon_eks_cluster_oidc_provider}:sub" = "system:serviceaccount:${var.prefect_worker_namespace}:${var.prefect_worker_service_account_name}" + } + } + } + ] + }) + tags = { + Environment = var.environment + Team = var.team + Name = "${local.aws_iam_role_name_prefix}-${var.prefect_worker_service_account_name}" + } +} +# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy +resource "aws_iam_role_policy" "iot_data_s3_policy" { + name = "${local.aws_iam_role_name_prefix}IotDataS3Policy-${var.prefect_worker_service_account_name}" + role = aws_iam_role.prefect_worker_iam_role.name + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:GetBucketLocation", + "s3:ListBucket" + ] + Resource = [ + "arn:aws:s3:::${var.iot_data_s3_bucket_name}" + ] + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:PutObject" + ] + Resource = [ + "arn:aws:s3:::${var.iot_data_s3_bucket_name}/*" + ] + } + ] + }) +} +# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy +resource "aws_iam_role_policy" "aws_glue_policy" { + name = "${local.aws_iam_role_name_prefix}AwsGluePolicy-${var.prefect_worker_service_account_name}" + role = aws_iam_role.prefect_worker_iam_role.name + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "glue:GetDatabase", + "glue:GetDatabases", + "glue:GetTable", + "glue:GetTables" + ] + Resource = flatten([ + "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:catalog", + [for database in var.aws_glue_database_names : + "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:database/${database}" + ], + [for database in var.aws_glue_database_names : + "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:table/${database}/*" + ] + ]) + } + ] + }) +} diff --git a/cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/variables.tf b/cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/variables.tf new file mode 100644 index 0000000000..5b336e140d --- /dev/null +++ b/cloud-infrastructure/terraform/modules/kubernetes/hm_prefect_worker_iam_role/variables.tf @@ -0,0 +1,24 @@ +variable "prefect_worker_service_account_name" { + type = string +} +variable "prefect_worker_namespace" { + type = string +} +variable "iot_data_s3_bucket_name" { + type = string +} +variable "aws_glue_database_names" { + type = list(string) +} +variable "amazon_eks_cluster_oidc_provider" { + type = string +} +variable "amazon_eks_cluster_oidc_provider_arn" { + type = string +} +variable "environment" { + type = string +} +variable "team" { + type = string +} diff --git a/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/main.tf b/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/main.tf index 34e1df16f2..6e92c56b14 100644 --- a/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/main.tf +++ b/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/main.tf @@ -10,7 +10,7 @@ locals { aws_iam_role_name_prefix = "RayClusterRole" } # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role -resource "aws_iam_role" "ray_cluster_role" { +resource "aws_iam_role" "ray_cluster_iam_role" { name = "${local.aws_iam_role_name_prefix}-${var.ray_cluster_service_account_name}" assume_role_policy = jsonencode({ Version = "2012-10-17" @@ -37,9 +37,9 @@ resource "aws_iam_role" "ray_cluster_role" { } } # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy -resource "aws_iam_role_policy" "ray_cluster_role_policy" { - name = "${local.aws_iam_role_name_prefix}Policy-${var.ray_cluster_service_account_name}" - role = aws_iam_role.ray_cluster_role.name +resource "aws_iam_role_policy" "mlflow_s3_policy" { + name = "${local.aws_iam_role_name_prefix}MLflowS3Policy-${var.ray_cluster_service_account_name}" + role = aws_iam_role.ray_cluster_iam_role.name policy = jsonencode({ Version = "2012-10-17" Statement = [ @@ -49,7 +49,36 @@ resource "aws_iam_role_policy" "ray_cluster_role_policy" { "s3:PutObject" ] Resource = [ - "arn:aws:s3:::${var.s3_bucket_name}/*" + "arn:aws:s3:::${var.mlflow_s3_bucket_name}/*" + ] + } + ] + }) +} +# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy +resource "aws_iam_role_policy" "iot_data_s3_policy" { + name = "${local.aws_iam_role_name_prefix}IotDataS3Policy-${var.ray_cluster_service_account_name}" + role = aws_iam_role.ray_cluster_iam_role.name + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:ListBucket" + ] + Resource = [ + "arn:aws:s3:::${var.iot_data_s3_bucket_name}" + ] + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:PutObject" + ] + Resource = [ + "arn:aws:s3:::${var.iot_data_s3_bucket_name}/*" ] } ] diff --git a/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/variables.tf b/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/variables.tf index 826cf77a62..9a5a6711a5 100644 --- a/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/variables.tf +++ b/cloud-infrastructure/terraform/modules/kubernetes/hm_ray_cluster_iam_role/variables.tf @@ -10,7 +10,10 @@ variable "amazon_eks_cluster_oidc_provider" { variable "amazon_eks_cluster_oidc_provider_arn" { type = string } -variable "s3_bucket_name" { +variable "mlflow_s3_bucket_name" { + type = string +} +variable "iot_data_s3_bucket_name" { type = string } variable "environment" { diff --git a/cloud-platform/aws/aws-glue/justfile b/cloud-platform/aws/aws-glue/justfile index cc7019b9ad..d1c5472329 100644 --- a/cloud-platform/aws/aws-glue/justfile +++ b/cloud-platform/aws/aws-glue/justfile @@ -8,13 +8,13 @@ aws-glue-crawler-connection-get: aws glue get-connection --name=hm-postgres-connection aws-glue-partition-index-list: - aws glue get-partition-indexes --database-name=production_hm_delta_db --table-name=motor + aws glue get-partition-indexes --database-name=production_motor_db --table-name=motor aws-glue-partition-index-create: - aws glue create-partition-index --database-name=production_hm_delta_db --table-name=motor --partition-index=Keys=_event_id,IndexName=_event_id_idx + aws glue create-partition-index --database-name=production_motor_db --table-name=motor --partition-index=Keys=_event_id,IndexName=_event_id_idx aws-glue-partition-index-delete: - aws glue delete-partition-index --database-name=production_hm_delta_db --table-name=motor --index-name=_event_id_idx + aws glue delete-partition-index --database-name=production_motor_db --table-name=motor --index-name=_event_id_idx create-adsb-db-delta-tables: bash bin/create_adsb_db_delta_tables.sh diff --git a/data-orchestration/hm-prefect/workflows/calculate/.prefectignore b/data-orchestration/hm-prefect/workflows/calculate/.prefectignore index fdfc280f7c..9a562837d3 100644 --- a/data-orchestration/hm-prefect/workflows/calculate/.prefectignore +++ b/data-orchestration/hm-prefect/workflows/calculate/.prefectignore @@ -1,4 +1,4 @@ -# https://docs.prefect.io/concepts/deployments/?h=.prefectignore#build-the-deployment +# https://docs.prefect.io/v3/deploy/infrastructure-concepts/store-flow-code#include-or-exclude-files-from-storage # Anywhere **/*.aliases diff --git a/data-orchestration/hm-prefect/workflows/calculate/src/deploy.py b/data-orchestration/hm-prefect/workflows/calculate/src/deploy.py index f37d16f1dc..ee23d58754 100644 --- a/data-orchestration/hm-prefect/workflows/calculate/src/deploy.py +++ b/data-orchestration/hm-prefect/workflows/calculate/src/deploy.py @@ -8,7 +8,7 @@ from workflow_deploy.utils.create_deployment import create_deployment -async def deploy() -> None: +async def hm_calculate() -> None: args = get_args() match args.environment: @@ -20,7 +20,7 @@ async def deploy() -> None: logging.error("Not supported environment.") return - docker_image_name = f"ghcr.io/hongbo-miao/hm-prefect-{config.BASE_WORKFLOW_NAME}" + docker_image_name = f"harbor.hongbomiao.com/hm/prefect-{config.BASE_WORKFLOW_NAME}" await create_deployment( args.environment, calculate, @@ -31,4 +31,4 @@ async def deploy() -> None: if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - asyncio.run(deploy()) + asyncio.run(hm_calculate()) diff --git a/data-orchestration/hm-prefect/workflows/calculate/src/main.py b/data-orchestration/hm-prefect/workflows/calculate/src/main.py index 6363ac8a85..d1bfeaef92 100644 --- a/data-orchestration/hm-prefect/workflows/calculate/src/main.py +++ b/data-orchestration/hm-prefect/workflows/calculate/src/main.py @@ -40,7 +40,7 @@ def sum_up(nums: list[int]) -> int: @flow -def calculate(model: Model) -> None: +def hm_calculate(model: Model) -> None: logger = get_run_logger() nums = expand(model.n) nums = power.map(nums, 2) @@ -56,4 +56,4 @@ def calculate(model: Model) -> None: if __name__ == "__main__": external_model = Model(n=4) - calculate(external_model) + hm_calculate(external_model) diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/.prefectignore b/data-orchestration/hm-prefect/workflows/daft-analysis/.prefectignore index fdfc280f7c..9a562837d3 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/.prefectignore +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/.prefectignore @@ -1,4 +1,4 @@ -# https://docs.prefect.io/concepts/deployments/?h=.prefectignore#build-the-deployment +# https://docs.prefect.io/v3/deploy/infrastructure-concepts/store-flow-code#include-or-exclude-files-from-storage # Anywhere **/*.aliases diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/pyproject.toml b/data-orchestration/hm-prefect/workflows/daft-analysis/pyproject.toml index 6787cbe972..10fd4e7910 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/pyproject.toml +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/pyproject.toml @@ -3,7 +3,7 @@ name = "prefect-daft-analysis" version = "1.0.0" requires-python = "~=3.12.0" dependencies = [ - "getdaft==0.4.6", + "getdaft[aws,deltalake,ray]==0.4.6", "prefect-client==3.2.11", ] diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/src/deploy.py b/data-orchestration/hm-prefect/workflows/daft-analysis/src/deploy.py index 38f9abbb88..0900d3012e 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/src/deploy.py +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/src/deploy.py @@ -1,7 +1,7 @@ import asyncio import logging -from main import daft_analysis +from main import hm_daft_analysis from workflow_deploy import config from workflow_deploy.args import get_args from workflow_deploy.environments import Environments @@ -20,10 +20,10 @@ async def deploy() -> None: logging.error("Not supported environment.") return - docker_image_name = f"ghcr.io/hongbo-miao/hm-prefect-{config.BASE_WORKFLOW_NAME}" + docker_image_name = f"harbor.hongbomiao.com/hm/prefect-{config.BASE_WORKFLOW_NAME}" await create_deployment( args.environment, - daft_analysis, + hm_daft_analysis, docker_image_name, deployment, ) diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py b/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py index b233bbc704..1c6239b989 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/src/main.py @@ -1,5 +1,7 @@ import daft import ray +from daft import DataCatalogType +from daft.io.catalog import DataCatalogTable from prefect import flow, get_run_logger @@ -7,24 +9,25 @@ def hm_daft_analysis() -> None: logger = get_run_logger() - ray_address = "ray://hm-ray-cluster-head-svc.production-hm-ray-cluster:10001" + ray_address = "ray://hm-ray-cluster-head-svc.production-hm-ray-cluster.svc:10001" ray.init( ray_address, runtime_env={ "pip": [ - "getdaft==0.4.6", + "getdaft[aws,deltalake,ray]==0.4.6", ], }, ) daft.context.set_runner_ray(ray_address) - df = daft.from_pydict( - { - "a": [3, 2, 5, 6, 1, 4], - "b": [True, False, False, True, True, False], - }, + table = DataCatalogTable( + catalog=DataCatalogType.GLUE, + database_name="production_motor_db", + table_name="motor_data", ) - df = df.where(df["b"]).sort(df["a"]) + + df = daft.read_deltalake(table) + df = df.where(df["_event_id"] == "ad7953cd-6d49-4929-8180-99555bebc255") df = df.collect() logger.info(f"{df = }") diff --git a/data-orchestration/hm-prefect/workflows/daft-analysis/uv.lock b/data-orchestration/hm-prefect/workflows/daft-analysis/uv.lock index 0ea0913421..99f8728523 100644 --- a/data-orchestration/hm-prefect/workflows/daft-analysis/uv.lock +++ b/data-orchestration/hm-prefect/workflows/daft-analysis/uv.lock @@ -1,6 +1,24 @@ version = 1 revision = 1 requires-python = ">=3.12.0, <3.13" +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'darwin'", + "platform_machine != 'x86_64' and sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'win32'", + "sys_platform == 'win32'", +] + +[[package]] +name = "aiosignal" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/b5/6d55e80f6d8a08ce22b982eafa278d823b541c925f11ee774b0b9c43473d/aiosignal-1.3.2.tar.gz", hash = "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54", size = 19424 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl", hash = "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", size = 7597 }, +] [[package]] name = "aiosqlite" @@ -105,6 +123,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", size = 63397 }, ] +[[package]] +name = "boto3" +version = "1.37.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/f7/b870fb8d2ca96a996db97c9d30d1eb087b341cec1004722e99672a79800d/boto3-1.37.9.tar.gz", hash = "sha256:51b76da93d7c2a3dff6155ee4aa25455940e7ade08292d22aeeed08b9e0dbf0b", size = 111409 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/75/625b7e1cc2a8b470760273082e0c4d36c8ac8250ad2ff9df54528d1379b4/boto3-1.37.9-py3-none-any.whl", hash = "sha256:cfff6e9cf4b987480fbbe85873f3e609aeea1906631c7148e5b7e3c778aebd40", size = 139544 }, +] + +[[package]] +name = "botocore" +version = "1.37.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/9e/97ee17ec39395c67edfa694ae9e87177fdb7433e65be91bc84fef983dd86/botocore-1.37.9.tar.gz", hash = "sha256:2fdafbb9c44196cd371f4890aedf9f54352348fbae624a3880862d35724f0956", size = 13635008 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7f/5463336d2bf8eb5131c063e37599fa2f504abaa26b197aea2d2faea78a4c/botocore-1.37.9-py3-none-any.whl", hash = "sha256:bf0ab085ae85a4a2fa1733321069c1039745fa65ca9f335a91b8712fd6745d5f", size = 13403070 }, +] + [[package]] name = "cachetools" version = "5.5.0" @@ -274,6 +320,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/0a/981c438c4cd84147c781e4e96c1d72df03775deb1bc76c5a6ee8afa89c62/dateparser-1.2.1-py3-none-any.whl", hash = "sha256:bdcac262a467e6260030040748ad7c10d6bacd4f3b9cdb4cfd2251939174508c", size = 295658 }, ] +[[package]] +name = "deltalake" +version = "0.24.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'x86_64' and sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'win32'", + "sys_platform == 'win32'", +] +dependencies = [ + { name = "pyarrow", version = "19.0.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/3d/fa40c41cc07fb911f58db3485e4c5a7d0d65686189e361749aa28176d71b/deltalake-0.24.0.tar.gz", hash = "sha256:0a33528b40067ee204b527efd88cc1a4cea4b61af4886ca57539d5b6b657e20f", size = 4954245 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/aa/df35473f67a5d88270c87dd95048096b747c630ec0c3981aac2ddab17f81/deltalake-0.24.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1c4c5b2d3b8d467fb35504ef5649344bf3ad4f656303101e463ece3db2843c79", size = 40048854 }, + { url = "https://files.pythonhosted.org/packages/8a/cc/1d790ccac050e7888d2e8b39beee2debe604fd4ee9bca2448f2c73dd4e48/deltalake-0.24.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:3d59e62305d42b76c78e0c3dcb479d9ad512a2bbe12867c24e3ed2d66492d2b4", size = 37646363 }, + { url = "https://files.pythonhosted.org/packages/2d/19/0fdb5dc29293384a74ac4b3327c2015bf467e0d57e2f50b23658acab0343/deltalake-0.24.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59487ae82c7de5950ec951e2dad7d4b2b281e81fac7b65d8187dac64b1506359", size = 44491693 }, + { url = "https://files.pythonhosted.org/packages/f7/af/6d423118b7ce1f20a53e6eba2598b8494062a1f40c0da4ca3e7fe0815e47/deltalake-0.24.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05dc5ee8adb05cc4aaf8988335c14720b4e7b042d0d1976bde10c2a5cc3b9654", size = 45955463 }, + { url = "https://files.pythonhosted.org/packages/a1/74/be1d4d465b22526ecd2d2dc6b6cfa257285ac091a90ac86b99fcd043a7a5/deltalake-0.24.0-cp39-abi3-win_amd64.whl", hash = "sha256:1c423bdab46fa3c01a9364ba5533167d3b58099fd4a0599e6d0c363e9ba64799", size = 36586590 }, +] + +[[package]] +name = "deltalake" +version = "0.25.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/c5/eff8788f48b0d11d1d3ee1d4b7e4efd9f24444c6091f10b74448fe6fe02e/deltalake-0.25.4.tar.gz", hash = "sha256:67670235e18d61e2878e22925cddbfa9c80347ff2955acfa12d0114db5b77d7d", size = 4978356 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/cc/1da3067fb7bc3c6b337c47fd7d5c4e1d36a7182c491290e0c401ac90daea/deltalake-0.25.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:083ef408412ec85fc2ec80a22adb4c2bd1b144ed19a6aa6866b144410456841e", size = 41763966 }, +] + [[package]] name = "deprecated" version = "1.2.15" @@ -323,6 +405,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/b3/7e4df40e585df024fac2f80d1a2d579c854ac37109675db2b0cc22c0bb9e/fastapi-0.115.6-py3-none-any.whl", hash = "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305", size = 94843 }, ] +[[package]] +name = "filelock" +version = "3.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/9c/0b15fb47b464e1b663b1acd1253a062aa5feecb07d4e597daea542ebd2b5/filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e", size = 18027 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/ec/00d68c4ddfedfe64159999e5f8a98fb8442729a63e2077eb9dcd89623d27/filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338", size = 16164 }, +] + +[[package]] +name = "frozenlist" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/ed/0f4cec13a93c02c47ec32d81d11c0c1efbadf4a471e3f3ce7cad366cbbd3/frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817", size = 39930 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/73/fa6d1a96ab7fd6e6d1c3500700963eab46813847f01ef0ccbaa726181dd5/frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21", size = 94026 }, + { url = "https://files.pythonhosted.org/packages/ab/04/ea8bf62c8868b8eada363f20ff1b647cf2e93377a7b284d36062d21d81d1/frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d", size = 54150 }, + { url = "https://files.pythonhosted.org/packages/d0/9a/8e479b482a6f2070b26bda572c5e6889bb3ba48977e81beea35b5ae13ece/frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e", size = 51927 }, + { url = "https://files.pythonhosted.org/packages/e3/12/2aad87deb08a4e7ccfb33600871bbe8f0e08cb6d8224371387f3303654d7/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a", size = 282647 }, + { url = "https://files.pythonhosted.org/packages/77/f2/07f06b05d8a427ea0060a9cef6e63405ea9e0d761846b95ef3fb3be57111/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a", size = 289052 }, + { url = "https://files.pythonhosted.org/packages/bd/9f/8bf45a2f1cd4aa401acd271b077989c9267ae8463e7c8b1eb0d3f561b65e/frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee", size = 291719 }, + { url = "https://files.pythonhosted.org/packages/41/d1/1f20fd05a6c42d3868709b7604c9f15538a29e4f734c694c6bcfc3d3b935/frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6", size = 267433 }, + { url = "https://files.pythonhosted.org/packages/af/f2/64b73a9bb86f5a89fb55450e97cd5c1f84a862d4ff90d9fd1a73ab0f64a5/frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e", size = 283591 }, + { url = "https://files.pythonhosted.org/packages/29/e2/ffbb1fae55a791fd6c2938dd9ea779509c977435ba3940b9f2e8dc9d5316/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9", size = 273249 }, + { url = "https://files.pythonhosted.org/packages/2e/6e/008136a30798bb63618a114b9321b5971172a5abddff44a100c7edc5ad4f/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039", size = 271075 }, + { url = "https://files.pythonhosted.org/packages/ae/f0/4e71e54a026b06724cec9b6c54f0b13a4e9e298cc8db0f82ec70e151f5ce/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784", size = 285398 }, + { url = "https://files.pythonhosted.org/packages/4d/36/70ec246851478b1c0b59f11ef8ade9c482ff447c1363c2bd5fad45098b12/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631", size = 294445 }, + { url = "https://files.pythonhosted.org/packages/37/e0/47f87544055b3349b633a03c4d94b405956cf2437f4ab46d0928b74b7526/frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f", size = 280569 }, + { url = "https://files.pythonhosted.org/packages/f9/7c/490133c160fb6b84ed374c266f42800e33b50c3bbab1652764e6e1fc498a/frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8", size = 44721 }, + { url = "https://files.pythonhosted.org/packages/b1/56/4e45136ffc6bdbfa68c29ca56ef53783ef4c2fd395f7cbf99a2624aa9aaa/frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f", size = 51329 }, + { url = "https://files.pythonhosted.org/packages/c6/c8/a5be5b7550c10858fcf9b0ea054baccab474da77d37f1e828ce043a3a5d4/frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3", size = 11901 }, +] + [[package]] name = "fsspec" version = "2024.12.0" @@ -338,7 +453,8 @@ version = "0.4.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fsspec" }, - { name = "pyarrow" }, + { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, + { name = "pyarrow", version = "19.0.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" }, { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/08/fbd83b89952926a2ed1b9947311bb795cb45fb2f19c51a6a1ab12204a1f9/getdaft-0.4.6.tar.gz", hash = "sha256:02ba217dccc40f8e6f86a5019473046cb3b93fccfaac086b20bdefa22e708acf", size = 4554552 } @@ -350,6 +466,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/85/6bb4eac9378cb69121bbac2b15520df15282abc821d6df8480be047f5803/getdaft-0.4.6-cp39-abi3-win_amd64.whl", hash = "sha256:4ee39ba6f8ea034bb4c89109d70a12ad0f3bf9291fdb59c27e2774407463e3d4", size = 33099006 }, ] +[package.optional-dependencies] +aws = [ + { name = "boto3" }, +] +deltalake = [ + { name = "deltalake", version = "0.24.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" }, + { name = "deltalake", version = "0.25.4", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, +] +ray = [ + { name = "packaging" }, + { name = "ray", extra = ["client", "data"] }, +] + [[package]] name = "graphviz" version = "0.20.3" @@ -388,6 +517,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/88/52c9422bc853cd7c2b6122090e887d17b5fad29b67f930e4277c9c557357/griffe-1.5.5-py3-none-any.whl", hash = "sha256:2761b1e8876c6f1f9ab1af274df93ea6bbadd65090de5f38f4cb5cc84897c7dd", size = 128221 }, ] +[[package]] +name = "grpcio" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/e1/4b21b5017c33f3600dcc32b802bb48fe44a4d36d6c066f52650c7c2690fa/grpcio-1.70.0.tar.gz", hash = "sha256:8d1584a68d5922330025881e63a6c1b54cc8117291d382e4fa69339b6d914c56", size = 12788932 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/a4/ddbda79dd176211b518f0f3795af78b38727a31ad32bc149d6a7b910a731/grpcio-1.70.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:ef4c14508299b1406c32bdbb9fb7b47612ab979b04cf2b27686ea31882387cff", size = 5198135 }, + { url = "https://files.pythonhosted.org/packages/30/5c/60eb8a063ea4cb8d7670af8fac3f2033230fc4b75f62669d67c66ac4e4b0/grpcio-1.70.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:aa47688a65643afd8b166928a1da6247d3f46a2784d301e48ca1cc394d2ffb40", size = 11447529 }, + { url = "https://files.pythonhosted.org/packages/fb/b9/1bf8ab66729f13b44e8f42c9de56417d3ee6ab2929591cfee78dce749b57/grpcio-1.70.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:880bfb43b1bb8905701b926274eafce5c70a105bc6b99e25f62e98ad59cb278e", size = 5664484 }, + { url = "https://files.pythonhosted.org/packages/d1/06/2f377d6906289bee066d96e9bdb91e5e96d605d173df9bb9856095cccb57/grpcio-1.70.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e654c4b17d07eab259d392e12b149c3a134ec52b11ecdc6a515b39aceeec898", size = 6303739 }, + { url = "https://files.pythonhosted.org/packages/ae/50/64c94cfc4db8d9ed07da71427a936b5a2bd2b27c66269b42fbda82c7c7a4/grpcio-1.70.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2394e3381071045a706ee2eeb6e08962dd87e8999b90ac15c55f56fa5a8c9597", size = 5910417 }, + { url = "https://files.pythonhosted.org/packages/53/89/8795dfc3db4389c15554eb1765e14cba8b4c88cc80ff828d02f5572965af/grpcio-1.70.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b3c76701428d2df01964bc6479422f20e62fcbc0a37d82ebd58050b86926ef8c", size = 6626797 }, + { url = "https://files.pythonhosted.org/packages/9c/b2/6a97ac91042a2c59d18244c479ee3894e7fb6f8c3a90619bb5a7757fa30c/grpcio-1.70.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ac073fe1c4cd856ebcf49e9ed6240f4f84d7a4e6ee95baa5d66ea05d3dd0df7f", size = 6190055 }, + { url = "https://files.pythonhosted.org/packages/86/2b/28db55c8c4d156053a8c6f4683e559cd0a6636f55a860f87afba1ac49a51/grpcio-1.70.0-cp312-cp312-win32.whl", hash = "sha256:cd24d2d9d380fbbee7a5ac86afe9787813f285e684b0271599f95a51bce33528", size = 3600214 }, + { url = "https://files.pythonhosted.org/packages/17/c3/a7a225645a965029ed432e5b5e9ed959a574e62100afab553eef58be0e37/grpcio-1.70.0-cp312-cp312-win_amd64.whl", hash = "sha256:0495c86a55a04a874c7627fd33e5beaee771917d92c0e6d9d797628ac40e7655", size = 4292538 }, +] + [[package]] name = "h11" version = "0.14.0" @@ -525,6 +671,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/b4/08c9d297edd5e1182506edecccbb88a92e1122a057953068cadac420ca5d/jinja2_humanize_extension-0.4.0-py3-none-any.whl", hash = "sha256:b6326e2da0f7d425338bebf58848e830421defbce785f12ae812e65128518156", size = 4769 }, ] +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256 }, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -633,6 +788,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, ] +[[package]] +name = "msgpack" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/d0/7555686ae7ff5731205df1012ede15dd9d927f6227ea151e901c7406af4f/msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e", size = 167260 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d6/716b7ca1dbde63290d2973d22bbef1b5032ca634c3ff4384a958ec3f093a/msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d", size = 152421 }, + { url = "https://files.pythonhosted.org/packages/70/da/5312b067f6773429cec2f8f08b021c06af416bba340c912c2ec778539ed6/msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2", size = 85277 }, + { url = "https://files.pythonhosted.org/packages/28/51/da7f3ae4462e8bb98af0d5bdf2707f1b8c65a0d4f496e46b6afb06cbc286/msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420", size = 82222 }, + { url = "https://files.pythonhosted.org/packages/33/af/dc95c4b2a49cff17ce47611ca9ba218198806cad7796c0b01d1e332c86bb/msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2", size = 392971 }, + { url = "https://files.pythonhosted.org/packages/f1/54/65af8de681fa8255402c80eda2a501ba467921d5a7a028c9c22a2c2eedb5/msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39", size = 401403 }, + { url = "https://files.pythonhosted.org/packages/97/8c/e333690777bd33919ab7024269dc3c41c76ef5137b211d776fbb404bfead/msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f", size = 385356 }, + { url = "https://files.pythonhosted.org/packages/57/52/406795ba478dc1c890559dd4e89280fa86506608a28ccf3a72fbf45df9f5/msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247", size = 383028 }, + { url = "https://files.pythonhosted.org/packages/e7/69/053b6549bf90a3acadcd8232eae03e2fefc87f066a5b9fbb37e2e608859f/msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c", size = 391100 }, + { url = "https://files.pythonhosted.org/packages/23/f0/d4101d4da054f04274995ddc4086c2715d9b93111eb9ed49686c0f7ccc8a/msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b", size = 394254 }, + { url = "https://files.pythonhosted.org/packages/1c/12/cf07458f35d0d775ff3a2dc5559fa2e1fcd06c46f1ef510e594ebefdca01/msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b", size = 69085 }, + { url = "https://files.pythonhosted.org/packages/73/80/2708a4641f7d553a63bc934a3eb7214806b5b39d200133ca7f7afb0a53e8/msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f", size = 75347 }, +] + +[[package]] +name = "numpy" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/90/8956572f5c4ae52201fdec7ba2044b2c882832dcec7d5d0922c9e9acf2de/numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020", size = 20262700 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ec/43628dcf98466e087812142eec6d1c1a6c6bdfdad30a0aa07b872dc01f6f/numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d", size = 20929458 }, + { url = "https://files.pythonhosted.org/packages/9b/c0/2f4225073e99a5c12350954949ed19b5d4a738f541d33e6f7439e33e98e4/numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95", size = 14115299 }, + { url = "https://files.pythonhosted.org/packages/ca/fa/d2c5575d9c734a7376cc1592fae50257ec95d061b27ee3dbdb0b3b551eb2/numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea", size = 5145723 }, + { url = "https://files.pythonhosted.org/packages/eb/dc/023dad5b268a7895e58e791f28dc1c60eb7b6c06fcbc2af8538ad069d5f3/numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532", size = 6678797 }, + { url = "https://files.pythonhosted.org/packages/3f/19/bcd641ccf19ac25abb6fb1dcd7744840c11f9d62519d7057b6ab2096eb60/numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e", size = 14067362 }, + { url = "https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe", size = 16116679 }, + { url = "https://files.pythonhosted.org/packages/d0/a1/e90f7aa66512be3150cb9d27f3d9995db330ad1b2046474a13b7040dfd92/numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021", size = 15264272 }, + { url = "https://files.pythonhosted.org/packages/dc/b6/50bd027cca494de4fa1fc7bf1662983d0ba5f256fa0ece2c376b5eb9b3f0/numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8", size = 17880549 }, + { url = "https://files.pythonhosted.org/packages/96/30/f7bf4acb5f8db10a96f73896bdeed7a63373137b131ca18bd3dab889db3b/numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe", size = 6293394 }, + { url = "https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d", size = 12626357 }, +] + [[package]] name = "oauthlib" version = "3.2.2" @@ -685,6 +877,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, ] +[[package]] +name = "pandas" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/a3/fb2734118db0af37ea7433f57f722c0a56687e14b14690edff0cdb4b7e58/pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9", size = 12529893 }, + { url = "https://files.pythonhosted.org/packages/e1/0c/ad295fd74bfac85358fd579e271cded3ac969de81f62dd0142c426b9da91/pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4", size = 11363475 }, + { url = "https://files.pythonhosted.org/packages/c6/2a/4bba3f03f7d07207481fed47f5b35f556c7441acddc368ec43d6643c5777/pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3", size = 15188645 }, + { url = "https://files.pythonhosted.org/packages/38/f8/d8fddee9ed0d0c0f4a2132c1dfcf0e3e53265055da8df952a53e7eaf178c/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319", size = 12739445 }, + { url = "https://files.pythonhosted.org/packages/20/e8/45a05d9c39d2cea61ab175dbe6a2de1d05b679e8de2011da4ee190d7e748/pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8", size = 16359235 }, + { url = "https://files.pythonhosted.org/packages/1d/99/617d07a6a5e429ff90c90da64d428516605a1ec7d7bea494235e1c3882de/pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a", size = 14056756 }, + { url = "https://files.pythonhosted.org/packages/29/d4/1244ab8edf173a10fd601f7e13b9566c1b525c4f365d6bee918e68381889/pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13", size = 11504248 }, +] + [[package]] name = "pastel" version = "0.2.1" @@ -865,7 +1078,7 @@ name = "prefect-daft-analysis" version = "1.0.0" source = { virtual = "." } dependencies = [ - { name = "getdaft" }, + { name = "getdaft", extra = ["aws", "deltalake", "ray"] }, { name = "prefect-client" }, ] @@ -879,7 +1092,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "getdaft", specifier = "==0.4.6" }, + { name = "getdaft", extras = ["aws", "deltalake", "ray"], specifier = "==0.4.6" }, { name = "prefect-client", specifier = "==3.2.11" }, ] @@ -900,10 +1113,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/c2/ab7d37426c179ceb9aeb109a85cda8948bb269b7561a0be870cc656eefe4/prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301", size = 54682 }, ] +[[package]] +name = "protobuf" +version = "6.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/6a/2629bb3529e5bdfbd6c4608ff5c7d942cd4beae85793f84ba543aab2548a/protobuf-6.30.0.tar.gz", hash = "sha256:852b675d276a7d028f660da075af1841c768618f76b90af771a8e2c29e6f5965", size = 429239 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/76/8b1cbf762d98b09fcb29bbc6eca97dc6e1cd865b97a49c443aa23f1a9f82/protobuf-6.30.0-cp310-abi3-win32.whl", hash = "sha256:7337d76d8efe65ee09ee566b47b5914c517190196f414e5418fa236dfd1aed3e", size = 419141 }, + { url = "https://files.pythonhosted.org/packages/57/50/2ea2fb4533321438f5106723c70c303529ba184540e619ebe75e790d402e/protobuf-6.30.0-cp310-abi3-win_amd64.whl", hash = "sha256:9b33d51cc95a7ec4f407004c8b744330b6911a37a782e2629c67e1e8ac41318f", size = 430995 }, + { url = "https://files.pythonhosted.org/packages/a1/7d/a7dfa7aa3deda114920b1ed57c0026e85a976e74658db2784a0443510252/protobuf-6.30.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:52d4bb6fe76005860e1d0b8bfa126f5c97c19cc82704961f60718f50be16942d", size = 417570 }, + { url = "https://files.pythonhosted.org/packages/11/87/a9c7b020c4072dc34e3a2a3cde69366ffc623afff0e7f10f4e5275aaec01/protobuf-6.30.0-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:7940ab4dfd60d514b2e1d3161549ea7aed5be37d53bafde16001ac470a3e202b", size = 317310 }, + { url = "https://files.pythonhosted.org/packages/95/66/424db2262723781dc94208ff9ce201df2f44f18a46fbff3c067812c6b5b9/protobuf-6.30.0-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:d79bf6a202a536b192b7e8d295d7eece0c86fbd9b583d147faf8cfeff46bf598", size = 316203 }, + { url = "https://files.pythonhosted.org/packages/51/6f/21c2b7de96c3051f847a4a88a12fdf015ed6b7d50fc131fb101a739bd7a5/protobuf-6.30.0-py3-none-any.whl", hash = "sha256:e5ef216ea061b262b8994cb6b7d6637a4fb27b3fb4d8e216a6040c0b93bd10d7", size = 167054 }, +] + +[[package]] +name = "pyarrow" +version = "17.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine == 'x86_64' and sys_platform == 'darwin'", +] +dependencies = [ + { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/4e/ea6d43f324169f8aec0e57569443a38bab4b398d09769ca64f7b4d467de3/pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28", size = 1112479 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/62/ce6ac1275a432b4a27c55fe96c58147f111d8ba1ad800a112d31859fae2f/pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22", size = 29019418 }, +] + [[package]] name = "pyarrow" version = "19.0.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "platform_machine != 'x86_64' and sys_platform == 'darwin'", + "sys_platform != 'darwin' and sys_platform != 'win32'", + "sys_platform == 'win32'", +] sdist = { url = "https://files.pythonhosted.org/packages/7f/09/a9046344212690f0632b9c709f9bf18506522feb333c894d0de81d62341a/pyarrow-19.0.1.tar.gz", hash = "sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e", size = 1129437 } wheels = [ { url = "https://files.pythonhosted.org/packages/78/b4/94e828704b050e723f67d67c3535cf7076c7432cd4cf046e4bb3b96a9c9d/pyarrow-19.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b", size = 30670749 }, @@ -1104,6 +1351,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, ] +[[package]] +name = "ray" +version = "2.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiosignal" }, + { name = "click" }, + { name = "filelock" }, + { name = "frozenlist" }, + { name = "jsonschema" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "pyyaml" }, + { name = "requests" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/f2/bafb73554f33bb9a84a3c02e7dee89c94bb06044a06db9d99e44cb2b2d2b/ray-2.43.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:5121fdf4bcbcb0fda3b9b71164dd6c8fcc79a2e258022a2a3957e401018913fb", size = 67858078 }, + { url = "https://files.pythonhosted.org/packages/3a/02/9ba73313664a26072d92cf797cd1d17514b47444d6ffea32aa1d41707b54/ray-2.43.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f26f7b72da04c3c4422269c31b067abd15cb38424b7012d812ddfb2c77462ea", size = 65155290 }, + { url = "https://files.pythonhosted.org/packages/84/f5/e567c655facc09df9a7cbe79eda8dc84465b83ca86b7092cd6c45461ce89/ray-2.43.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:73770d4c8a989730985ff2b4292129249e28c1e29e84589470c9ba1ae91ca832", size = 66871584 }, + { url = "https://files.pythonhosted.org/packages/a1/52/99604614000eb5c12e46a3c423c3078494ce032451ac12d8a573cbffb710/ray-2.43.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:b45f478d29ce5df3fc19861df64fef9ed5c25f1e83fa10028d33fadefdeca095", size = 67783962 }, + { url = "https://files.pythonhosted.org/packages/df/49/5663b8bf55dc85512783cb36fa8696bed552ed52e552c1ab8f410f5f1599/ray-2.43.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c3a9880112a8d561280a34e8ef9471070f81ca467e08b669e5e77a85e173c9c", size = 25565643 }, +] + +[package.optional-dependencies] +client = [ + { name = "grpcio" }, +] +data = [ + { name = "fsspec" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'darwin'" }, + { name = "pyarrow", version = "19.0.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" }, +] + [[package]] name = "readchar" version = "4.2.1" @@ -1253,6 +1536,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/8f/c3654f6f1ddb75daf3922c3d8fc6005b1ab56671ad56ffb874d908bfa668/ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4", size = 115523 }, ] +[[package]] +name = "s3transfer" +version = "0.11.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/ec/aa1a215e5c126fe5decbee2e107468f51d9ce190b9763cb649f76bb45938/s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679", size = 148419 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/62/8d3fc3ec6640161a5649b2cddbbf2b9fa39c92541225b33f117c37c5a2eb/s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d", size = 84412 }, +] + [[package]] name = "shellingham" version = "1.5.4" diff --git a/data-orchestration/hm-prefect/workflows/greet/.prefectignore b/data-orchestration/hm-prefect/workflows/greet/.prefectignore index fdfc280f7c..9a562837d3 100644 --- a/data-orchestration/hm-prefect/workflows/greet/.prefectignore +++ b/data-orchestration/hm-prefect/workflows/greet/.prefectignore @@ -1,4 +1,4 @@ -# https://docs.prefect.io/concepts/deployments/?h=.prefectignore#build-the-deployment +# https://docs.prefect.io/v3/deploy/infrastructure-concepts/store-flow-code#include-or-exclude-files-from-storage # Anywhere **/*.aliases diff --git a/data-orchestration/hm-prefect/workflows/greet/src/deploy.py b/data-orchestration/hm-prefect/workflows/greet/src/deploy.py index f323f57788..96a528d072 100644 --- a/data-orchestration/hm-prefect/workflows/greet/src/deploy.py +++ b/data-orchestration/hm-prefect/workflows/greet/src/deploy.py @@ -8,7 +8,7 @@ from workflow_deploy.utils.create_deployment import create_deployment -async def deploy() -> None: +async def hm_greet() -> None: args = get_args() match args.environment: @@ -20,7 +20,7 @@ async def deploy() -> None: logging.error("Not supported environment.") return - docker_image_name = f"ghcr.io/hongbo-miao/hm-prefect-{config.BASE_WORKFLOW_NAME}" + docker_image_name = f"harbor.hongbomiao.com/hm/prefect-{config.BASE_WORKFLOW_NAME}" await create_deployment( args.environment, greet, @@ -31,4 +31,4 @@ async def deploy() -> None: if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - asyncio.run(deploy()) + asyncio.run(hm_greet()) diff --git a/data-orchestration/hm-prefect/workflows/greet/src/main.py b/data-orchestration/hm-prefect/workflows/greet/src/main.py index 83cbb94e51..770586d247 100644 --- a/data-orchestration/hm-prefect/workflows/greet/src/main.py +++ b/data-orchestration/hm-prefect/workflows/greet/src/main.py @@ -24,7 +24,7 @@ def create_farewell_subflow(age: int) -> str: @flow -def greet(user: User) -> None: +def hm_greet(user: User) -> None: greeting = create_greeting_subflow(user.name) farewell = create_farewell_subflow(user.age) logger = get_run_logger() @@ -33,4 +33,4 @@ def greet(user: User) -> None: if __name__ == "__main__": external_user = User(name="Rose", age=20) - greet(external_user) + hm_greet(external_user) diff --git a/data-orchestration/hm-prefect/workflows/print-platform/.prefectignore b/data-orchestration/hm-prefect/workflows/print-platform/.prefectignore index fdfc280f7c..9a562837d3 100644 --- a/data-orchestration/hm-prefect/workflows/print-platform/.prefectignore +++ b/data-orchestration/hm-prefect/workflows/print-platform/.prefectignore @@ -1,4 +1,4 @@ -# https://docs.prefect.io/concepts/deployments/?h=.prefectignore#build-the-deployment +# https://docs.prefect.io/v3/deploy/infrastructure-concepts/store-flow-code#include-or-exclude-files-from-storage # Anywhere **/*.aliases diff --git a/data-orchestration/hm-prefect/workflows/print-platform/src/deploy.py b/data-orchestration/hm-prefect/workflows/print-platform/src/deploy.py index 561b9942eb..b46a70d3d7 100644 --- a/data-orchestration/hm-prefect/workflows/print-platform/src/deploy.py +++ b/data-orchestration/hm-prefect/workflows/print-platform/src/deploy.py @@ -1,7 +1,7 @@ import asyncio import logging -from main import print_platform +from main import hm_print_platform from workflow_deploy import config from workflow_deploy.args import get_args from workflow_deploy.environments import Environments @@ -20,10 +20,10 @@ async def deploy() -> None: logging.error("Not supported environment.") return - docker_image_name = f"ghcr.io/hongbo-miao/hm-prefect-{config.BASE_WORKFLOW_NAME}" + docker_image_name = f"harbor.hongbomiao.com/hm/prefect-{config.BASE_WORKFLOW_NAME}" await create_deployment( args.environment, - print_platform, + hm_print_platform, docker_image_name, deployment, ) diff --git a/data-orchestration/hm-prefect/workflows/print-platform/src/main.py b/data-orchestration/hm-prefect/workflows/print-platform/src/main.py index dfebd8f6d5..7fd7709278 100644 --- a/data-orchestration/hm-prefect/workflows/print-platform/src/main.py +++ b/data-orchestration/hm-prefect/workflows/print-platform/src/main.py @@ -4,7 +4,7 @@ @flow -def print_platform() -> None: +def hm_print_platform() -> None: logger = get_run_logger() logger.info(f"Node: {platform.node()}") logger.info(f"Platform: {platform.platform()}") @@ -12,4 +12,4 @@ def print_platform() -> None: if __name__ == "__main__": - print_platform() + hm_print_platform() diff --git a/data-storage/delta-lake/read-delta-lake-by-trino-matlab/main.m b/data-storage/delta-lake/read-delta-lake-by-trino-matlab/main.m index 290916471c..bff42f5d7c 100644 --- a/data-storage/delta-lake/read-delta-lake-by-trino-matlab/main.m +++ b/data-storage/delta-lake/read-delta-lake-by-trino-matlab/main.m @@ -6,22 +6,22 @@ columnNames = strjoin(fieldNames, ', '); sqlQuery = [ 'with ', ... - 't0 as (select * from delta.production_hm_delta_db.motor_data_0 where _event_id = ''', eventId, '''), ', ... - 't1 as (select * from delta.production_hm_delta_db.motor_data_1 where _event_id = ''', eventId, '''), ', ... - 't2 as (select * from delta.production_hm_delta_db.motor_data_2 where _event_id = ''', eventId, '''), ', ... - 't3 as (select * from delta.production_hm_delta_db.motor_data_3 where _event_id = ''', eventId, '''), ', ... - 't4 as (select * from delta.production_hm_delta_db.motor_data_4 where _event_id = ''', eventId, '''), ', ... - 't5 as (select * from delta.production_hm_delta_db.motor_data_5 where _event_id = ''', eventId, '''), ', ... - 't6 as (select * from delta.production_hm_delta_db.motor_data_6 where _event_id = ''', eventId, '''), ', ... - 't7 as (select * from delta.production_hm_delta_db.motor_data_7 where _event_id = ''', eventId, '''), ', ... - 't8 as (select * from delta.production_hm_delta_db.motor_data_8 where _event_id = ''', eventId, '''), ', ... - 't9 as (select * from delta.production_hm_delta_db.motor_data_9 where _event_id = ''', eventId, '''), ', ... - 'ta as (select * from delta.production_hm_delta_db.motor_data_a where _event_id = ''', eventId, '''), ', ... - 'tb as (select * from delta.production_hm_delta_db.motor_data_b where _event_id = ''', eventId, '''), ', ... - 'tc as (select * from delta.production_hm_delta_db.motor_data_c where _event_id = ''', eventId, '''), ', ... - 'td as (select * from delta.production_hm_delta_db.motor_data_d where _event_id = ''', eventId, '''), ', ... - 'te as (select * from delta.production_hm_delta_db.motor_data_e where _event_id = ''', eventId, '''), ', ... - 'tf as (select * from delta.production_hm_delta_db.motor_data_f where _event_id = ''', eventId, ''') ', ... + 't0 as (select * from delta.production_motor_db.motor_data_0 where _event_id = ''', eventId, '''), ', ... + 't1 as (select * from delta.production_motor_db.motor_data_1 where _event_id = ''', eventId, '''), ', ... + 't2 as (select * from delta.production_motor_db.motor_data_2 where _event_id = ''', eventId, '''), ', ... + 't3 as (select * from delta.production_motor_db.motor_data_3 where _event_id = ''', eventId, '''), ', ... + 't4 as (select * from delta.production_motor_db.motor_data_4 where _event_id = ''', eventId, '''), ', ... + 't5 as (select * from delta.production_motor_db.motor_data_5 where _event_id = ''', eventId, '''), ', ... + 't6 as (select * from delta.production_motor_db.motor_data_6 where _event_id = ''', eventId, '''), ', ... + 't7 as (select * from delta.production_motor_db.motor_data_7 where _event_id = ''', eventId, '''), ', ... + 't8 as (select * from delta.production_motor_db.motor_data_8 where _event_id = ''', eventId, '''), ', ... + 't9 as (select * from delta.production_motor_db.motor_data_9 where _event_id = ''', eventId, '''), ', ... + 'ta as (select * from delta.production_motor_db.motor_data_a where _event_id = ''', eventId, '''), ', ... + 'tb as (select * from delta.production_motor_db.motor_data_b where _event_id = ''', eventId, '''), ', ... + 'tc as (select * from delta.production_motor_db.motor_data_c where _event_id = ''', eventId, '''), ', ... + 'td as (select * from delta.production_motor_db.motor_data_d where _event_id = ''', eventId, '''), ', ... + 'te as (select * from delta.production_motor_db.motor_data_e where _event_id = ''', eventId, '''), ', ... + 'tf as (select * from delta.production_motor_db.motor_data_f where _event_id = ''', eventId, ''') ', ... 'select from_unixtime_nanos(t0._time) as _time, ', columnNames, ' ', ... 'from t0 ', ... 'join t1 on t0._time = t1._time ', ... diff --git a/data-storage/delta-lake/read-delta-lake-by-trino/src/main.py b/data-storage/delta-lake/read-delta-lake-by-trino/src/main.py index b7e8b66e2a..9c7e31e723 100644 --- a/data-storage/delta-lake/read-delta-lake-by-trino/src/main.py +++ b/data-storage/delta-lake/read-delta-lake-by-trino/src/main.py @@ -21,22 +21,22 @@ def main() -> None: sql_query = text( f""" with - t0 as (select * from delta.production_hm_delta_db.motor_data_0 where _event_id = :event_id), - t1 as (select * from delta.production_hm_delta_db.motor_data_1 where _event_id = :event_id), - t2 as (select * from delta.production_hm_delta_db.motor_data_2 where _event_id = :event_id), - t3 as (select * from delta.production_hm_delta_db.motor_data_3 where _event_id = :event_id), - t4 as (select * from delta.production_hm_delta_db.motor_data_4 where _event_id = :event_id), - t5 as (select * from delta.production_hm_delta_db.motor_data_5 where _event_id = :event_id), - t6 as (select * from delta.production_hm_delta_db.motor_data_6 where _event_id = :event_id), - t7 as (select * from delta.production_hm_delta_db.motor_data_7 where _event_id = :event_id), - t8 as (select * from delta.production_hm_delta_db.motor_data_8 where _event_id = :event_id), - t9 as (select * from delta.production_hm_delta_db.motor_data_9 where _event_id = :event_id), - ta as (select * from delta.production_hm_delta_db.motor_data_a where _event_id = :event_id), - tb as (select * from delta.production_hm_delta_db.motor_data_b where _event_id = :event_id), - tc as (select * from delta.production_hm_delta_db.motor_data_c where _event_id = :event_id), - td as (select * from delta.production_hm_delta_db.motor_data_d where _event_id = :event_id), - te as (select * from delta.production_hm_delta_db.motor_data_e where _event_id = :event_id), - tf as (select * from delta.production_hm_delta_db.motor_data_f where _event_id = :event_id) + t0 as (select * from delta.production_motor_db.motor_data_0 where _event_id = :event_id), + t1 as (select * from delta.production_motor_db.motor_data_1 where _event_id = :event_id), + t2 as (select * from delta.production_motor_db.motor_data_2 where _event_id = :event_id), + t3 as (select * from delta.production_motor_db.motor_data_3 where _event_id = :event_id), + t4 as (select * from delta.production_motor_db.motor_data_4 where _event_id = :event_id), + t5 as (select * from delta.production_motor_db.motor_data_5 where _event_id = :event_id), + t6 as (select * from delta.production_motor_db.motor_data_6 where _event_id = :event_id), + t7 as (select * from delta.production_motor_db.motor_data_7 where _event_id = :event_id), + t8 as (select * from delta.production_motor_db.motor_data_8 where _event_id = :event_id), + t9 as (select * from delta.production_motor_db.motor_data_9 where _event_id = :event_id), + ta as (select * from delta.production_motor_db.motor_data_a where _event_id = :event_id), + tb as (select * from delta.production_motor_db.motor_data_b where _event_id = :event_id), + tc as (select * from delta.production_motor_db.motor_data_c where _event_id = :event_id), + td as (select * from delta.production_motor_db.motor_data_d where _event_id = :event_id), + te as (select * from delta.production_motor_db.motor_data_e where _event_id = :event_id), + tf as (select * from delta.production_motor_db.motor_data_f where _event_id = :event_id) select from_unixtime_nanos(t0._time) as _time, {column_names} from t0 join t1 on t0._time = t1._time diff --git a/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml b/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml index d85ce02091..f76fd24cf8 100644 --- a/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml +++ b/kubernetes/argo-cd/applications/production-hm/prefect-worker/argo-cd-manifests/hm-prefect-worker-application.yaml @@ -74,7 +74,7 @@ spec: ], "parallelism": 1, "restartPolicy": "Never", - "serviceAccountName": "{{ service_account_name }}" + "serviceAccountName": "hm-prefect-worker" } }, "ttlSecondsAfterFinished": "{{ finished_job_ttl }}" diff --git a/trino/queries/delta_lake_motor_data.sql b/trino/queries/delta_lake_motor_data.sql index d55b7b943a..9119835df1 100644 --- a/trino/queries/delta_lake_motor_data.sql +++ b/trino/queries/delta_lake_motor_data.sql @@ -7,22 +7,22 @@ from delta.production_hm_iot_db.motor limit 100; -- Get full data with -t0 as (select * from delta.production_hm_delta_db.motor_data_0 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t1 as (select * from delta.production_hm_delta_db.motor_data_1 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t2 as (select * from delta.production_hm_delta_db.motor_data_2 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t3 as (select * from delta.production_hm_delta_db.motor_data_3 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t4 as (select * from delta.production_hm_delta_db.motor_data_4 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t5 as (select * from delta.production_hm_delta_db.motor_data_5 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t6 as (select * from delta.production_hm_delta_db.motor_data_6 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t7 as (select * from delta.production_hm_delta_db.motor_data_7 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t8 as (select * from delta.production_hm_delta_db.motor_data_8 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t9 as (select * from delta.production_hm_delta_db.motor_data_9 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -ta as (select * from delta.production_hm_delta_db.motor_data_a where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -tb as (select * from delta.production_hm_delta_db.motor_data_b where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -tc as (select * from delta.production_hm_delta_db.motor_data_c where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -td as (select * from delta.production_hm_delta_db.motor_data_d where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -te as (select * from delta.production_hm_delta_db.motor_data_e where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -tf as (select * from delta.production_hm_delta_db.motor_data_f where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255') +t0 as (select * from delta.production_motor_db.motor_data_0 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t1 as (select * from delta.production_motor_db.motor_data_1 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t2 as (select * from delta.production_motor_db.motor_data_2 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t3 as (select * from delta.production_motor_db.motor_data_3 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t4 as (select * from delta.production_motor_db.motor_data_4 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t5 as (select * from delta.production_motor_db.motor_data_5 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t6 as (select * from delta.production_motor_db.motor_data_6 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t7 as (select * from delta.production_motor_db.motor_data_7 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t8 as (select * from delta.production_motor_db.motor_data_8 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t9 as (select * from delta.production_motor_db.motor_data_9 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +ta as (select * from delta.production_motor_db.motor_data_a where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +tb as (select * from delta.production_motor_db.motor_data_b where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +tc as (select * from delta.production_motor_db.motor_data_c where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +td as (select * from delta.production_motor_db.motor_data_d where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +te as (select * from delta.production_motor_db.motor_data_e where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +tf as (select * from delta.production_motor_db.motor_data_f where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255') select from_unixtime_nanos(_time) as _time, current, voltage, temperature from t0 join t1 on t0._time = t1._time @@ -46,25 +46,25 @@ order by t_0._time asc; with t_sec as ( select max(_time) as _time - from delta.production_hm_delta_db.motor_data_0 + from delta.production_motor_db.motor_data_0 group by date_trunc('second', from_unixtime_nanos(_time)) ), -t0 as (select * from delta.production_hm_delta_db.motor_data_0 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t1 as (select * from delta.production_hm_delta_db.motor_data_1 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t2 as (select * from delta.production_hm_delta_db.motor_data_2 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t3 as (select * from delta.production_hm_delta_db.motor_data_3 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t4 as (select * from delta.production_hm_delta_db.motor_data_4 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t5 as (select * from delta.production_hm_delta_db.motor_data_5 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t6 as (select * from delta.production_hm_delta_db.motor_data_6 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t7 as (select * from delta.production_hm_delta_db.motor_data_7 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t8 as (select * from delta.production_hm_delta_db.motor_data_8 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -t9 as (select * from delta.production_hm_delta_db.motor_data_9 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -ta as (select * from delta.production_hm_delta_db.motor_data_a where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -tb as (select * from delta.production_hm_delta_db.motor_data_b where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -tc as (select * from delta.production_hm_delta_db.motor_data_c where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -td as (select * from delta.production_hm_delta_db.motor_data_d where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -te as (select * from delta.production_hm_delta_db.motor_data_e where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), -tf as (select * from delta.production_hm_delta_db.motor_data_f where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255') +t0 as (select * from delta.production_motor_db.motor_data_0 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t1 as (select * from delta.production_motor_db.motor_data_1 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t2 as (select * from delta.production_motor_db.motor_data_2 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t3 as (select * from delta.production_motor_db.motor_data_3 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t4 as (select * from delta.production_motor_db.motor_data_4 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t5 as (select * from delta.production_motor_db.motor_data_5 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t6 as (select * from delta.production_motor_db.motor_data_6 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t7 as (select * from delta.production_motor_db.motor_data_7 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t8 as (select * from delta.production_motor_db.motor_data_8 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +t9 as (select * from delta.production_motor_db.motor_data_9 where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +ta as (select * from delta.production_motor_db.motor_data_a where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +tb as (select * from delta.production_motor_db.motor_data_b where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +tc as (select * from delta.production_motor_db.motor_data_c where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +td as (select * from delta.production_motor_db.motor_data_d where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +te as (select * from delta.production_motor_db.motor_data_e where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255'), +tf as (select * from delta.production_motor_db.motor_data_f where _event_id = 'ad7953cd-6d49-4929-8180-99555bebc255') select from_unixtime_nanos(_time) as _time, current, voltage, temperature from t_sec join t0 on t_sec._time = t0._time