From bf5607b4da16aaf254f8169ca9d4ebd61f22c071 Mon Sep 17 00:00:00 2001 From: Prem Shankar Kumar Date: Fri, 24 Jan 2025 09:58:55 -0800 Subject: [PATCH 1/5] soopervisor --skip-docker should use the configured repository when running with AWS Batch Fixed https://github.com/ploomber/soopervisor/issues/130 Tested in AWS Batch, earlier it was submitting task using invalid image. Now, it's able to re-use previously created docker image. ``` $aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin xxx.dkr.ecr.us-west-2.amazonaws.com/prod-repo && soopervisor export deploy --skip-tests --ignore-git --skip-docker ``` --- src/soopervisor/aws/batch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/soopervisor/aws/batch.py b/src/soopervisor/aws/batch.py index 4c4a4d2..d89e144 100644 --- a/src/soopervisor/aws/batch.py +++ b/src/soopervisor/aws/batch.py @@ -283,7 +283,8 @@ def _export( pkg_name, version = source.find_package_name_and_version() default_image_key = get_default_image_key() if default_image_key: - image_local = f"{pkg_name}:{version}-" + #image_local = f"{pkg_name}:{version}-" + image_local = f"{cfg.repository}:{version}" # use docker image previously created from ECR repository specified in the config file. f"{docker.modify_wildcard(default_image_key)}" image_map = {} image_map[default_image_key] = image_local From f28b581bf14b7327cf43da26a0c1fe183de5e989 Mon Sep 17 00:00:00 2001 From: Prem Shankar Kumar Date: Mon, 27 Jan 2025 09:14:11 -0800 Subject: [PATCH 2/5] skip-docker should use the configured repository when running with AWS Batch --- src/soopervisor/aws/batch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/soopervisor/aws/batch.py b/src/soopervisor/aws/batch.py index d89e144..6831346 100644 --- a/src/soopervisor/aws/batch.py +++ b/src/soopervisor/aws/batch.py @@ -283,8 +283,8 @@ def _export( pkg_name, version = source.find_package_name_and_version() default_image_key = get_default_image_key() if default_image_key: - #image_local = f"{pkg_name}:{version}-" - image_local = f"{cfg.repository}:{version}" # use docker image previously created from ECR repository specified in the config file. + # Use image from ECR repository from the config file. + image_local = f"{cfg.repository}:{version}" f"{docker.modify_wildcard(default_image_key)}" image_map = {} image_map[default_image_key] = image_local From ae57755d3a1d55b8d87c100d77091a3e9129a572 Mon Sep 17 00:00:00 2001 From: Prem Kumar Date: Sun, 16 Feb 2025 18:30:31 -0800 Subject: [PATCH 3/5] Fix build --- setup.py | 3 ++- src/soopervisor/assets/airflow/kubernetes.py | 2 +- tests/airflow/test_airflow_export.py | 2 +- tutorials/airflow/ml-intermediate.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 2bec3bc..dcf804e 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(*names, **kwargs): "tqdm", "pydantic", "Jinja2", - "pyyaml", + "PyYAML==6.0", "ploomber>=0.14.6", "ploomber-core>=0.0.11", # sdist is generated using python -m build, so adding this here. @@ -60,6 +60,7 @@ def read(*names, **kwargs): "kfp", # to validate argo specs "argo-workflows-dsl", + "cython", # for testing aws (newer versions break) # see: https://github.com/spulec/moto/issues/1793 "moto==1.3.14", diff --git a/src/soopervisor/assets/airflow/kubernetes.py b/src/soopervisor/assets/airflow/kubernetes.py index 83b4dee..2912085 100644 --- a/src/soopervisor/assets/airflow/kubernetes.py +++ b/src/soopervisor/assets/airflow/kubernetes.py @@ -3,7 +3,7 @@ from airflow import DAG from airflow.utils.dates import days_ago -from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import ( +from airflow.providers.cncf.kubernetes.operators.pod import ( KubernetesPodOperator, ) diff --git a/tests/airflow/test_airflow_export.py b/tests/airflow/test_airflow_export.py index d01a5de..f0355c6 100644 --- a/tests/airflow/test_airflow_export.py +++ b/tests/airflow/test_airflow_export.py @@ -6,7 +6,7 @@ import json from airflow import DAG -from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import ( +from airflow.providers.cncf.kubernetes.operators.pod import ( KubernetesPodOperator, ) from airflow.operators.bash import BashOperator diff --git a/tutorials/airflow/ml-intermediate.py b/tutorials/airflow/ml-intermediate.py index 5bc96c8..0eecbd6 100644 --- a/tutorials/airflow/ml-intermediate.py +++ b/tutorials/airflow/ml-intermediate.py @@ -5,7 +5,7 @@ from airflow import DAG from airflow.utils.dates import days_ago -from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import ( +from airflow.providers.cncf.kubernetes.operators.pod import ( KubernetesPodOperator, ) From dbe16df25a41ea0a8dce716fa0276b4b11cd8d5b Mon Sep 17 00:00:00 2001 From: prem Date: Mon, 17 Feb 2025 14:52:15 -0800 Subject: [PATCH 4/5] Update python 3.8 to 3.9 --- tests/assets/callables/environment.lock.yml | 2 +- tests/assets/my_project/environment.lock.yml | 2 +- tests/assets/my_project/environment.yml | 2 +- tests/assets/sample_project/environment.lock.yml | 2 +- tests/assets/sample_project/environment.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/assets/callables/environment.lock.yml b/tests/assets/callables/environment.lock.yml index 4e96554..dcc143e 100644 --- a/tests/assets/callables/environment.lock.yml +++ b/tests/assets/callables/environment.lock.yml @@ -4,7 +4,7 @@ channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - pip - pip: diff --git a/tests/assets/my_project/environment.lock.yml b/tests/assets/my_project/environment.lock.yml index 4bfc2f9..a64f087 100644 --- a/tests/assets/my_project/environment.lock.yml +++ b/tests/assets/my_project/environment.lock.yml @@ -4,5 +4,5 @@ channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - pip diff --git a/tests/assets/my_project/environment.yml b/tests/assets/my_project/environment.yml index 4bfc2f9..a64f087 100644 --- a/tests/assets/my_project/environment.yml +++ b/tests/assets/my_project/environment.yml @@ -4,5 +4,5 @@ channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - pip diff --git a/tests/assets/sample_project/environment.lock.yml b/tests/assets/sample_project/environment.lock.yml index c1acfc4..ad048db 100644 --- a/tests/assets/sample_project/environment.lock.yml +++ b/tests/assets/sample_project/environment.lock.yml @@ -2,7 +2,7 @@ name: sample-project-env dependencies: # change the version here if you need to - - python=3.8 + - python=3.9 # add dependencies here - numpy - pandas diff --git a/tests/assets/sample_project/environment.yml b/tests/assets/sample_project/environment.yml index c1acfc4..ad048db 100644 --- a/tests/assets/sample_project/environment.yml +++ b/tests/assets/sample_project/environment.yml @@ -2,7 +2,7 @@ name: sample-project-env dependencies: # change the version here if you need to - - python=3.8 + - python=3.9 # add dependencies here - numpy - pandas From f8e54e24b9983977586a0aa6fec4b789e4df8ef6 Mon Sep 17 00:00:00 2001 From: prem Date: Sat, 22 Feb 2025 14:01:04 -0800 Subject: [PATCH 5/5] Test build on all three platforms after removing latest PyYAML and cython --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index dcf804e..2bec3bc 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(*names, **kwargs): "tqdm", "pydantic", "Jinja2", - "PyYAML==6.0", + "pyyaml", "ploomber>=0.14.6", "ploomber-core>=0.0.11", # sdist is generated using python -m build, so adding this here. @@ -60,7 +60,6 @@ def read(*names, **kwargs): "kfp", # to validate argo specs "argo-workflows-dsl", - "cython", # for testing aws (newer versions break) # see: https://github.com/spulec/moto/issues/1793 "moto==1.3.14",