From 6ab314991701f9b1c377797455f5cc678ecca3a8 Mon Sep 17 00:00:00 2001 From: Sam Gamble Date: Mon, 21 Oct 2024 15:30:41 +0100 Subject: [PATCH 1/3] fix pkg update update images to 24.04 fix package update script regen python packages for py3.12 break all the packages fix user create ffs Fix user ids and remove ubuntu user Fix branch selection on image tests Switch actions to py3.12 Fix py12 test failures try pinned trivy version Revert "try pinned trivy version" This reverts commit c87bc80276eb62793464d2c9d3882a99254ccef0. Load tivy db from cache --- .github/workflows/build-images.yml | 2 ++ .github/workflows/build-schema.yml | 4 ++-- .github/workflows/code-quality.yml | 2 +- .github/workflows/scan-external.yml | 2 ++ .github/workflows/scan.yml | 3 +++ .github/workflows/test-images.yml | 7 +++---- .github/workflows/test-python_api.yml | 4 ++-- .github/workflows/update-python.yml | 4 ++-- .github/workflows/version.yml | 4 ++-- Dockerfile.api_server | 12 ++++++++---- Dockerfile.model_worker | 9 ++++++--- kubernetes/worker-controller/requirements.txt | 4 +--- requirements-worker.in | 6 +++--- scripts/update-packages.sh | 4 ++-- .../oasisapi/permissions/tests/test_group_auth.py | 2 +- 15 files changed, 40 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e4bd2a285..e9874aac1 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -72,6 +72,8 @@ env: IGNORE_UNFIXED: 'true' SEVERITY: 'CRITICAL,HIGH' IMAGE_REPO: 'coreoasis/github-actions' + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true jobs: build: diff --git a/.github/workflows/build-schema.yml b/.github/workflows/build-schema.yml index 54a785b1c..fc98c6d0d 100644 --- a/.github/workflows/build-schema.yml +++ b/.github/workflows/build-schema.yml @@ -39,10 +39,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install requirments run: pip install -r requirements-server.txt diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 2f39f0125..27d7b7590 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: install pip tools run: | diff --git a/.github/workflows/scan-external.yml b/.github/workflows/scan-external.yml index c7bd588d0..7b3679f2c 100644 --- a/.github/workflows/scan-external.yml +++ b/.github/workflows/scan-external.yml @@ -15,6 +15,8 @@ on: env: SEVERITY: 'HIGH,CRITICAL' IGNORE_UNFIXED: 'true' + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true jobs: scan_external_images: diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index 7bfff319f..343bd3e48 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -45,6 +45,9 @@ jobs: REPORT: 'repo-results.sarif' IGNORE_UNFIXED: 'true' EXIT_CODE: '1' + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + runs-on: ubuntu-latest steps: - name: Adding code-scanning URL diff --git a/.github/workflows/test-images.yml b/.github/workflows/test-images.yml index f7a056a16..2ad4e5ef3 100644 --- a/.github/workflows/test-images.yml +++ b/.github/workflows/test-images.yml @@ -112,16 +112,15 @@ jobs: # Select matching base branch on piwind if [[ "${{ github.event_name }}" = "pull_request" ]]; then BRANCH=${{ github.base_ref }} + if [[ ! $BRANCH == stable/* ]]; then + BRANCH='main' + fi elif [[ "${{ github.event_name }}" = "push" ]]; then BRANCH=${{ github.ref_name }} else BRANCH=${{ inputs.piwind_branch }} fi - #override 'main-platform1' -> 'main' - if [[ "$BRANCH" = 'main-platform1' ]]; then - BRANCH=main - fi echo "branch=$BRANCH" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-python_api.yml b/.github/workflows/test-python_api.yml index ccac376c8..6bf2b823c 100644 --- a/.github/workflows/test-python_api.yml +++ b/.github/workflows/test-python_api.yml @@ -51,10 +51,10 @@ jobs: repository: OasisLMF/OasisPlatform ref: ${{ env.PLAT_BRANCH }} - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - run: pip install -r requirements.txt - name: Download package diff --git a/.github/workflows/update-python.yml b/.github/workflows/update-python.yml index fa6dd3af9..2c82d9504 100644 --- a/.github/workflows/update-python.yml +++ b/.github/workflows/update-python.yml @@ -32,10 +32,10 @@ jobs: GIT_EMAIL: ${{ secrets.BUILD_GIT_EMAIL }} GIT_USERNAME: ${{ secrets.BUILD_GIT_USERNAME }} - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install piptools run: pip install pip-tools setuptools-rust diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 4d9c683e6..4c634c9f1 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -77,11 +77,11 @@ jobs: echo ${{ inputs.platform_version }} > VERSION git add VERSION - - name: Set up Python 3.10 + - name: Set up Python 3.12 if: inputs.oasislmf_version != '' uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install piptools run: pip install pip-tools diff --git a/Dockerfile.api_server b/Dockerfile.api_server index b66cf6601..af5e6c764 100755 --- a/Dockerfile.api_server +++ b/Dockerfile.api_server @@ -1,9 +1,10 @@ # ---- STAGE 1 ----- -FROM ubuntu:22.04 AS build-packages +FROM ubuntu:24.04 AS build-packages +ENV PIP_BREAK_SYSTEM_PACKAGES 1 ENV DEBIAN_FRONTEND noninteractive COPY ./requirements-server.txt ./requirements-server.txt -RUN apt-get update && apt-get install -y --no-install-recommends gcc build-essential python3 python3-pip python3-dev libmariadbclient-dev-compat && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends gcc build-essential python3 python3-pip python3-dev pkg-config libmariadb-dev-compat && rm -rf /var/lib/apt/lists/* RUN pip install --user --no-warn-script-location -r ./requirements-server.txt && pip install --no-warn-script-location --user mysqlclient # Install ODS-Tools from git branch (Optional) 'docker build --build-arg ods_tools_branch=develop' @@ -25,13 +26,16 @@ RUN if [ ! -z "$odm_branch" ] ; then \ USER server # ---- STAGE 2 ---- -FROM ubuntu:22.04 +FROM ubuntu:24.04 RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends sudo python3 python3-pkg-resources curl libmariadbclient-dev-compat \ && rm -rf /var/lib/apt/lists/ -RUN adduser --home /home/server --shell /bin/bash --disabled-password --gecos "" server +ARG USER_ID=1000 +RUN userdel -r ubuntu +RUN useradd -u ${USER_ID} --home /home/server --shell /bin/bash server + COPY --chown=server:server --from=build-packages /root/.local /home/server/.local RUN mkdir -p /var/log/oasis /shared-fs && chmod 777 -R /var/log/oasis diff --git a/Dockerfile.model_worker b/Dockerfile.model_worker index 1c54b0063..108cc255f 100755 --- a/Dockerfile.model_worker +++ b/Dockerfile.model_worker @@ -1,7 +1,8 @@ # ---- STAGE 1 ----- -FROM ubuntu:22.04 AS build-packages +FROM ubuntu:24.04 AS build-packages # Build python packages +ENV PIP_BREAK_SYSTEM_PACKAGES 1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y libspatialindex-dev git curl g++ build-essential libtool autoconf automake python3-dev python3 python3-pip pkg-config COPY ./requirements-worker.txt ./requirements-worker.txt @@ -35,14 +36,16 @@ RUN if [ ! -z "$odm_branch" ] ; then \ USER worker # ---- STAGE 2 ---- -FROM ubuntu:22.04 +FROM ubuntu:24.04 RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends vim git python3 python3-pip libspatialindex-dev curl procps \ && rm -rf /var/lib/apt/lists/* # Copy built python packages -RUN adduser --home /home/worker --shell /bin/bash --disabled-password --gecos "" worker +ARG USER_ID=1000 +RUN userdel -r ubuntu +RUN useradd -u ${USER_ID} --home /home/worker --shell /bin/bash worker COPY --chown=worker:worker --from=build-packages /root/.local /home/worker/.local # Copy in worker files diff --git a/kubernetes/worker-controller/requirements.txt b/kubernetes/worker-controller/requirements.txt index 8e2a1cf82..304595e94 100644 --- a/kubernetes/worker-controller/requirements.txt +++ b/kubernetes/worker-controller/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile kubernetes/worker-controller/requirements.in @@ -10,8 +10,6 @@ aiohttp==3.9.4 # kubernetes-asyncio aiosignal==1.3.1 # via aiohttp -async-timeout==4.0.3 - # via aiohttp attrs==22.1.0 # via aiohttp certifi==2023.7.22 diff --git a/requirements-worker.in b/requirements-worker.in index 023cc9cdf..af8f8de1d 100644 --- a/requirements-worker.in +++ b/requirements-worker.in @@ -2,15 +2,15 @@ oasislmf[extra] oasis-data-manager adlfs s3fs -azure-core>=1.21.1 -azure-storage-blob>=12.9.0 +azure-core +azure-storage-blob billiard boto3 celery configparser fasteners filelock -joblib>=1.2.0 +joblib pathlib2 psycopg2-binary pymysql diff --git a/scripts/update-packages.sh b/scripts/update-packages.sh index 2fec8f422..329382942 100755 --- a/scripts/update-packages.sh +++ b/scripts/update-packages.sh @@ -2,9 +2,9 @@ pkg_list=( 'ods-tools==3.*' - 'oasislmf==1.27.*' - 'pandas==1.*' + 'oasislmf==2.*' 'django==3.*' + 'djangorestframework==3.14.*' 'celery==5.*' ) diff --git a/src/server/oasisapi/permissions/tests/test_group_auth.py b/src/server/oasisapi/permissions/tests/test_group_auth.py index bec615362..b52bdeb72 100644 --- a/src/server/oasisapi/permissions/tests/test_group_auth.py +++ b/src/server/oasisapi/permissions/tests/test_group_auth.py @@ -27,7 +27,7 @@ def test_verify_and_get_groups__no_or_empty(self): user_no_group = create_user([]) self.assertEqual([], verify_and_get_groups(user_no_group, None)) - self.assertEquals([], verify_and_get_groups(user_no_group, [])) + self.assertEqual([], verify_and_get_groups(user_no_group, [])) def test_verify_and_get_groups__invalid_groups(self): From f4f88f2357580ff6065406b5031fe25acc975234 Mon Sep 17 00:00:00 2001 From: Sam Gamble Date: Mon, 28 Oct 2024 10:01:57 +0000 Subject: [PATCH 2/3] update python packages --- requirements-server.txt | 149 ++++++++++---------- requirements-worker.txt | 176 ++++++++++++------------ requirements.txt | 294 +++++++++++++++++++--------------------- 3 files changed, 291 insertions(+), 328 deletions(-) diff --git a/requirements-server.txt b/requirements-server.txt index 4200697ba..570a533dd 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements-server.in @@ -8,48 +8,45 @@ aioredis==1.3.1 # via channels-redis amqp==5.2.0 # via kombu -asgiref==3.7.2 +asgiref==3.8.1 # via # channels # channels-redis # daphne # django async-timeout==4.0.3 + # via aioredis +attrs==24.2.0 # via - # aioredis - # redis -attrs==23.2.0 - # via - # automat # jsonschema # referencing # service-identity # twisted -autobahn==23.6.2 +autobahn==24.4.2 # via daphne -automat==22.10.0 +automat==24.8.1 # via twisted -azure-core==1.30.1 +azure-core==1.31.0 # via # azure-storage-blob # django-storages -azure-storage-blob==12.19.1 +azure-storage-blob==12.23.1 # via django-storages -billiard==4.2.0 +billiard==4.2.1 # via celery -boto3==1.34.56 +boto3==1.35.49 # via -r requirements-server.in -botocore==1.34.56 +botocore==1.35.49 # via # boto3 # s3transfer -celery==5.3.6 +celery==5.4.0 # via # -r requirements-server.in # django-celery-results -certifi==2024.2.2 +certifi==2024.8.30 # via requests -cffi==1.16.0 +cffi==1.17.1 # via cryptography chainmap==1.0.3 # via -r requirements-server.in @@ -61,7 +58,7 @@ channels-redis==2.4.2 # via -r requirements-server.in chardet==5.2.0 # via ods-tools -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -69,7 +66,7 @@ click==8.1.7 # click-didyoumean # click-plugins # click-repl -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery @@ -81,9 +78,9 @@ coreapi==2.3.3 # via -r requirements-server.in coreschema==0.0.4 # via coreapi -cramjam==2.8.2 +cramjam==2.9.0 # via fastparquet -cryptography==42.0.5 +cryptography==43.0.3 # via # autobahn # azure-storage-blob @@ -109,17 +106,17 @@ django==3.2.25 # mozilla-django-oidc django-celery-results==2.5.1 # via -r requirements-server.in -django-cleanup==8.1.0 +django-cleanup==9.0.0 # via -r requirements-server.in django-debug-toolbar==4.3.0 # via -r requirements-server.in django-filter==23.5 # via -r requirements-server.in -django-model-utils==4.4.0 +django-model-utils==5.0.0 # via -r requirements-server.in django-request-logging==0.7.5 # via -r requirements-server.in -django-storages[azure]==1.14.2 +django-storages[azure]==1.14.4 # via -r requirements-server.in djangorestframework==3.14.0 # via @@ -129,21 +126,21 @@ djangorestframework==3.14.0 # drf-yasg djangorestframework-simplejwt==5.3.1 # via -r requirements-server.in -drf-nested-routers==0.93.5 +drf-nested-routers==0.94.0 # via -r requirements-server.in -drf-yasg==1.21.7 +drf-yasg==1.21.8 # via -r requirements-server.in -fastparquet==2024.2.0 +fastparquet==2024.5.0 # via oasis-data-manager -fsspec==2024.2.0 +fsspec==2024.10.0 # via # fastparquet # oasis-data-manager -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -gunicorn==22.0.0 +gunicorn==23.0.0 # via -r requirements-server.in -hiredis==2.3.2 +hiredis==3.0.0 # via aioredis httplib2==0.22.0 # via pyrabbit @@ -151,7 +148,7 @@ hyperlink==21.0.0 # via # autobahn # twisted -idna==3.6 +idna==3.10 # via # hyperlink # requests @@ -160,47 +157,47 @@ incremental==24.7.2 # via twisted inflection==0.5.1 # via drf-yasg -isodate==0.6.1 +isodate==0.7.2 # via azure-storage-blob itypes==1.2.0 # via coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via coreschema jmespath==1.0.1 # via # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via -r requirements-server.in josepy==1.14.0 # via mozilla-django-oidc -jsonpickle==3.0.3 +jsonpickle==3.3.0 # via -r requirements-server.in jsonref==1.1.0 # via ods-tools -jsonschema==4.21.1 +jsonschema==4.23.0 # via # -r requirements-server.in # ods-tools -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -kombu==5.3.5 +kombu==5.4.2 # via celery -llvmlite==0.41.1 +llvmlite==0.43.0 # via numba -markdown==3.5.2 +markdown==3.7 # via -r requirements-server.in -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 -mozilla-django-oidc==4.0.0 +mozilla-django-oidc==4.0.1 # via -r requirements-server.in msgpack==0.6.2 # via channels-redis mysqlclient==2.1.1 # via -r requirements-server.in -numba==0.58.0 +numba==0.60.0 # via ods-tools -numpy==1.25.2 +numpy==2.0.2 # via # fastparquet # numba @@ -210,15 +207,15 @@ oasis-data-manager==0.1.3 # via # -r requirements-server.in # ods-tools -ods-tools==3.2.6 +ods-tools==3.2.7 # via -r requirements-server.in -packaging==23.2 +packaging==24.1 # via # drf-yasg # fastparquet # gunicorn # ods-tools -pandas==2.1.4 +pandas==2.2.3 # via # -r requirements-server.in # fastparquet @@ -226,29 +223,29 @@ pandas==2.1.4 # ods-tools pathlib2==2.3.7.post1 # via -r requirements-server.in -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via click-repl -psycopg2-binary==2.9.9 +psycopg2-binary==2.9.10 # via -r requirements-server.in -pyarrow==15.0.0 +pyarrow==17.0.0 # via -r requirements-server.in -pyasn1==0.5.1 +pyasn1==0.6.1 # via # pyasn1-modules # service-identity -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via service-identity -pycparser==2.21 +pycparser==2.22 # via cffi -pyjwt==2.8.0 +pyjwt==2.9.0 # via djangorestframework-simplejwt pymysql==1.1.1 # via -r requirements-server.in -pyopenssl==24.0.0 +pyopenssl==24.2.1 # via # josepy # twisted -pyparsing==3.1.2 +pyparsing==3.2.0 # via httplib2 pyrabbit==1.1.0 # via -r requirements-server.in @@ -257,69 +254,65 @@ python-dateutil==2.9.0.post0 # botocore # celery # pandas -pytz==2024.1 +pytz==2024.2 # via # django # djangorestframework # drf-yasg # pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via drf-yasg -redis==5.0.2 +redis==5.2.0 # via -r requirements-server.in -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.3 # via # azure-core # coreapi # mozilla-django-oidc -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing -s3transfer==0.10.0 +s3transfer==0.10.3 # via boto3 -service-identity==24.1.0 +service-identity==24.2.0 # via twisted six==1.16.0 # via - # automat # azure-core - # isodate # pathlib2 # python-dateutil -sqlalchemy==2.0.28 +sqlalchemy==2.0.36 # via -r requirements-server.in -sqlparse==0.5.0 +sqlparse==0.5.1 # via # django # django-debug-toolbar -tomli==2.0.1 - # via incremental -twisted[tls]==24.7.0rc1 +twisted[tls]==24.10.0 # via daphne txaio==23.1.1 # via autobahn -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via - # asgiref # azure-core # azure-storage-blob # oasis-data-manager # sqlalchemy # twisted -tzdata==2024.1 +tzdata==2024.2 # via # celery + # kombu # pandas uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==2.0.7 +urllib3==2.2.3 # via # botocore # requests @@ -330,9 +323,9 @@ vine==5.1.0 # kombu wcwidth==0.2.13 # via prompt-toolkit -whitenoise==6.6.0 +whitenoise==6.7.0 # via -r requirements-server.in -zope-interface==6.2 +zope-interface==7.1.1 # via twisted # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements-worker.txt b/requirements-worker.txt index 8f52661f4..dc79d8ebe 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -1,19 +1,21 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements-worker.in # -adlfs==2024.2.0 +adlfs==2024.7.0 # via -r requirements-worker.in -aiobotocore==2.12.1 +aiobotocore==2.15.2 # via s3fs -aiohttp==3.9.4 +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.10.10 # via # adlfs # aiobotocore # s3fs -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore aiosignal==1.3.1 # via aiohttp @@ -23,17 +25,12 @@ anytree==2.12.1 # via oasislmf argparsetree==0.0.6 # via oasislmf -async-timeout==4.0.3 - # via - # aiohttp - # redis -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp - # fiona # jsonschema # referencing -azure-core==1.30.1 +azure-core==1.31.0 # via # -r requirements-worker.in # adlfs @@ -41,31 +38,31 @@ azure-core==1.30.1 # azure-storage-blob azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.15.0 +azure-identity==1.19.0 # via adlfs -azure-storage-blob==12.19.1 +azure-storage-blob==12.23.1 # via # -r requirements-worker.in # adlfs -billiard==4.2.0 +billiard==4.2.1 # via # -r requirements-worker.in # celery -boto3==1.34.51 +boto3==1.35.36 # via -r requirements-worker.in -botocore==1.34.51 +botocore==1.35.36 # via # aiobotocore # boto3 # s3transfer -celery==5.3.6 +celery==5.4.0 # via -r requirements-worker.in -certifi==2024.2.2 +certifi==2024.8.30 # via - # fiona + # pyogrio # pyproj # requests -cffi==1.16.0 +cffi==1.17.1 # via # azure-datalake-store # cryptography @@ -75,7 +72,7 @@ chardet==5.2.0 # via # oasislmf # ods-tools -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -83,111 +80,103 @@ click==8.1.7 # click-didyoumean # click-plugins # click-repl - # cligj - # fiona -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 - # via - # celery - # fiona + # via celery click-repl==0.3.0 # via celery -cligj==0.7.2 - # via fiona -configparser==6.0.1 +configparser==7.1.0 # via -r requirements-worker.in -cramjam==2.8.2 +cramjam==2.9.0 # via fastparquet -cryptography==42.0.5 +cryptography==43.0.3 # via # azure-identity # azure-storage-blob # msal # pyjwt -exceptiongroup==1.2.1 - # via pytest fasteners==0.19 # via -r requirements-worker.in -fastparquet==2024.2.0 +fastparquet==2024.5.0 # via # oasis-data-manager # oasislmf -filelock==3.13.1 +filelock==3.16.1 # via -r requirements-worker.in -fiona==1.10b2 - # via geopandas forex-python==1.8 # via oasislmf -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal -fsspec==2024.2.0 +fsspec==2024.10.0 # via # adlfs # fastparquet # oasis-data-manager # s3fs -geopandas==0.14.3 +geopandas==1.0.1 # via oasislmf -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -idna==3.6 +idna==3.10 # via # requests # yarl iniconfig==2.0.0 # via pytest -isodate==0.6.1 +isodate==0.7.2 # via azure-storage-blob jmespath==1.0.1 # via # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via # -r requirements-worker.in # scikit-learn jsonref==1.1.0 # via ods-tools -jsonschema==4.21.1 +jsonschema==4.23.0 # via ods-tools -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -kombu==5.3.5 +kombu==5.4.2 # via celery -llvmlite==0.41.1 +llvmlite==0.43.0 # via numba -msal==1.27.0 +msal==1.31.0 # via # azure-datalake-store # azure-identity # msal-extensions -msal-extensions==1.1.0 +msal-extensions==1.2.0 # via azure-identity -msgpack==1.0.8 +msgpack==1.1.0 # via oasislmf -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl natsort==8.4.0 # via -r requirements-worker.in -numba==0.58.0 +numba==0.60.0 # via # oasislmf # ods-tools -numexpr==2.9.0 +numexpr==2.10.1 # via oasislmf -numpy==1.25.2 +numpy==1.26.4 # via # fastparquet + # geopandas # numba # numexpr # oasislmf # pandas # pyarrow + # pyogrio # scikit-learn # scipy # shapely @@ -196,16 +185,16 @@ oasis-data-manager==0.1.3 # -r requirements-worker.in # oasislmf # ods-tools -oasislmf[extra]==2.3.7 +oasislmf[extra]==2.3.9 # via -r requirements-worker.in -ods-tools==3.2.6 +ods-tools==3.2.7 # via oasislmf -packaging==23.2 +packaging==24.1 # via # fastparquet # geopandas - # msal-extensions # ods-tools + # pyogrio # pytest pandas==2.1.4 # via @@ -216,44 +205,48 @@ pandas==2.1.4 # ods-tools pathlib2==2.3.7.post1 # via -r requirements-worker.in -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -portalocker==2.8.2 +portalocker==2.10.1 # via msal-extensions -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via click-repl -psycopg2-binary==2.9.9 +propcache==0.2.0 + # via yarl +psycopg2-binary==2.9.10 # via -r requirements-worker.in -pyarrow==16.0.0 +pyarrow==17.0.0 # via oasislmf -pycparser==2.21 +pycparser==2.22 # via cffi -pyjwt[crypto]==2.8.0 +pyjwt[crypto]==2.9.0 # via # msal # pyjwt pymysql==1.1.1 # via -r requirements-worker.in -pyproj==3.6.1 +pyogrio==0.10.0 + # via geopandas +pyproj==3.7.0 # via geopandas -pytest==8.0.2 +pytest==8.3.3 # via -r requirements-worker.in python-dateutil==2.9.0.post0 # via # botocore # celery # pandas -pytz==2024.1 +pytz==2024.2 # via # oasislmf # pandas -redis==5.0.2 +redis==5.2.0 # via -r requirements-worker.in -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.3 # via # azure-core # azure-datalake-store @@ -263,60 +256,59 @@ requests==2.31.0 # requests-toolbelt requests-toolbelt==1.0.0 # via oasislmf -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing -rtree==1.2.0 +rtree==1.3.0 # via oasislmf -s3fs==2024.2.0 +s3fs==2024.10.0 # via -r requirements-worker.in -s3transfer==0.10.0 +s3transfer==0.10.3 # via boto3 -scikit-learn==1.4.1.post1 +scikit-learn==1.5.2 # via oasislmf -scipy==1.12.0 +scipy==1.14.1 # via # oasislmf # scikit-learn -shapely==2.0.3 +shapely==2.0.6 # via # geopandas # oasislmf shutilwhich==1.1.0 # via oasislmf -simplejson==3.19.2 +simplejson==3.19.3 # via forex-python six==1.16.0 # via # anytree # azure-core - # isodate # pathlib2 # python-dateutil -sqlalchemy==2.0.28 +sqlalchemy==2.0.36 # via -r requirements-worker.in tabulate==0.9.0 # via oasislmf tblib==3.0.0 # via oasislmf -threadpoolctl==3.3.0 +threadpoolctl==3.5.0 # via scikit-learn -tomli==2.0.1 - # via pytest -tqdm==4.66.2 +tqdm==4.66.5 # via oasislmf -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # azure-core + # azure-identity # azure-storage-blob # oasis-data-manager # sqlalchemy -tzdata==2024.1 +tzdata==2024.2 # via # celery + # kombu # pandas -urllib3==2.0.7 +urllib3==2.2.3 # via # botocore # requests @@ -329,5 +321,5 @@ wcwidth==0.2.13 # via prompt-toolkit wrapt==1.16.0 # via aiobotocore -yarl==1.9.4 +yarl==1.16.0 # via aiohttp diff --git a/requirements.txt b/requirements.txt index 6c65136ee..6c6dd876f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,21 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements.in # -adlfs==2024.2.0 +adlfs==2024.7.0 # via -r ./requirements-worker.in -aiobotocore==2.12.1 +aiobotocore==2.15.2 # via s3fs -aiohttp==3.9.4 +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.10.10 # via # adlfs # aiobotocore # s3fs -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore aioredis==1.3.1 # via channels-redis @@ -25,7 +27,7 @@ anytree==2.12.1 # via oasislmf argparsetree==0.0.6 # via oasislmf -asgiref==3.7.2 +asgiref==3.8.1 # via # channels # channels-redis @@ -34,25 +36,20 @@ asgiref==3.7.2 asttokens==2.4.1 # via stack-data async-timeout==4.0.3 + # via aioredis +attrs==24.2.0 # via # aiohttp - # aioredis - # redis -attrs==23.2.0 - # via - # aiohttp - # automat - # fiona # hypothesis # jsonschema # referencing # service-identity # twisted -autobahn==23.6.2 +autobahn==24.4.2 # via daphne -automat==22.10.0 +automat==24.8.1 # via twisted -azure-core==1.30.1 +azure-core==1.31.0 # via # -r ./requirements-worker.in # adlfs @@ -61,9 +58,9 @@ azure-core==1.30.1 # django-storages azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.15.0 +azure-identity==1.19.0 # via adlfs -azure-storage-blob==12.19.1 +azure-storage-blob==12.23.1 # via # -r ./requirements-worker.in # adlfs @@ -74,34 +71,34 @@ backports-weakref==1.0.post1 # via backports-tempfile beautifulsoup4==4.12.3 # via webtest -billiard==4.2.0 +billiard==4.2.1 # via # -r ./requirements-worker.in # celery -boto3==1.34.51 +boto3==1.35.36 # via # -r ./requirements-server.in # -r ./requirements-worker.in -botocore==1.34.51 +botocore==1.35.36 # via # aiobotocore # boto3 # s3transfer -build==1.1.1 +build==1.2.2.post1 # via pip-tools -cachetools==5.3.3 +cachetools==5.5.0 # via tox -celery==5.3.6 +celery==5.4.0 # via # -r ./requirements-server.in # -r ./requirements-worker.in # django-celery-results -certifi==2024.2.2 +certifi==2024.8.30 # via - # fiona + # pyogrio # pyproj # requests -cffi==1.16.0 +cffi==1.17.1 # via # azure-datalake-store # cryptography @@ -120,7 +117,7 @@ chardet==5.2.0 # oasislmf # ods-tools # tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -128,22 +125,16 @@ click==8.1.7 # click-didyoumean # click-plugins # click-repl - # cligj - # fiona # pip-tools -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 - # via - # celery - # fiona + # via celery click-repl==0.3.0 # via celery -cligj==0.7.2 - # via fiona colorama==0.4.6 # via tox -configparser==6.0.1 +configparser==7.1.0 # via -r ./requirements-worker.in constantly==23.10.4 # via twisted @@ -151,13 +142,13 @@ coreapi==2.3.3 # via -r ./requirements-server.in coreschema==0.0.4 # via coreapi -coverage[toml]==7.4.3 +coverage[toml]==7.6.4 # via # -r requirements.in # pytest-cov -cramjam==2.8.2 +cramjam==2.9.0 # via fastparquet -cryptography==42.0.5 +cryptography==43.0.3 # via # autobahn # azure-identity @@ -174,7 +165,7 @@ decorator==5.1.1 # via # ipdb # ipython -distlib==0.3.8 +distlib==0.3.9 # via virtualenv django==3.2.25 # via @@ -193,7 +184,7 @@ django==3.2.25 # mozilla-django-oidc django-celery-results==2.5.1 # via -r ./requirements-server.in -django-cleanup==8.1.0 +django-cleanup==9.0.0 # via -r ./requirements-server.in django-debug-toolbar==4.3.0 # via @@ -201,13 +192,13 @@ django-debug-toolbar==4.3.0 # -r requirements.in django-filter==23.5 # via -r ./requirements-server.in -django-model-utils==4.4.0 +django-model-utils==5.0.0 # via -r ./requirements-server.in django-request-logging==0.7.5 # via -r ./requirements-server.in -django-storages[azure]==1.14.2 +django-storages[azure]==1.14.4 # via -r ./requirements-server.in -django-webtest==1.9.11 +django-webtest==1.9.12 # via -r requirements.in djangorestframework==3.14.0 # via @@ -217,55 +208,48 @@ djangorestframework==3.14.0 # drf-yasg djangorestframework-simplejwt==5.3.1 # via -r ./requirements-server.in -drf-nested-routers==0.93.5 +drf-nested-routers==0.94.0 # via -r ./requirements-server.in -drf-yasg==1.21.7 +drf-yasg==1.21.8 # via -r ./requirements-server.in -exceptiongroup==1.2.0 - # via - # hypothesis - # ipython - # pytest -executing==2.0.1 +executing==2.1.0 # via stack-data fasteners==0.19 # via # -r ./requirements-worker.in # -r requirements.in -fastparquet==2024.2.0 +fastparquet==2024.5.0 # via # oasis-data-manager # oasislmf -filelock==3.13.1 +filelock==3.16.1 # via # -r ./requirements-worker.in # tox # virtualenv -fiona==1.10b2 - # via geopandas -flake8==7.0.0 +flake8==7.1.1 # via -r requirements.in forex-python==1.8 # via oasislmf -freezegun==1.4.0 +freezegun==1.5.1 # via -r requirements.in -frozenlist==1.4.1 +frozenlist==1.5.0 # via # aiohttp # aiosignal -fsspec==2024.2.0 +fsspec==2024.10.0 # via # adlfs # fastparquet # oasis-data-manager # s3fs -geopandas==0.14.3 +geopandas==1.0.1 # via oasislmf -greenlet==3.0.3 +greenlet==3.1.1 # via sqlalchemy -gunicorn==22.0.0 +gunicorn==23.0.0 # via -r ./requirements-server.in -hiredis==2.3.2 +hiredis==3.0.0 # via aioredis httplib2==0.22.0 # via pyrabbit @@ -273,9 +257,9 @@ hyperlink==21.0.0 # via # autobahn # twisted -hypothesis==6.98.17 +hypothesis==6.115.5 # via -r requirements.in -idna==3.6 +idna==3.10 # via # hyperlink # requests @@ -289,46 +273,46 @@ iniconfig==2.0.0 # via pytest ipdb==0.13.13 # via -r requirements.in -ipython==8.22.2 +ipython==8.29.0 # via ipdb -isodate==0.6.1 +isodate==0.7.2 # via azure-storage-blob itypes==1.2.0 # via coreapi jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via coreschema jmespath==1.0.1 # via # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via # -r ./requirements-server.in # -r ./requirements-worker.in # scikit-learn josepy==1.14.0 # via mozilla-django-oidc -jsonpickle==3.0.3 +jsonpickle==3.3.0 # via -r ./requirements-server.in jsonref==1.1.0 # via ods-tools -jsonschema==4.21.1 +jsonschema==4.23.0 # via # -r ./requirements-server.in # ods-tools -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -kombu==5.3.5 +kombu==5.4.2 # via celery -llvmlite==0.41.1 +llvmlite==0.43.0 # via numba -markdown==3.5.2 +markdown==3.7 # via -r ./requirements-server.in -markupsafe==2.1.5 +markupsafe==3.0.2 # via jinja2 -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via ipython mccabe==0.7.0 # via flake8 @@ -336,20 +320,20 @@ mock==5.1.0 # via -r requirements.in model-mommy==2.0.0 # via -r requirements.in -mozilla-django-oidc==4.0.0 +mozilla-django-oidc==4.0.1 # via -r ./requirements-server.in -msal==1.27.0 +msal==1.31.0 # via # azure-datalake-store # azure-identity # msal-extensions -msal-extensions==1.1.0 +msal-extensions==1.2.0 # via azure-identity msgpack==0.6.2 # via # channels-redis # oasislmf -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl @@ -357,20 +341,22 @@ mysqlclient==2.1.1 # via -r ./requirements-server.in natsort==8.4.0 # via -r ./requirements-worker.in -numba==0.58.0 +numba==0.60.0 # via # oasislmf # ods-tools -numexpr==2.9.0 +numexpr==2.10.1 # via oasislmf -numpy==1.25.2 +numpy==1.26.4 # via # fastparquet + # geopandas # numba # numexpr # oasislmf # pandas # pyarrow + # pyogrio # scikit-learn # scipy # shapely @@ -380,21 +366,21 @@ oasis-data-manager==0.1.3 # -r ./requirements-worker.in # oasislmf # ods-tools -oasislmf[extra]==2.3.7 +oasislmf[extra]==2.3.9 # via -r ./requirements-worker.in -ods-tools==3.2.6 +ods-tools==3.2.7 # via # -r ./requirements-server.in # oasislmf -packaging==23.2 +packaging==24.1 # via # build # drf-yasg # fastparquet # geopandas # gunicorn - # msal-extensions # ods-tools + # pyogrio # pyproject-api # pytest # tox @@ -406,7 +392,7 @@ pandas==2.1.4 # oasis-data-manager # oasislmf # ods-tools -parso==0.8.3 +parso==0.8.4 # via jedi pathlib2==2.3.7.post1 # via @@ -416,47 +402,49 @@ pexpect==4.9.0 # via ipython pip-tools==7.4.1 # via -r requirements.in -platformdirs==4.2.0 +platformdirs==4.3.6 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via # pytest # tox -portalocker==2.8.2 +portalocker==2.10.1 # via msal-extensions -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via # click-repl # ipython -psycopg2-binary==2.9.9 +propcache==0.2.0 + # via yarl +psycopg2-binary==2.9.10 # via # -r ./requirements-server.in # -r ./requirements-worker.in ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data -pyarrow==16.0.0 +pyarrow==17.0.0 # via # -r ./requirements-server.in # oasislmf -pyasn1==0.5.1 +pyasn1==0.6.1 # via # pyasn1-modules # service-identity -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via service-identity -pycodestyle==2.11.1 +pycodestyle==2.12.1 # via flake8 -pycparser==2.21 +pycparser==2.22 # via cffi pyflakes==3.2.0 # via flake8 -pygments==2.17.2 +pygments==2.18.0 # via ipython -pyjwt[crypto]==2.8.0 +pyjwt[crypto]==2.9.0 # via # djangorestframework-simplejwt # msal @@ -464,32 +452,34 @@ pymysql==1.1.1 # via # -r ./requirements-server.in # -r ./requirements-worker.in -pyopenssl==24.0.0 +pyogrio==0.10.0 + # via geopandas +pyopenssl==24.2.1 # via # -r requirements.in # josepy # twisted -pyparsing==3.1.2 +pyparsing==3.2.0 # via httplib2 -pyproj==3.6.1 +pyproj==3.7.0 # via geopandas -pyproject-api==1.6.1 +pyproject-api==1.8.0 # via tox -pyproject-hooks==1.0.0 +pyproject-hooks==1.2.0 # via # build # pip-tools pyrabbit==1.1.0 # via -r ./requirements-server.in -pytest==8.0.2 +pytest==8.3.3 # via # -r ./requirements-worker.in # -r requirements.in # pytest-cov # pytest-django -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements.in -pytest-django==4.8.0 +pytest-django==4.9.0 # via -r requirements.in python-dateutil==2.9.0.post0 # via @@ -497,24 +487,24 @@ python-dateutil==2.9.0.post0 # celery # freezegun # pandas -pytz==2024.1 +pytz==2024.2 # via # django # djangorestframework # drf-yasg # oasislmf # pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via drf-yasg -redis==5.0.2 +redis==5.2.0 # via # -r ./requirements-server.in # -r ./requirements-worker.in -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.3 # via # -r requirements.in # azure-core @@ -527,50 +517,48 @@ requests==2.31.0 # requests-toolbelt requests-toolbelt==1.0.0 # via oasislmf -rpds-py==0.18.0 +rpds-py==0.20.0 # via # jsonschema # referencing -rtree==1.2.0 +rtree==1.3.0 # via oasislmf -s3fs==2024.2.0 +s3fs==2024.10.0 # via -r ./requirements-worker.in -s3transfer==0.10.0 +s3transfer==0.10.3 # via boto3 -scikit-learn==1.4.1.post1 +scikit-learn==1.5.2 # via oasislmf -scipy==1.12.0 +scipy==1.14.1 # via # oasislmf # scikit-learn -service-identity==24.1.0 +service-identity==24.2.0 # via twisted -shapely==2.0.3 +shapely==2.0.6 # via # geopandas # oasislmf shutilwhich==1.1.0 # via oasislmf -simplejson==3.19.2 +simplejson==3.19.3 # via forex-python six==1.16.0 # via # anytree # asttokens - # automat # azure-core - # isodate # pathlib2 # python-dateutil sortedcontainers==2.4.0 # via hypothesis -soupsieve==2.5 +soupsieve==2.6 # via beautifulsoup4 -sqlalchemy==2.0.28 +sqlalchemy==2.0.36 # via # -r ./requirements-server.in # -r ./requirements-worker.in -sqlparse==0.5.0 +sqlparse==0.5.1 # via # django # django-debug-toolbar @@ -580,48 +568,38 @@ tabulate==0.9.0 # via oasislmf tblib==3.0.0 # via oasislmf -threadpoolctl==3.3.0 +threadpoolctl==3.5.0 # via scikit-learn -tomli==2.0.1 - # via - # build - # coverage - # incremental - # ipdb - # pip-tools - # pyproject-api - # pyproject-hooks - # pytest - # tox -tox==4.4.11 +tox==4.23.2 # via -r requirements.in -tqdm==4.66.2 +tqdm==4.66.5 # via oasislmf -traitlets==5.14.1 +traitlets==5.14.3 # via # ipython # matplotlib-inline -twisted[tls]==24.7.0rc1 +twisted[tls]==24.10.0 # via daphne txaio==23.1.1 # via autobahn -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via - # asgiref # azure-core + # azure-identity # azure-storage-blob # oasis-data-manager # sqlalchemy # twisted -tzdata==2024.1 +tzdata==2024.2 # via # celery + # kombu # pandas uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==2.0.7 +urllib3==2.2.3 # via # botocore # requests @@ -630,25 +608,25 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.25.1 +virtualenv==20.27.0 # via tox waitress==3.0.0 # via webtest wcwidth==0.2.13 # via prompt-toolkit -webob==1.8.7 +webob==1.8.9 # via webtest -webtest==3.0.0 +webtest==3.0.1 # via django-webtest -wheel==0.42.0 +wheel==0.44.0 # via pip-tools -whitenoise==6.6.0 +whitenoise==6.7.0 # via -r ./requirements-server.in wrapt==1.16.0 # via aiobotocore -yarl==1.9.4 +yarl==1.16.0 # via aiohttp -zope-interface==6.2 +zope-interface==7.1.1 # via twisted # The following packages are considered to be unsafe in a requirements file: From 47be68a156df065cf8db97a3bd45fd4daa2e5a85 Mon Sep 17 00:00:00 2001 From: Sam Gamble Date: Mon, 28 Oct 2024 10:12:47 +0000 Subject: [PATCH 3/3] PEP --- src/model_execution_worker/distributed_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/model_execution_worker/distributed_tasks.py b/src/model_execution_worker/distributed_tasks.py index 98cc9c711..75648dc3b 100644 --- a/src/model_execution_worker/distributed_tasks.py +++ b/src/model_execution_worker/distributed_tasks.py @@ -562,7 +562,7 @@ def prepare_keys_file_chunk( ): with TemporaryDir() as chunk_target_dir: - chunk_target_dir = os.path.join(chunk_target_dir, f'lookup-{chunk_idx+1}') + chunk_target_dir = os.path.join(chunk_target_dir, f'lookup-{chunk_idx + 1}') Path(chunk_target_dir).mkdir(parents=True, exist_ok=True) _, lookup = OasisLookupFactory.create( @@ -599,7 +599,7 @@ def prepare_keys_file_chunk( # Store chunks params['chunk_keys'] = filestore.put( chunk_target_dir, - filename=f'lookup-{chunk_idx+1}.tar.gz', + filename=f'lookup-{chunk_idx + 1}.tar.gz', subdir=params['storage_subdir'] ) @@ -1000,12 +1000,12 @@ def generate_losses_chunk(self, params, chunk_idx, num_chunks, analysis_id=None, **params, 'chunk_work_location': filestore.put( chunk_params['ktools_work_dir'], - filename=f'work-{chunk_idx+1}.tar.gz', + filename=f'work-{chunk_idx + 1}.tar.gz', subdir=params['storage_subdir'] ), 'chunk_log_location': filestore.put( chunk_params['ktools_log_dir'], - filename=f'log-{chunk_idx+1}.tar.gz', + filename=f'log-{chunk_idx + 1}.tar.gz', subdir=params['storage_subdir'] ), 'ktools_work_dir': chunk_params['ktools_work_dir'],