From ab9bb3a615bef176cddedb472881a345e78fe8ff Mon Sep 17 00:00:00 2001 From: Sam Gamble Date: Mon, 21 Oct 2024 15:30:41 +0100 Subject: [PATCH 1/6] Update python 3.12 (#1124) --- .github/workflows/build-images.yml | 2 + .github/workflows/build-schema.yml | 4 +- .github/workflows/code-quality.yml | 2 +- .github/workflows/scan-external.yml | 2 + .github/workflows/scan.yml | 3 + .github/workflows/test-images.yml | 7 +- .github/workflows/test-python_api.yml | 4 +- .github/workflows/update-python.yml | 4 +- .github/workflows/version.yml | 4 +- Dockerfile.api_server | 12 +- Dockerfile.model_worker | 9 +- kubernetes/worker-controller/requirements.txt | 4 +- requirements-server.txt | 204 ++++++----- requirements-worker.in | 8 +- requirements-worker.txt | 154 ++++---- requirements.txt | 330 ++++++++---------- scripts/update-packages.sh | 4 +- .../distributed_tasks.py | 8 +- .../permissions/tests/test_group_auth.py | 2 +- 19 files changed, 378 insertions(+), 389 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e4bd2a285..e9874aac1 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -72,6 +72,8 @@ env: IGNORE_UNFIXED: 'true' SEVERITY: 'CRITICAL,HIGH' IMAGE_REPO: 'coreoasis/github-actions' + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true jobs: build: diff --git a/.github/workflows/build-schema.yml b/.github/workflows/build-schema.yml index c8cd2d4a3..038ce2aee 100644 --- a/.github/workflows/build-schema.yml +++ b/.github/workflows/build-schema.yml @@ -36,10 +36,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install requirments run: pip install -r requirements-server.txt diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 700f2f4f9..bc7c5613b 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: install pip tools run: | diff --git a/.github/workflows/scan-external.yml b/.github/workflows/scan-external.yml index c7bd588d0..7b3679f2c 100644 --- a/.github/workflows/scan-external.yml +++ b/.github/workflows/scan-external.yml @@ -15,6 +15,8 @@ on: env: SEVERITY: 'HIGH,CRITICAL' IGNORE_UNFIXED: 'true' + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true jobs: scan_external_images: diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index 1b0e10d08..ef4316f46 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -42,6 +42,9 @@ jobs: REPORT: 'repo-results.sarif' IGNORE_UNFIXED: 'true' EXIT_CODE: '1' + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + runs-on: ubuntu-latest steps: - name: Adding code-scanning URL diff --git a/.github/workflows/test-images.yml b/.github/workflows/test-images.yml index 233476dd9..813c8c5b7 100644 --- a/.github/workflows/test-images.yml +++ b/.github/workflows/test-images.yml @@ -109,16 +109,15 @@ jobs: # Select matching base branch on piwind if [[ "${{ github.event_name }}" = "pull_request" ]]; then BRANCH=${{ github.base_ref }} + if [[ ! $BRANCH == stable/* ]]; then + BRANCH='main' + fi elif [[ "${{ github.event_name }}" = "push" ]]; then BRANCH=${{ github.ref_name }} else BRANCH=${{ inputs.piwind_branch }} fi - #override 'main-platform1' -> 'main' - if [[ "$BRANCH" = 'main-platform1' ]]; then - BRANCH=main - fi echo "branch=$BRANCH" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-python_api.yml b/.github/workflows/test-python_api.yml index c576181dd..c26ba5bec 100644 --- a/.github/workflows/test-python_api.yml +++ b/.github/workflows/test-python_api.yml @@ -48,10 +48,10 @@ jobs: repository: OasisLMF/OasisPlatform ref: ${{ env.PLAT_BRANCH }} - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - run: pip install -r requirements.txt - name: Download package diff --git a/.github/workflows/update-python.yml b/.github/workflows/update-python.yml index 1430096b4..222764ee1 100644 --- a/.github/workflows/update-python.yml +++ b/.github/workflows/update-python.yml @@ -32,10 +32,10 @@ jobs: GIT_EMAIL: ${{ secrets.BUILD_GIT_EMAIL }} GIT_USERNAME: ${{ secrets.BUILD_GIT_USERNAME }} - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install piptools run: pip install pip-tools diff --git a/.github/workflows/version.yml b/.github/workflows/version.yml index 4d9c683e6..4c634c9f1 100644 --- a/.github/workflows/version.yml +++ b/.github/workflows/version.yml @@ -77,11 +77,11 @@ jobs: echo ${{ inputs.platform_version }} > VERSION git add VERSION - - name: Set up Python 3.10 + - name: Set up Python 3.12 if: inputs.oasislmf_version != '' uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install piptools run: pip install pip-tools diff --git a/Dockerfile.api_server b/Dockerfile.api_server index 7417b8a79..b2ddc8a9f 100755 --- a/Dockerfile.api_server +++ b/Dockerfile.api_server @@ -1,9 +1,10 @@ # ---- STAGE 1 ----- -FROM ubuntu:22.04 AS build-packages +FROM ubuntu:24.04 AS build-packages +ENV PIP_BREAK_SYSTEM_PACKAGES 1 ENV DEBIAN_FRONTEND noninteractive COPY ./requirements-server.txt ./requirements-server.txt -RUN apt-get update && apt-get install -y --no-install-recommends gcc build-essential python3 python3-pip python3-dev libmariadbclient-dev-compat && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends gcc build-essential python3 python3-pip python3-dev pkg-config libmariadb-dev-compat && rm -rf /var/lib/apt/lists/* RUN pip install --user --no-warn-script-location -r ./requirements-server.txt && pip install --no-warn-script-location --user mysqlclient # Install ODS-Tools from git branch (Optional) 'docker build --build-arg ods_tools_branch=develop' @@ -17,13 +18,16 @@ RUN if [ ! -z "$ods_tools_branch" ] ; then \ USER server # ---- STAGE 2 ---- -FROM ubuntu:22.04 +FROM ubuntu:24.04 RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends sudo python3 python3-pkg-resources curl libmariadbclient-dev-compat \ && rm -rf /var/lib/apt/lists/ -RUN adduser --home /home/server --shell /bin/bash --disabled-password --gecos "" server +ARG USER_ID=1000 +RUN userdel -r ubuntu +RUN useradd -u ${USER_ID} --home /home/server --shell /bin/bash server + COPY --chown=server:server --from=build-packages /root/.local /home/server/.local RUN mkdir -p /var/log/oasis /shared-fs && chmod 777 -R /var/log/oasis diff --git a/Dockerfile.model_worker b/Dockerfile.model_worker index 8b2b96ac8..cbbbc5a1c 100755 --- a/Dockerfile.model_worker +++ b/Dockerfile.model_worker @@ -1,7 +1,8 @@ # ---- STAGE 1 ----- -FROM ubuntu:22.04 AS build-packages +FROM ubuntu:24.04 AS build-packages # Build python packages +ENV PIP_BREAK_SYSTEM_PACKAGES 1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y libspatialindex-dev git curl g++ build-essential libtool autoconf automake python3-dev python3 python3-pip pkg-config COPY ./requirements-worker.txt ./requirements-worker.txt @@ -26,14 +27,16 @@ RUN if [ ! -z "$ods_tools_branch" ] ; then \ USER worker # ---- STAGE 2 ---- -FROM ubuntu:22.04 +FROM ubuntu:24.04 RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends vim git python3 python3-pip libspatialindex-dev curl procps \ && rm -rf /var/lib/apt/lists/* # Copy built python packages -RUN adduser --home /home/worker --shell /bin/bash --disabled-password --gecos "" worker +ARG USER_ID=1000 +RUN userdel -r ubuntu +RUN useradd -u ${USER_ID} --home /home/worker --shell /bin/bash worker COPY --chown=worker:worker --from=build-packages /root/.local /home/worker/.local # Copy in worker files diff --git a/kubernetes/worker-controller/requirements.txt b/kubernetes/worker-controller/requirements.txt index 8e2a1cf82..304595e94 100644 --- a/kubernetes/worker-controller/requirements.txt +++ b/kubernetes/worker-controller/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile kubernetes/worker-controller/requirements.in @@ -10,8 +10,6 @@ aiohttp==3.9.4 # kubernetes-asyncio aiosignal==1.3.1 # via aiohttp -async-timeout==4.0.3 - # via aiohttp attrs==22.1.0 # via aiohttp certifi==2023.7.22 diff --git a/requirements-server.txt b/requirements-server.txt index d7edfa7c9..a3501439e 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -1,52 +1,52 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements-server.in # aioredis==1.3.1 # via channels-redis -amqp==5.1.1 +amqp==5.2.0 # via kombu -asgiref==3.6.0 +asgiref==3.8.1 # via # channels # channels-redis # daphne # django -async-timeout==4.0.2 - # via - # aioredis - # redis -attrs==22.2.0 +async-timeout==4.0.3 + # via aioredis +attrs==24.2.0 # via - # automat # jsonschema + # referencing # service-identity # twisted -autobahn==23.6.2 +autobahn==24.4.2 # via daphne -automat==22.10.0 +automat==24.8.1 # via twisted -azure-core==1.26.3 - # via azure-storage-blob -azure-storage-blob==12.15.0 +azure-core==1.31.0 + # via + # azure-storage-blob + # django-storages +azure-storage-blob==12.23.1 # via django-storages -billiard==4.1.0 +billiard==4.2.1 # via celery -boto3==1.26.107 +boto3==1.35.44 # via -r requirements-server.in -botocore==1.29.107 +botocore==1.35.44 # via # boto3 # s3transfer -celery==5.3.0 +celery==5.4.0 # via # -r requirements-server.in # django-celery-results -certifi==2023.7.22 +certifi==2024.8.30 # via requests -cffi==1.15.1 +cffi==1.17.1 # via cryptography chainmap==1.0.3 # via -r requirements-server.in @@ -56,35 +56,31 @@ channels==2.4.0 # channels-redis channels-redis==2.4.2 # via -r requirements-server.in -chardet==5.1.0 +chardet==5.2.0 # via ods-tools -charset-normalizer==3.1.0 +charset-normalizer==3.4.0 # via requests -click==8.1.3 +click==8.1.7 # via # celery # click-didyoumean # click-plugins # click-repl -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery -click-repl==0.2.0 +click-repl==0.3.0 # via celery -constantly==15.1.0 +constantly==23.10.4 # via twisted coreapi==2.3.3 - # via - # -r requirements-server.in - # drf-yasg + # via -r requirements-server.in coreschema==0.0.4 - # via - # coreapi - # drf-yasg -cramjam==2.8.1 + # via coreapi +cramjam==2.9.0 # via fastparquet -cryptography==42.0.4 +cryptography==43.0.3 # via # autobahn # azure-storage-blob @@ -94,7 +90,7 @@ cryptography==42.0.4 # service-identity daphne==2.5.0 # via channels -django==3.2.23 +django==3.2.25 # via # channels # django-celery-results @@ -108,19 +104,19 @@ django==3.2.23 # drf-nested-routers # drf-yasg # mozilla-django-oidc -django-celery-results==2.5.0 +django-celery-results==2.5.1 # via -r requirements-server.in -django-cleanup==7.0.0 +django-cleanup==9.0.0 # via -r requirements-server.in -django-debug-toolbar==4.0.0 +django-debug-toolbar==4.3.0 # via -r requirements-server.in -django-filter==23.1 +django-filter==23.5 # via -r requirements-server.in -django-model-utils==4.3.1 +django-model-utils==5.0.0 # via -r requirements-server.in django-request-logging==0.7.5 # via -r requirements-server.in -django-storages[azure]==1.13.2 +django-storages[azure]==1.14.4 # via -r requirements-server.in djangorestframework==3.14.0 # via @@ -128,23 +124,23 @@ djangorestframework==3.14.0 # djangorestframework-simplejwt # drf-nested-routers # drf-yasg -djangorestframework-simplejwt==5.2.2 +djangorestframework-simplejwt==5.3.1 # via -r requirements-server.in -drf-nested-routers==0.93.4 +drf-nested-routers==0.94.0 # via -r requirements-server.in -drf-yasg==1.21.5 +drf-yasg==1.21.8 # via -r requirements-server.in -fastparquet==2023.10.1 +fastparquet==2024.5.0 # via oasis-data-manager -fsspec==2024.2.0 +fsspec==2024.10.0 # via # fastparquet # oasis-data-manager -greenlet==2.0.2 +greenlet==3.1.1 # via sqlalchemy -gunicorn==22.0.0 +gunicorn==23.0.0 # via -r requirements-server.in -hiredis==2.2.2 +hiredis==3.0.0 # via aioredis httplib2==0.22.0 # via pyrabbit @@ -152,7 +148,7 @@ hyperlink==21.0.0 # via # autobahn # twisted -idna==3.4 +idna==3.10 # via # hyperlink # requests @@ -161,45 +157,47 @@ incremental==24.7.2 # via twisted inflection==0.5.1 # via drf-yasg -isodate==0.6.1 +isodate==0.7.2 # via azure-storage-blob itypes==1.2.0 # via coreapi -jinja2==3.1.2 +jinja2==3.1.4 # via coreschema jmespath==1.0.1 # via # boto3 # botocore -joblib==1.2.0 +joblib==1.4.2 # via -r requirements-server.in -josepy==1.13.0 +josepy==1.14.0 # via mozilla-django-oidc -jsonpickle==3.0.1 +jsonpickle==3.3.0 # via -r requirements-server.in jsonref==1.1.0 # via ods-tools -jsonschema==4.17.3 +jsonschema==4.23.0 # via # -r requirements-server.in # ods-tools -kombu==5.3.0 +jsonschema-specifications==2024.10.1 + # via jsonschema +kombu==5.4.2 # via celery -llvmlite==0.41.1 +llvmlite==0.43.0 # via numba -markdown==3.4.3 +markdown==3.7 # via -r requirements-server.in -markupsafe==2.1.2 +markupsafe==3.0.2 # via jinja2 -mozilla-django-oidc==3.0.0 +mozilla-django-oidc==4.0.1 # via -r requirements-server.in msgpack==0.6.2 # via channels-redis -mysqlclient==2.1.1 +mysqlclient==2.2.5 # via -r requirements-server.in -numba==0.58.0 +numba==0.60.0 # via ods-tools -numpy==1.25.2 +numpy==2.0.2 # via # fastparquet # numba @@ -209,13 +207,13 @@ oasis-data-manager==0.1.3 # via ods-tools ods-tools==3.2.7 # via -r requirements-server.in -packaging==23.0 +packaging==24.1 # via # drf-yasg # fastparquet # gunicorn # ods-tools -pandas==2.1.4 +pandas==2.2.3 # via # -r requirements-server.in # fastparquet @@ -223,109 +221,109 @@ pandas==2.1.4 # ods-tools pathlib2==2.3.7.post1 # via -r requirements-server.in -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.48 # via click-repl -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.10 # via -r requirements-server.in -pyarrow==14.0.1 +pyarrow==17.0.0 # via -r requirements-server.in -pyasn1==0.4.8 +pyasn1==0.6.1 # via # pyasn1-modules # service-identity -pyasn1-modules==0.2.8 +pyasn1-modules==0.4.1 # via service-identity -pycparser==2.21 +pycparser==2.22 # via cffi -pyjwt==2.6.0 +pyjwt==2.9.0 # via djangorestframework-simplejwt pymysql==1.1.1 # via -r requirements-server.in -pyopenssl==24.0.0 +pyopenssl==24.2.1 # via # josepy # twisted -pyparsing==3.0.9 +pyparsing==3.2.0 # via httplib2 pyrabbit==1.1.0 # via -r requirements-server.in -pyrsistent==0.19.3 - # via jsonschema -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore # celery # pandas -pytz==2023.3 +pytz==2024.2 # via # django # djangorestframework # drf-yasg # pandas -redis==4.5.4 +pyyaml==6.0.2 + # via drf-yasg +redis==5.1.1 # via -r requirements-server.in -requests==2.28.2 +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 # via # azure-core # coreapi # mozilla-django-oidc -ruamel-yaml==0.17.21 - # via drf-yasg -ruamel-yaml-clib==0.2.7 - # via ruamel-yaml -s3transfer==0.6.0 +rpds-py==0.20.0 + # via + # jsonschema + # referencing +s3transfer==0.10.3 # via boto3 -service-identity==21.1.0 +service-identity==24.1.0 # via twisted six==1.16.0 # via - # automat # azure-core - # click-repl - # isodate # pathlib2 # python-dateutil - # service-identity -sqlalchemy==1.4.47 +sqlalchemy==2.0.36 # via -r requirements-server.in -sqlparse==0.5.0 +sqlparse==0.5.1 # via # django # django-debug-toolbar -tomli==2.0.1 - # via incremental -twisted[tls]==24.7.0rc1 +twisted[tls]==24.7.0 # via daphne txaio==23.1.1 # via autobahn -typing-extensions==4.5.0 +typing-extensions==4.12.2 # via # azure-core # azure-storage-blob # oasis-data-manager + # sqlalchemy # twisted -tzdata==2023.3 +tzdata==2024.2 # via # celery + # kombu # pandas uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==1.26.15 +urllib3==2.2.3 # via # botocore # requests -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit -whitenoise==6.4.0 +whitenoise==6.7.0 # via -r requirements-server.in -zope-interface==6.0 +zope-interface==7.1.0 # via twisted # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements-worker.in b/requirements-worker.in index b2143723c..0fef887f1 100644 --- a/requirements-worker.in +++ b/requirements-worker.in @@ -1,14 +1,14 @@ #-e git://github.com/OasisLMF/OasisLMF.git@feature/refactor-arch2020#egg=oasislmf[extra] -oasislmf[extra]>=1.26.5 -azure-core>=1.21.1 -azure-storage-blob>=12.9.0 +oasislmf[extra] +azure-core +azure-storage-blob billiard boto3 celery configparser fasteners filelock -joblib>=1.2.0 +joblib pathlib2 psycopg2-binary pymysql diff --git a/requirements-worker.txt b/requirements-worker.txt index c761aee06..ec1d3a234 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -1,125 +1,123 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements-worker.in # -amqp==5.1.1 +amqp==5.2.0 # via kombu -anytree==2.8.0 +anytree==2.12.1 # via oasislmf argparsetree==0.0.6 # via oasislmf -async-timeout==4.0.2 - # via redis -attrs==22.2.0 +attrs==24.2.0 # via # jsonschema - # pytest -azure-core==1.26.3 + # referencing +azure-core==1.31.0 # via # -r requirements-worker.in # azure-storage-blob -azure-storage-blob==12.15.0 +azure-storage-blob==12.23.1 # via -r requirements-worker.in -billiard==4.1.0 +billiard==4.2.1 # via # -r requirements-worker.in # celery -boto3==1.26.107 +boto3==1.35.44 # via -r requirements-worker.in -botocore==1.29.107 +botocore==1.35.44 # via # boto3 # s3transfer -celery==5.3.0 +celery==5.4.0 # via -r requirements-worker.in -certifi==2023.7.22 +certifi==2024.8.30 # via # pyogrio # pyproj # requests -cffi==1.15.1 +cffi==1.17.1 # via cryptography chainmap==1.0.3 # via oasislmf -chardet==5.1.0 +chardet==5.2.0 # via # oasislmf # ods-tools -charset-normalizer==3.1.0 +charset-normalizer==3.4.0 # via requests -click==8.1.3 +click==8.1.7 # via # celery # click-didyoumean # click-plugins # click-repl -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery -click-repl==0.2.0 +click-repl==0.3.0 # via celery -configparser==5.3.0 +configparser==7.1.0 # via -r requirements-worker.in -cramjam==2.6.2 +cramjam==2.9.0 # via fastparquet -cryptography==42.0.4 +cryptography==43.0.3 # via azure-storage-blob -exceptiongroup==1.1.1 - # via pytest -fasteners==0.18 +fasteners==0.19 # via -r requirements-worker.in -fastparquet==2023.2.0 +fastparquet==2024.5.0 # via # oasis-data-manager # oasislmf -filelock==3.10.7 +filelock==3.16.1 # via -r requirements-worker.in forex-python==1.8 # via oasislmf -fsspec==2023.3.0 +fsspec==2024.10.0 # via # fastparquet # oasis-data-manager geopandas==1.0.1 # via oasislmf -greenlet==2.0.2 +greenlet==3.1.1 # via sqlalchemy -idna==3.4 +idna==3.10 # via requests iniconfig==2.0.0 # via pytest -isodate==0.6.1 +isodate==0.7.2 # via azure-storage-blob jmespath==1.0.1 # via # boto3 # botocore -joblib==1.2.0 +joblib==1.4.2 # via # -r requirements-worker.in # scikit-learn jsonref==1.1.0 # via ods-tools -jsonschema==4.17.3 +jsonschema==4.23.0 # via ods-tools -kombu==5.3.0 +jsonschema-specifications==2024.10.1 + # via jsonschema +kombu==5.4.2 # via celery -llvmlite==0.41.1 +llvmlite==0.43.0 # via numba -msgpack==1.0.5 +msgpack==1.1.0 # via oasislmf -natsort==8.3.1 +natsort==8.4.0 # via -r requirements-worker.in -numba==0.58.0 +numba==0.60.0 # via # oasislmf # ods-tools numexpr==2.10.1 # via oasislmf -numpy==1.25.2 +numpy==1.26.4 # via # fastparquet # geopandas @@ -132,7 +130,7 @@ numpy==1.25.2 # scikit-learn # scipy # shapely -oasis-data-manager==0.1.1 +oasis-data-manager==0.1.3 # via # oasislmf # ods-tools @@ -140,7 +138,7 @@ oasislmf[extra]==2.3.9 # via -r requirements-worker.in ods-tools==3.2.7 # via oasislmf -packaging==23.0 +packaging==24.1 # via # fastparquet # geopandas @@ -156,102 +154,104 @@ pandas==2.1.4 # ods-tools pathlib2==2.3.7.post1 # via -r requirements-worker.in -pluggy==1.0.0 +pluggy==1.5.0 # via pytest -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.48 # via click-repl -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.10 # via -r requirements-worker.in -pyarrow==16.0.0 +pyarrow==17.0.0 # via oasislmf -pycparser==2.21 +pycparser==2.22 # via cffi pymysql==1.1.1 # via -r requirements-worker.in pyogrio==0.10.0 # via geopandas -pyproj==3.5.0 +pyproj==3.7.0 # via geopandas -pyrsistent==0.19.3 - # via jsonschema -pytest==7.2.2 +pytest==8.3.3 # via -r requirements-worker.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore # celery # pandas -pytz==2023.3 +pytz==2024.2 # via # oasislmf # pandas -redis==4.5.5 +redis==5.1.1 # via -r requirements-worker.in -requests==2.28.2 +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 # via # azure-core # forex-python # oasislmf # requests-toolbelt -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via oasislmf -rtree==1.0.1 +rpds-py==0.20.0 + # via + # jsonschema + # referencing +rtree==1.3.0 # via oasislmf -s3transfer==0.6.0 +s3transfer==0.10.3 # via boto3 -scikit-learn==1.2.2 +scikit-learn==1.5.2 # via oasislmf -scipy==1.10.1 +scipy==1.14.1 # via # oasislmf # scikit-learn -shapely==2.0.1 +shapely==2.0.6 # via # geopandas # oasislmf shutilwhich==1.1.0 # via oasislmf -simplejson==3.18.4 +simplejson==3.19.3 # via forex-python six==1.16.0 # via # anytree # azure-core - # click-repl - # isodate # pathlib2 # python-dateutil -sqlalchemy==1.4.47 +sqlalchemy==2.0.36 # via -r requirements-worker.in tabulate==0.9.0 # via oasislmf -tblib==1.7.0 +tblib==3.0.0 # via oasislmf -threadpoolctl==3.1.0 +threadpoolctl==3.5.0 # via scikit-learn -tomli==2.0.1 - # via pytest -tqdm==4.65.0 +tqdm==4.66.5 # via oasislmf -typing==3.7.4.3 - # via oasis-data-manager -typing-extensions==4.5.0 +typing-extensions==4.12.2 # via # azure-core # azure-storage-blob # oasis-data-manager -tzdata==2023.3 + # sqlalchemy +tzdata==2024.2 # via # celery + # kombu # pandas -urllib3==1.26.15 +urllib3==2.2.3 # via # botocore # requests -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit diff --git a/requirements.txt b/requirements.txt index d0ffa3c45..6a142bb66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,84 +1,80 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements.in # aioredis==1.3.1 # via channels-redis -amqp==5.1.1 +amqp==5.2.0 # via kombu -anytree==2.8.0 +anytree==2.12.1 # via oasislmf argparsetree==0.0.6 # via oasislmf -asgiref==3.6.0 +asgiref==3.8.1 # via # channels # channels-redis # daphne # django -asttokens==2.2.1 +asttokens==2.4.1 # via stack-data -async-timeout==4.0.2 - # via - # aioredis - # redis -attrs==22.2.0 +async-timeout==4.0.3 + # via aioredis +attrs==24.2.0 # via - # automat # hypothesis # jsonschema - # pytest + # referencing # service-identity # twisted -autobahn==23.6.2 +autobahn==24.4.2 # via daphne -automat==22.10.0 +automat==24.8.1 # via twisted -azure-core==1.26.3 +azure-core==1.31.0 # via # -r ./requirements-worker.in # azure-storage-blob -azure-storage-blob==12.15.0 + # django-storages +azure-storage-blob==12.23.1 # via # -r ./requirements-worker.in # django-storages -backcall==0.2.0 - # via ipython backports-tempfile==1.0 # via -r requirements.in backports-weakref==1.0.post1 # via backports-tempfile -beautifulsoup4==4.12.1 +beautifulsoup4==4.12.3 # via webtest -billiard==4.1.0 +billiard==4.2.1 # via # -r ./requirements-worker.in # celery -boto3==1.26.107 +boto3==1.35.44 # via # -r ./requirements-server.in # -r ./requirements-worker.in -botocore==1.29.107 +botocore==1.35.44 # via # boto3 # s3transfer -build==0.10.0 +build==1.2.2.post1 # via pip-tools -cachetools==5.3.0 +cachetools==5.5.0 # via tox -celery==5.3.0 +celery==5.4.0 # via # -r ./requirements-server.in # -r ./requirements-worker.in # django-celery-results -certifi==2023.7.22 +certifi==2024.8.30 # via # pyogrio # pyproj # requests -cffi==1.15.1 +cffi==1.17.1 # via cryptography chainmap==1.0.3 # via @@ -90,47 +86,43 @@ channels==2.4.0 # channels-redis channels-redis==2.4.2 # via -r ./requirements-server.in -chardet==5.1.0 +chardet==5.2.0 # via # oasislmf # ods-tools # tox -charset-normalizer==3.1.0 +charset-normalizer==3.4.0 # via requests -click==8.1.3 +click==8.1.7 # via # celery # click-didyoumean # click-plugins # click-repl # pip-tools -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery -click-repl==0.2.0 +click-repl==0.3.0 # via celery colorama==0.4.6 # via tox -configparser==5.3.0 +configparser==7.1.0 # via -r ./requirements-worker.in -constantly==15.1.0 +constantly==23.10.4 # via twisted coreapi==2.3.3 - # via - # -r ./requirements-server.in - # drf-yasg + # via -r ./requirements-server.in coreschema==0.0.4 - # via - # coreapi - # drf-yasg -coverage[toml]==7.2.2 + # via coreapi +coverage[toml]==7.6.4 # via # -r requirements.in # pytest-cov -cramjam==2.6.2 +cramjam==2.9.0 # via fastparquet -cryptography==42.0.4 +cryptography==43.0.3 # via # autobahn # azure-storage-blob @@ -144,9 +136,9 @@ decorator==5.1.1 # via # ipdb # ipython -distlib==0.3.6 +distlib==0.3.9 # via virtualenv -django==3.2.23 +django==3.2.25 # via # channels # django-celery-results @@ -161,23 +153,23 @@ django==3.2.23 # drf-yasg # model-mommy # mozilla-django-oidc -django-celery-results==2.5.0 +django-celery-results==2.5.1 # via -r ./requirements-server.in -django-cleanup==7.0.0 +django-cleanup==9.0.0 # via -r ./requirements-server.in -django-debug-toolbar==4.0.0 +django-debug-toolbar==4.3.0 # via # -r ./requirements-server.in # -r requirements.in -django-filter==23.1 +django-filter==23.5 # via -r ./requirements-server.in -django-model-utils==4.3.1 +django-model-utils==5.0.0 # via -r ./requirements-server.in django-request-logging==0.7.5 # via -r ./requirements-server.in -django-storages[azure]==1.13.2 +django-storages[azure]==1.14.4 # via -r ./requirements-server.in -django-webtest==1.9.10 +django-webtest==1.9.12 # via -r requirements.in djangorestframework==3.14.0 # via @@ -185,48 +177,44 @@ djangorestframework==3.14.0 # djangorestframework-simplejwt # drf-nested-routers # drf-yasg -djangorestframework-simplejwt==5.2.2 +djangorestframework-simplejwt==5.3.1 # via -r ./requirements-server.in -drf-nested-routers==0.93.4 +drf-nested-routers==0.94.0 # via -r ./requirements-server.in -drf-yasg==1.21.5 +drf-yasg==1.21.8 # via -r ./requirements-server.in -exceptiongroup==1.2.0 - # via - # hypothesis - # pytest -executing==1.2.0 +executing==2.1.0 # via stack-data -fasteners==0.18 +fasteners==0.19 # via # -r ./requirements-worker.in # -r requirements.in -fastparquet==2023.2.0 +fastparquet==2024.5.0 # via # oasis-data-manager # oasislmf -filelock==3.10.7 +filelock==3.16.1 # via # -r ./requirements-worker.in # tox # virtualenv -flake8==6.0.0 +flake8==7.1.1 # via -r requirements.in forex-python==1.8 # via oasislmf -freezegun==1.2.2 +freezegun==1.5.1 # via -r requirements.in -fsspec==2023.3.0 +fsspec==2024.10.0 # via # fastparquet # oasis-data-manager geopandas==1.0.1 # via oasislmf -greenlet==2.0.2 +greenlet==3.1.1 # via sqlalchemy -gunicorn==22.0.0 +gunicorn==23.0.0 # via -r ./requirements-server.in -hiredis==2.2.2 +hiredis==3.0.0 # via aioredis httplib2==0.22.0 # via pyrabbit @@ -234,9 +222,9 @@ hyperlink==21.0.0 # via # autobahn # twisted -hypothesis==6.70.2 +hypothesis==6.115.3 # via -r requirements.in -idna==3.4 +idna==3.10 # via # hyperlink # requests @@ -249,68 +237,70 @@ iniconfig==2.0.0 # via pytest ipdb==0.13.13 # via -r requirements.in -ipython==8.12.0 +ipython==8.28.0 # via ipdb -isodate==0.6.1 +isodate==0.7.2 # via azure-storage-blob itypes==1.2.0 # via coreapi -jedi==0.18.2 +jedi==0.19.1 # via ipython -jinja2==3.1.2 +jinja2==3.1.4 # via coreschema jmespath==1.0.1 # via # boto3 # botocore -joblib==1.2.0 +joblib==1.4.2 # via # -r ./requirements-server.in # -r ./requirements-worker.in # scikit-learn -josepy==1.13.0 +josepy==1.14.0 # via mozilla-django-oidc -jsonpickle==3.0.1 +jsonpickle==3.3.0 # via -r ./requirements-server.in jsonref==1.1.0 # via ods-tools -jsonschema==4.17.3 +jsonschema==4.23.0 # via # -r ./requirements-server.in # ods-tools -kombu==5.3.0 +jsonschema-specifications==2024.10.1 + # via jsonschema +kombu==5.4.2 # via celery -llvmlite==0.41.1 +llvmlite==0.43.0 # via numba -markdown==3.4.3 +markdown==3.7 # via -r ./requirements-server.in -markupsafe==2.1.2 +markupsafe==3.0.2 # via jinja2 -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via ipython mccabe==0.7.0 # via flake8 -mock==5.0.1 +mock==5.1.0 # via -r requirements.in model-mommy==2.0.0 # via -r requirements.in -mozilla-django-oidc==3.0.0 +mozilla-django-oidc==4.0.1 # via -r ./requirements-server.in msgpack==0.6.2 # via # channels-redis # oasislmf -mysqlclient==2.1.1 +mysqlclient==2.2.5 # via -r ./requirements-server.in -natsort==8.3.1 +natsort==8.4.0 # via -r ./requirements-worker.in -numba==0.58.0 +numba==0.60.0 # via # oasislmf # ods-tools numexpr==2.10.1 # via oasislmf -numpy==1.25.2 +numpy==1.26.4 # via # fastparquet # geopandas @@ -323,7 +313,7 @@ numpy==1.25.2 # scikit-learn # scipy # shapely -oasis-data-manager==0.1.1 +oasis-data-manager==0.1.3 # via # oasislmf # ods-tools @@ -333,7 +323,7 @@ ods-tools==3.2.7 # via # -r ./requirements-server.in # oasislmf -packaging==23.0 +packaging==24.1 # via # build # drf-yasg @@ -353,57 +343,55 @@ pandas==2.1.4 # oasis-data-manager # oasislmf # ods-tools -parso==0.8.3 +parso==0.8.4 # via jedi pathlib2==2.3.7.post1 # via # -r ./requirements-server.in # -r ./requirements-worker.in -pexpect==4.8.0 +pexpect==4.9.0 # via ipython -pickleshare==0.7.5 - # via ipython -pip-tools==6.12.3 +pip-tools==7.4.1 # via -r requirements.in -platformdirs==3.2.0 +platformdirs==4.3.6 # via # tox # virtualenv -pluggy==1.0.0 +pluggy==1.5.0 # via # pytest # tox -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.48 # via # click-repl # ipython -psycopg2-binary==2.9.6 +psycopg2-binary==2.9.10 # via # -r ./requirements-server.in # -r ./requirements-worker.in ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data -pyarrow==16.0.0 +pyarrow==17.0.0 # via # -r ./requirements-server.in # oasislmf -pyasn1==0.4.8 +pyasn1==0.6.1 # via # pyasn1-modules # service-identity -pyasn1-modules==0.2.8 +pyasn1-modules==0.4.1 # via service-identity -pycodestyle==2.10.0 +pycodestyle==2.12.1 # via flake8 -pycparser==2.21 +pycparser==2.22 # via cffi -pyflakes==3.0.1 +pyflakes==3.2.0 # via flake8 -pygments==2.14.0 +pygments==2.18.0 # via ipython -pyjwt==2.8.0 +pyjwt==2.9.0 # via djangorestframework-simplejwt pymysql==1.1.1 # via @@ -411,51 +399,57 @@ pymysql==1.1.1 # -r ./requirements-worker.in pyogrio==0.10.0 # via geopandas -pyopenssl==24.0.0 +pyopenssl==24.2.1 # via # -r requirements.in # josepy # twisted -pyparsing==3.0.9 +pyparsing==3.2.0 # via httplib2 -pyproj==3.5.0 +pyproj==3.7.0 # via geopandas -pyproject-api==1.5.1 +pyproject-api==1.8.0 # via tox -pyproject-hooks==1.0.0 - # via build +pyproject-hooks==1.2.0 + # via + # build + # pip-tools pyrabbit==1.1.0 # via -r ./requirements-server.in -pyrsistent==0.19.3 - # via jsonschema -pytest==7.2.2 +pytest==8.3.3 # via # -r ./requirements-worker.in # -r requirements.in # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==5.0.0 # via -r requirements.in -pytest-django==4.5.2 +pytest-django==4.9.0 # via -r requirements.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore # celery # freezegun # pandas -pytz==2023.3 +pytz==2024.2 # via # django # djangorestframework # drf-yasg # oasislmf # pandas -redis==4.5.4 +pyyaml==6.0.2 + # via drf-yasg +redis==5.1.1 # via # -r ./requirements-server.in # -r ./requirements-worker.in -requests==2.28.2 +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 # via # -r requirements.in # azure-core @@ -464,125 +458,111 @@ requests==2.28.2 # mozilla-django-oidc # oasislmf # requests-toolbelt -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via oasislmf -rtree==1.0.1 +rpds-py==0.20.0 + # via + # jsonschema + # referencing +rtree==1.3.0 # via oasislmf -ruamel-yaml==0.17.21 - # via drf-yasg -ruamel-yaml-clib==0.2.8 - # via ruamel-yaml -s3transfer==0.6.0 +s3transfer==0.10.3 # via boto3 -scikit-learn==1.2.2 +scikit-learn==1.5.2 # via oasislmf -scipy==1.10.1 +scipy==1.14.1 # via # oasislmf # scikit-learn -service-identity==21.1.0 +service-identity==24.1.0 # via twisted -shapely==2.0.1 +shapely==2.0.6 # via # geopandas # oasislmf shutilwhich==1.1.0 # via oasislmf -simplejson==3.18.4 +simplejson==3.19.3 # via forex-python six==1.16.0 # via # anytree # asttokens - # automat # azure-core - # click-repl - # isodate # pathlib2 # python-dateutil - # service-identity sortedcontainers==2.4.0 # via hypothesis -soupsieve==2.4 +soupsieve==2.6 # via beautifulsoup4 -sqlalchemy==1.4.47 +sqlalchemy==2.0.36 # via # -r ./requirements-server.in # -r ./requirements-worker.in -sqlparse==0.5.0 +sqlparse==0.5.1 # via # django # django-debug-toolbar -stack-data==0.6.2 +stack-data==0.6.3 # via ipython tabulate==0.9.0 # via oasislmf -tblib==1.7.0 +tblib==3.0.0 # via oasislmf -threadpoolctl==3.1.0 +threadpoolctl==3.5.0 # via scikit-learn -tomli==2.0.1 - # via - # build - # coverage - # incremental - # ipdb - # pyproject-api - # pyproject-hooks - # pytest - # tox -tox==4.4.11 +tox==4.23.0 # via -r requirements.in -tqdm==4.65.0 +tqdm==4.66.5 # via oasislmf -traitlets==5.9.0 +traitlets==5.14.3 # via # ipython # matplotlib-inline -twisted[tls]==24.7.0rc1 +twisted[tls]==24.7.0 # via daphne txaio==23.1.1 # via autobahn -typing==3.7.4.3 - # via oasis-data-manager -typing-extensions==4.5.0 +typing-extensions==4.12.2 # via # azure-core # azure-storage-blob # oasis-data-manager + # sqlalchemy # twisted -tzdata==2023.3 +tzdata==2024.2 # via # celery + # kombu # pandas uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==1.26.15 +urllib3==2.2.3 # via # botocore # requests -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -virtualenv==20.21.0 +virtualenv==20.27.0 # via tox -waitress==2.1.2 +waitress==3.0.0 # via webtest -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit -webob==1.8.7 +webob==1.8.8 # via webtest -webtest==3.0.0 +webtest==3.0.1 # via django-webtest -wheel==0.40.0 +wheel==0.44.0 # via pip-tools -whitenoise==6.4.0 +whitenoise==6.7.0 # via -r ./requirements-server.in -zope-interface==6.0 +zope-interface==7.1.0 # via twisted # The following packages are considered to be unsafe in a requirements file: diff --git a/scripts/update-packages.sh b/scripts/update-packages.sh index 2fec8f422..329382942 100755 --- a/scripts/update-packages.sh +++ b/scripts/update-packages.sh @@ -2,9 +2,9 @@ pkg_list=( 'ods-tools==3.*' - 'oasislmf==1.27.*' - 'pandas==1.*' + 'oasislmf==2.*' 'django==3.*' + 'djangorestframework==3.14.*' 'celery==5.*' ) diff --git a/src/model_execution_worker/distributed_tasks.py b/src/model_execution_worker/distributed_tasks.py index 18e5e391f..a589b7b86 100644 --- a/src/model_execution_worker/distributed_tasks.py +++ b/src/model_execution_worker/distributed_tasks.py @@ -560,7 +560,7 @@ def prepare_keys_file_chunk( **kwargs ): with TemporaryDir() as chunk_target_dir: - chunk_target_dir = os.path.join(chunk_target_dir, f'lookup-{chunk_idx+1}') + chunk_target_dir = os.path.join(chunk_target_dir, f'lookup-{chunk_idx + 1}') Path(chunk_target_dir).mkdir(parents=True, exist_ok=True) _, lookup = OasisLookupFactory.create( @@ -597,7 +597,7 @@ def prepare_keys_file_chunk( # Store chunks params['chunk_keys'] = filestore.put( chunk_target_dir, - filename=f'lookup-{chunk_idx+1}.tar.gz', + filename=f'lookup-{chunk_idx + 1}.tar.gz', subdir=params['storage_subdir'] ) @@ -970,12 +970,12 @@ def generate_losses_chunk(self, params, chunk_idx, num_chunks, analysis_id=None, **params, 'chunk_work_location': filestore.put( chunk_params['ktools_work_dir'], - filename=f'work-{chunk_idx+1}.tar.gz', + filename=f'work-{chunk_idx + 1}.tar.gz', subdir=params['storage_subdir'] ), 'chunk_log_location': filestore.put( chunk_params['ktools_log_dir'], - filename=f'log-{chunk_idx+1}.tar.gz', + filename=f'log-{chunk_idx + 1}.tar.gz', subdir=params['storage_subdir'] ), 'ktools_work_dir': chunk_params['ktools_work_dir'], diff --git a/src/server/oasisapi/permissions/tests/test_group_auth.py b/src/server/oasisapi/permissions/tests/test_group_auth.py index bec615362..b52bdeb72 100644 --- a/src/server/oasisapi/permissions/tests/test_group_auth.py +++ b/src/server/oasisapi/permissions/tests/test_group_auth.py @@ -27,7 +27,7 @@ def test_verify_and_get_groups__no_or_empty(self): user_no_group = create_user([]) self.assertEqual([], verify_and_get_groups(user_no_group, None)) - self.assertEquals([], verify_and_get_groups(user_no_group, [])) + self.assertEqual([], verify_and_get_groups(user_no_group, [])) def test_verify_and_get_groups__invalid_groups(self): From ab2a5f38e31b00c089689f68bba9ab49ac9cbc32 Mon Sep 17 00:00:00 2001 From: awsbuild Date: Thu, 31 Oct 2024 11:36:10 +0000 Subject: [PATCH 2/6] Updated Package Requirements: waitress==3.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a142bb66..7370128cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -550,7 +550,7 @@ vine==5.1.0 # kombu virtualenv==20.27.0 # via tox -waitress==3.0.0 +waitress==3.0.1 # via webtest wcwidth==0.2.13 # via prompt-toolkit From 744e633a3931cd137f92a2f80ceb9ba3a18a18e5 Mon Sep 17 00:00:00 2001 From: sambles Date: Thu, 31 Oct 2024 11:45:33 +0000 Subject: [PATCH 3/6] Fixed pre-analysis exposure files not working in V2 worker runs (#1130) * store files based on source type * pep * Updated Package Requirements: waitress==3.0.1 --------- Co-authored-by: awsbuild --- .../distributed_tasks.py | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/model_execution_worker/distributed_tasks.py b/src/model_execution_worker/distributed_tasks.py index a589b7b86..3ba456d2d 100644 --- a/src/model_execution_worker/distributed_tasks.py +++ b/src/model_execution_worker/distributed_tasks.py @@ -346,12 +346,12 @@ def get_file_ref(kwargs, params, arg_name): file_from_hook = params.get(f'pre_{arg_name}') if not file_from_server: logger.info(f'{arg_name}: (Not loaded)') - return None + return None, None elif file_from_hook: logger.info(f'{arg_name}: {file_from_hook} (pre-analysis-hook)') - return file_from_hook + return file_from_hook, 'pre_' logger.info(f'{arg_name}: {file_from_server} (portfolio)') - return file_from_server + return file_from_server, 'raw_' def log_task_entry(slug, request_id, analysis_id): if slug: @@ -398,32 +398,35 @@ def _prepare_directories(params, analysis_id, run_data_uuid, kwargs): complex_data_files = kwargs.get('complex_data_files') # Load OED file references (filenames or object keys) - loc_file = get_file_ref(kwargs, params, 'loc_file') - acc_file = get_file_ref(kwargs, params, 'acc_file') - info_file = get_file_ref(kwargs, params, 'info_file') - scope_file = get_file_ref(kwargs, params, 'scope_file') + # {filetype}_source => 'raw_' is returned if not modifed + # 'pre_' is returned for files from pre-analysis + + loc_filepath, loc_source = get_file_ref(kwargs, params, 'loc_file') + acc_filepath, acc_source = get_file_ref(kwargs, params, 'acc_file') + info_filepath, info_source = get_file_ref(kwargs, params, 'info_file') + scope_filepath, scope_source = get_file_ref(kwargs, params, 'scope_file') # Prepare 'generate-oasis-files' input files - if loc_file: - loc_extention = "".join(pathlib.Path(loc_file).suffixes) + if loc_filepath: + loc_extention = "".join(pathlib.Path(loc_filepath).suffixes) loc_subdir = params.get('storage_subdir', '') if params.get('pre_loc_file') else '' - params['oed_location_csv'] = os.path.join(params['root_run_dir'], f'location{loc_extention}') - maybe_fetch_file(loc_file, params['oed_location_csv'], loc_subdir) - if acc_file: - acc_extention = "".join(pathlib.Path(acc_file).suffixes) + params['oed_location_csv'] = os.path.join(params['root_run_dir'], f'{loc_source}location{loc_extention}') + maybe_fetch_file(loc_filepath, params['oed_location_csv'], loc_subdir) + if acc_filepath: + acc_extention = "".join(pathlib.Path(acc_filepath).suffixes) acc_subdir = params.get('storage_subdir', '') if params.get('pre_acc_file') else '' - params['oed_accounts_csv'] = os.path.join(params['root_run_dir'], f'account{acc_extention}') - maybe_fetch_file(acc_file, params['oed_accounts_csv'], acc_subdir) - if info_file: - info_extention = "".join(pathlib.Path(info_file).suffixes) + params['oed_accounts_csv'] = os.path.join(params['root_run_dir'], f'{acc_source}account{acc_extention}') + maybe_fetch_file(acc_filepath, params['oed_accounts_csv'], acc_subdir) + if info_filepath: + info_extention = "".join(pathlib.Path(info_filepath).suffixes) info_subdir = params.get('storage_subdir', '') if params.get('pre_info_file') else '' - params['oed_info_csv'] = os.path.join(params['root_run_dir'], f'reinsinfo{info_extention}') - maybe_fetch_file(info_file, params['oed_info_csv'], info_subdir) - if scope_file: - scope_extention = "".join(pathlib.Path(scope_file).suffixes) + params['oed_info_csv'] = os.path.join(params['root_run_dir'], f'{info_source}reinsinfo{info_extention}') + maybe_fetch_file(info_filepath, params['oed_info_csv'], info_subdir) + if scope_filepath: + scope_extention = "".join(pathlib.Path(scope_filepath).suffixes) scope_subdir = params.get('storage_subdir', '') if params.get('pre_scope_file') else '' - params['oed_scope_csv'] = os.path.join(params['root_run_dir'], f'reinsscope{scope_extention}') - maybe_fetch_file(scope_file, params['oed_scope_csv'], scope_subdir) + params['oed_scope_csv'] = os.path.join(params['root_run_dir'], f'{scope_source}reinsscope{scope_extention}') + maybe_fetch_file(scope_filepath, params['oed_scope_csv'], scope_subdir) # Complex model lookup files if settings_file: From 4f99ff460ee5c7ce4ff96c791725a5eb477ed88b Mon Sep 17 00:00:00 2001 From: awsbuild Date: Thu, 31 Oct 2024 16:55:40 +0000 Subject: [PATCH 4/6] Set version 2.3.10 --- VERSION | 2 +- requirements-server.txt | 2 +- requirements-worker.txt | 4 ++-- requirements.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index 5aa7c5232..9fa5f12ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.9 +2.3.10 diff --git a/requirements-server.txt b/requirements-server.txt index a3501439e..0ab699985 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -205,7 +205,7 @@ numpy==2.0.2 # pyarrow oasis-data-manager==0.1.3 # via ods-tools -ods-tools==3.2.7 +ods-tools==3.2.8 # via -r requirements-server.in packaging==24.1 # via diff --git a/requirements-worker.txt b/requirements-worker.txt index ec1d3a234..e0bd33e44 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -134,9 +134,9 @@ oasis-data-manager==0.1.3 # via # oasislmf # ods-tools -oasislmf[extra]==2.3.9 +oasislmf[extra]==2.3.10 # via -r requirements-worker.in -ods-tools==3.2.7 +ods-tools==3.2.8 # via oasislmf packaging==24.1 # via diff --git a/requirements.txt b/requirements.txt index 7370128cd..85df042ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -317,9 +317,9 @@ oasis-data-manager==0.1.3 # via # oasislmf # ods-tools -oasislmf[extra]==2.3.9 +oasislmf[extra]==2.3.10 # via -r ./requirements-worker.in -ods-tools==3.2.7 +ods-tools==3.2.8 # via # -r ./requirements-server.in # oasislmf From 4576ad588d9d23bdf8326b8f9e24d040d969e0d8 Mon Sep 17 00:00:00 2001 From: sambles Date: Thu, 31 Oct 2024 17:09:20 +0000 Subject: [PATCH 5/6] Add cybermodels support in the OasisAPI - V1 Only (#1133) * tmp * retun checks for V2 runs * Only block input generation with a missing loc file on run_mode = V2 * Revert "retun checks for V2 runs" This reverts commit 925474dd592ba9f56a8853b2fc92a45310b4ae95. * fix * Update error in v2 test * Fix testing and validation * update port val messages --- src/model_execution_worker/tasks.py | 2 +- src/server/oasisapi/analyses/models.py | 47 +++++++++++-------- .../oasisapi/analyses/v1_api/serializers.py | 4 +- .../v1_api/tests/test_analysis_model.py | 3 +- .../oasisapi/analyses/v2_api/serializers.py | 4 +- .../v2_api/tests/test_analysis_model.py | 2 +- .../oasisapi/portfolios/v1_api/serializers.py | 4 +- .../portfolios/v1_api/tests/test_portfolio.py | 2 +- .../oasisapi/portfolios/v2_api/serializers.py | 4 +- .../portfolios/v2_api/tests/test_portfolio.py | 2 +- 10 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/model_execution_worker/tasks.py b/src/model_execution_worker/tasks.py index 2bd767d68..76ba83299 100755 --- a/src/model_execution_worker/tasks.py +++ b/src/model_execution_worker/tasks.py @@ -412,7 +412,7 @@ def generate_input(self, with tmp_dir as oasis_files_dir, tmp_input_dir as input_data_dir: # Fetch input files - location_file = filestore.get(loc_file, oasis_files_dir, required=True) + location_file = filestore.get(loc_file, oasis_files_dir) accounts_file = filestore.get(acc_file, oasis_files_dir) ri_info_file = filestore.get(info_file, oasis_files_dir) ri_scope_file = filestore.get(scope_file, oasis_files_dir) diff --git a/src/server/oasisapi/analyses/models.py b/src/server/oasisapi/analyses/models.py index e6ac9624d..83431d50f 100644 --- a/src/server/oasisapi/analyses/models.py +++ b/src/server/oasisapi/analyses/models.py @@ -586,7 +586,19 @@ def generate_inputs(self, initiator, run_mode_override=None): self.status_choices.RUN_CANCELLED, self.status_choices.RUN_ERROR, ] + valid_run_modes = [ + self.run_mode_choices.V1, + self.run_mode_choices.V2, + ] + # check run model + run_mode = run_mode_override if run_mode_override else self.model.run_mode + if run_mode not in valid_run_modes: + raise ValidationError( + {'run_mode': ['run_mode must be [{}]'.format(', '.join(valid_run_modes))]} + ) + + # check everything else errors = {} if self.status not in valid_choices: errors['status'] = ['Analysis status must be one of [{}]'.format(', '.join(valid_choices))] @@ -595,29 +607,26 @@ def generate_inputs(self, initiator, run_mode_override=None): if (self.model.run_mode is None) and (run_mode_override is None): errors['model'] = ['Model pk "{}" - "run_mode" must not be null'.format(self.model.id)] - if not self.portfolio.location_file: - errors['portfolio'] = ['"location_file" must not be null'] - else: - try: - loc_lines = self.portfolio.location_file_len() - except Exception as e: - errors['portfolio'] = [f"Failed to read location file size for chunking: {e}"] - if loc_lines < 1: - errors['portfolio'] = ['"location_file" must at least one row'] + # check for eitehr location or account file if V1 + if run_mode == self.run_mode_choices.V1: + if (not self.portfolio.location_file) and (not self.portfolio.accounts_file): + errors['portfolio'] = ['Either "location_file" or "accounts_file" must not be null for run_mode = V1'] + + # check for location file if V2 + if run_mode == self.run_mode_choices.V2: + if not self.portfolio.location_file: + errors['portfolio'] = ['"location_file" must not be null for run_mode = V2'] + else: + try: + loc_lines = self.portfolio.location_file_len() + except Exception as e: + errors['portfolio'] = [f"Failed to read location file size for chunking: {e}"] + if loc_lines < 1: + errors['portfolio'] = ['"location_file" must at least one row'] if errors: raise ValidationError(errors) - valid_run_modes = [ - self.run_mode_choices.V1, - self.run_mode_choices.V2, - ] - run_mode = run_mode_override if run_mode_override else self.model.run_mode - if run_mode not in valid_run_modes: - raise ValidationError( - {'run_mode': ['run_mode must be [{}]'.format(', '.join(valid_run_modes))]} - ) - self.status = self.status_choices.INPUTS_GENERATION_QUEUED self.lookup_errors_file = None self.lookup_success_file = None diff --git a/src/server/oasisapi/analyses/v1_api/serializers.py b/src/server/oasisapi/analyses/v1_api/serializers.py index b5c2b7aca..debd08220 100644 --- a/src/server/oasisapi/analyses/v1_api/serializers.py +++ b/src/server/oasisapi/analyses/v1_api/serializers.py @@ -210,8 +210,8 @@ def validate(self, attrs): # Check that portfilio has a location file if attrs.get('portfolio'): - if not attrs['portfolio'].location_file: - raise ValidationError({'portfolio': '"location_file" must not be null'}) + if (not attrs['portfolio'].location_file) and (not attrs['portfolio'].accounts_file): + raise ValidationError({'portfolio': 'either "location_file" or "accounts_file" must not be null'}) # check that model isn't soft-deleted if attrs.get('model'): diff --git a/src/server/oasisapi/analyses/v1_api/tests/test_analysis_model.py b/src/server/oasisapi/analyses/v1_api/tests/test_analysis_model.py index fd1d8ead2..fe45dec5f 100644 --- a/src/server/oasisapi/analyses/v1_api/tests/test_analysis_model.py +++ b/src/server/oasisapi/analyses/v1_api/tests/test_analysis_model.py @@ -226,7 +226,8 @@ def test_portfolio_has_no_location_file___validation_error_is_raised_revoke_is_n with self.assertRaises(ValidationError) as ex: analysis.generate_inputs(initiator, run_mode_override='V1') - self.assertEqual({'portfolio': ['"location_file" must not be null']}, ex.exception.detail) + self.assertEqual( + {'portfolio': ['Either "location_file" or "accounts_file" must not be null for run_mode = V1']}, ex.exception.detail) self.assertEqual(Analysis.status_choices.NEW, analysis.status) self.assertFalse(res_factory.revoke_called) diff --git a/src/server/oasisapi/analyses/v2_api/serializers.py b/src/server/oasisapi/analyses/v2_api/serializers.py index d4c67f516..0e50fc776 100644 --- a/src/server/oasisapi/analyses/v2_api/serializers.py +++ b/src/server/oasisapi/analyses/v2_api/serializers.py @@ -403,8 +403,8 @@ def validate(self, attrs): except ValidationError: raise ValidationError({'portfolio': 'You are not allowed to use this portfolio'}) - if not attrs['portfolio'].location_file: - raise ValidationError({'portfolio': '"location_file" must not be null'}) + if (not attrs['portfolio'].location_file) and (not attrs['portfolio'].accounts_file): + raise ValidationError({'portfolio': 'either "location_file" or "accounts_file" must not be null'}) # check that model isn't soft-deleted if attrs.get('model'): diff --git a/src/server/oasisapi/analyses/v2_api/tests/test_analysis_model.py b/src/server/oasisapi/analyses/v2_api/tests/test_analysis_model.py index 31d123700..4784ccda6 100644 --- a/src/server/oasisapi/analyses/v2_api/tests/test_analysis_model.py +++ b/src/server/oasisapi/analyses/v2_api/tests/test_analysis_model.py @@ -394,7 +394,7 @@ def test_portfolio_has_no_location_file___validation_error_is_raised_revoke_is_n with self.assertRaises(ValidationError) as ex: analysis.generate_inputs(initiator, run_mode_override='V2') - self.assertEqual({'portfolio': ['"location_file" must not be null']}, ex.exception.detail) + self.assertEqual({'portfolio': ['"location_file" must not be null for run_mode = V2']}, ex.exception.detail) self.assertEqual(Analysis.status_choices.NEW, analysis.status) self.assertFalse(res_factory.revoke_called) diff --git a/src/server/oasisapi/portfolios/v1_api/serializers.py b/src/server/oasisapi/portfolios/v1_api/serializers.py index aeaf20b01..6dd87fe65 100644 --- a/src/server/oasisapi/portfolios/v1_api/serializers.py +++ b/src/server/oasisapi/portfolios/v1_api/serializers.py @@ -387,8 +387,8 @@ def __init__(self, portfolio=None, *args, **kwargs): def validate(self, attrs): attrs['portfolio'] = self.portfolio - if not self.portfolio.location_file: - raise ValidationError({'portfolio': '"location_file" must not be null'}) + if (not self.portfolio.location_file) and (not self.portfolio.accounts_file): + raise ValidationError({'portfolio': 'either "location_file" or "accounts_file" must not be null'}) return attrs diff --git a/src/server/oasisapi/portfolios/v1_api/tests/test_portfolio.py b/src/server/oasisapi/portfolios/v1_api/tests/test_portfolio.py index 015cbb28e..1a8daab6d 100644 --- a/src/server/oasisapi/portfolios/v1_api/tests/test_portfolio.py +++ b/src/server/oasisapi/portfolios/v1_api/tests/test_portfolio.py @@ -195,7 +195,7 @@ def test_portfolio_does_not_have_location_file_set___response_is_400(self): ) self.assertEqual(400, response.status_code) - self.assertIn('"location_file" must not be null', response.json['portfolio']) + self.assertIn('either "location_file" or "accounts_file" must not be null', response.json['portfolio']) def test_model_is_not_provided___response_is_400(self): user = fake_user() diff --git a/src/server/oasisapi/portfolios/v2_api/serializers.py b/src/server/oasisapi/portfolios/v2_api/serializers.py index 2a8b69b88..00961e36b 100644 --- a/src/server/oasisapi/portfolios/v2_api/serializers.py +++ b/src/server/oasisapi/portfolios/v2_api/serializers.py @@ -411,8 +411,8 @@ def __init__(self, portfolio=None, *args, **kwargs): def validate(self, attrs): attrs['portfolio'] = self.portfolio - if not self.portfolio.location_file: - raise ValidationError({'portfolio': '"location_file" must not be null'}) + if (not self.portfolio.location_file) and (not self.portfolio.accounts_file): + raise ValidationError({'portfolio': 'either "location_file" or "accounts_file" must not be null'}) # Validate and update groups parameter validate_and_update_groups(self.partial, self.context.get('request').user, attrs) diff --git a/src/server/oasisapi/portfolios/v2_api/tests/test_portfolio.py b/src/server/oasisapi/portfolios/v2_api/tests/test_portfolio.py index eaa680f4f..74eb0acee 100644 --- a/src/server/oasisapi/portfolios/v2_api/tests/test_portfolio.py +++ b/src/server/oasisapi/portfolios/v2_api/tests/test_portfolio.py @@ -243,7 +243,7 @@ def test_portfolio_does_not_have_location_file_set___response_is_400(self): ) self.assertEqual(400, response.status_code) - self.assertIn('"location_file" must not be null', response.json['portfolio']) + self.assertIn('either "location_file" or "accounts_file" must not be null', response.json['portfolio']) def test_model_is_not_provided___response_is_400(self): user = fake_user() From 4f0682b5aed32df4f30027f308a474d19cb2ef4c Mon Sep 17 00:00:00 2001 From: awsbuild Date: Thu, 31 Oct 2024 17:57:20 +0000 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15108085b..8e9d314d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ OasisPlatform Changelog ======================= +`2.3.10`_ + --------- +* [#1121](https://github.com/OasisLMF/OasisPlatform/pull/1121) - Release 2.3.9 +* [#1122](https://github.com/OasisLMF/OasisPlatform/pull/1122) - Update python packages for 2.3.9 +* [#1124](https://github.com/OasisLMF/OasisPlatform/pull/1124) - Update docker base images and python packages +* [#1123](https://github.com/OasisLMF/OasisPlatform/pull/1130) - Generate oasis files is not using the pre-analysis adjusted location file for V2 runs +* [#1125](https://github.com/OasisLMF/OasisPlatform/pull/1133) - Support Cyber models on OasisPlatform +.. _`2.3.10`: https://github.com/OasisLMF/OasisPlatform/compare/2.3.9...2.3.10 + `2.3.9`_ --------- * [#1105](https://github.com/OasisLMF/OasisPlatform/pull/1105) - Release 2.3.8