From a8fc39ab2965383f5402441a4a776e9f66692bea Mon Sep 17 00:00:00 2001 From: Serhii Koropets Date: Tue, 23 Jan 2024 20:18:49 +0200 Subject: [PATCH 1/3] Update dependencies 2024.01 --- requirements/full_requirements.txt | 164 ++++++++++++++--------------- 1 file changed, 81 insertions(+), 83 deletions(-) diff --git a/requirements/full_requirements.txt b/requirements/full_requirements.txt index 7b2e2114f..d3ce099fd 100644 --- a/requirements/full_requirements.txt +++ b/requirements/full_requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --no-emit-index-url --output-file=full_requirements.txt mlflow_requirements.in postgres_requirements.in requirements.in # -absl-py==1.4.0 +absl-py==2.1.0 # via # tensorboard # tensorflow @@ -12,9 +12,9 @@ adal==1.2.7 # via # azureml-core # msrestazure -alembic==1.12.0 +alembic==1.13.1 # via mlflow -argcomplete==2.1.2 +argcomplete==3.2.1 # via # azureml-core # knack @@ -37,9 +37,9 @@ azure-core==1.29.1 # msrest azure-graphrbac==0.61.1 # via azureml-core -azure-mgmt-authorization==3.0.0 +azure-mgmt-authorization==4.0.0 # via azureml-core -azure-mgmt-containerregistry==10.1.0 +azure-mgmt-containerregistry==10.3.0 # via azureml-core azure-mgmt-core==1.4.0 # via @@ -49,39 +49,39 @@ azure-mgmt-core==1.4.0 # azure-mgmt-network # azure-mgmt-resource # azure-mgmt-storage -azure-mgmt-keyvault==10.2.3 +azure-mgmt-keyvault==10.3.0 # via azureml-core -azure-mgmt-network==21.0.1 +azure-mgmt-network==25.1.0 # via azureml-core -azure-mgmt-resource==22.0.0 +azure-mgmt-resource==23.0.1 # via azureml-core -azure-mgmt-storage==21.0.0 +azure-mgmt-storage==21.1.0 # via azureml-core -azureml-core==1.53.0 +azureml-core==1.54.0.post1 # via -r mlflow_requirements.in backports-tempfile==1.0 # via azureml-core backports-weakref==1.0.post1 # via backports-tempfile -bcrypt==4.0.1 +bcrypt==4.1.2 # via paramiko -blinker==1.6.2 +blinker==1.7.0 # via flask -cachetools==5.3.1 +cachetools==5.3.2 # via # google-auth # gordo-core catboost==1.2.2 # via -r requirements.in -certifi==2023.7.22 +certifi==2023.11.17 # via # msrest # requests -cffi==1.15.1 +cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -89,26 +89,25 @@ click==8.1.7 # flask # gordo-client # mlflow -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via mlflow contextlib2==21.6.0 # via azureml-core -contourpy==1.1.0 +contourpy==1.2.0 # via matplotlib -cryptography==41.0.5 +cryptography==41.0.7 # via # adal - # azureml-core # msal # paramiko # pyjwt # pyopenssl # secretstorage -cycler==0.11.0 +cycler==0.12.1 # via matplotlib -databricks-cli==0.17.7 +databricks-cli==0.18.0 # via mlflow -dataclasses-json==0.5.14 +dataclasses-json==0.6.3 # via -r requirements.in dictdiffer==0.9.0 # via -r requirements.in @@ -124,15 +123,15 @@ flask==2.3.3 # mlflow flatbuffers==23.5.26 # via tensorflow -fonttools==4.42.1 +fonttools==4.47.2 # via matplotlib gast==0.4.0 # via tensorflow -gitdb==4.0.10 +gitdb==4.0.11 # via gitpython -gitpython==3.1.37 +gitpython==3.1.41 # via mlflow -google-auth==2.22.0 +google-auth==2.26.2 # via # google-auth-oauthlib # tensorboard @@ -140,15 +139,15 @@ google-auth-oauthlib==1.0.0 # via tensorboard google-pasta==0.2.0 # via tensorflow -gordo-client==6.2.7 +gordo-client==6.2.8 # via -r requirements.in gordo-core==0.3.5 # via gordo-client graphviz==0.20.1 # via catboost -greenlet==2.0.2 +greenlet==3.0.3 # via sqlalchemy -grpcio==1.57.0 +grpcio==1.60.0 # via # tensorboard # tensorflow @@ -156,29 +155,34 @@ gunicorn==20.1.0 # via # -r requirements.in # mlflow -h5py==3.9.0 +h5py==3.10.0 # via # -r requirements.in # tensorflow humanfriendly==10.0 # via azureml-core -idna==3.4 +idna==3.6 # via requests -importlib-metadata==6.8.0 +importlib-metadata==7.0.1 # via mlflow influxdb==5.3.1 # via gordo-core isodate==0.6.1 # via + # azure-mgmt-authorization + # azure-mgmt-containerregistry # azure-mgmt-keyvault + # azure-mgmt-network + # azure-mgmt-resource + # azure-mgmt-storage # msrest itsdangerous==2.1.2 # via flask -jax==0.4.14 +jax==0.4.23 # via tensorflow jeepney==0.8.0 # via secretstorage -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements.in # flask @@ -195,47 +199,42 @@ keras==2.12.0 # via tensorflow kiwisolver==1.4.5 # via matplotlib -knack==0.10.1 +knack==0.11.0 # via azureml-core libclang==16.0.6 # via tensorflow -mako==1.2.4 +mako==1.3.1 # via alembic -markdown==3.4.4 +markdown==3.5.2 # via # mlflow # tensorboard -markupsafe==2.1.3 +markupsafe==2.1.4 # via # jinja2 # mako # werkzeug -marshmallow==3.20.1 +marshmallow==3.20.2 # via dataclasses-json -matplotlib==3.7.2 +matplotlib==3.8.2 # via # catboost # mlflow -ml-dtypes==0.2.0 +ml-dtypes==0.3.2 # via jax mlflow==2.9.2 # via -r mlflow_requirements.in -msal==1.24.1 +msal==1.26.0 # via # azureml-core # msal-extensions msal-extensions==1.0.0 # via azureml-core -msgpack==1.0.5 +msgpack==1.0.7 # via influxdb msrest==0.7.1 # via # azure-graphrbac - # azure-mgmt-authorization - # azure-mgmt-containerregistry - # azure-mgmt-network - # azure-mgmt-resource - # azure-mgmt-storage # azureml-core # msrestazure msrestazure==0.6.4 @@ -246,7 +245,7 @@ mypy-extensions==1.0.0 # via typing-inspect ndg-httpsclient==0.5.1 # via azureml-core -numexpr==2.8.5 +numexpr==2.8.8 # via gordo-core numpy==1.24.3 # via @@ -280,6 +279,7 @@ packaging==21.3 # -r requirements.in # azureml-core # docker + # knack # marshmallow # matplotlib # mlflow @@ -292,34 +292,34 @@ pandas==1.5.3 # gordo-core # mlflow # xarray -paramiko==3.3.1 +paramiko==3.4.0 # via azureml-core -pathspec==0.11.2 +pathspec==0.12.1 # via azureml-core -peewee==3.16.3 +peewee==3.17.0 # via -r postgres_requirements.in -pillow==10.0.1 +pillow==10.2.0 # via matplotlib pkginfo==1.9.6 # via azureml-core -plotly==5.16.1 +plotly==5.18.0 # via catboost -portalocker==2.7.0 +portalocker==2.8.2 # via msal-extensions -prometheus-client==0.17.1 +prometheus-client==0.19.0 # via -r requirements.in -protobuf==4.24.2 +protobuf==4.25.2 # via # mlflow # tensorboard # tensorflow -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # via -r postgres_requirements.in -pyarrow==14.0.1 +pyarrow==14.0.2 # via # gordo-core # mlflow -pyasn1==0.5.0 +pyasn1==0.5.1 # via # ndg-httpsclient # pyasn1-modules @@ -328,9 +328,9 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pydantic==1.10.13 +pydantic==1.10.14 # via gordo-client -pygments==2.16.1 +pygments==2.17.2 # via knack pyjwt[crypto]==2.8.0 # via @@ -340,11 +340,11 @@ pyjwt[crypto]==2.8.0 # msal pynacl==1.5.0 # via paramiko -pyopenssl==23.2.0 +pyopenssl==23.3.0 # via # azureml-core # ndg-httpsclient -pyparsing==3.0.9 +pyparsing==3.1.1 # via # matplotlib # packaging @@ -358,7 +358,7 @@ python-dateutil==2.8.2 # influxdb # matplotlib # pandas -pytz==2023.3 +pytz==2023.3.post1 # via # azureml-core # influxdb @@ -391,11 +391,11 @@ requests-oauthlib==1.3.1 # msrest rsa==4.9 # via google-auth -scikit-learn==1.3.0 +scikit-learn==1.4.0 # via # gordo-core # mlflow -scipy==1.11.2 +scipy==1.12.0 # via # catboost # gordo-core @@ -404,7 +404,7 @@ scipy==1.11.2 # scikit-learn secretstorage==3.3.3 # via azureml-core -simplejson==3.19.1 +simplejson==3.19.2 # via # -r requirements.in # gordo-client @@ -414,7 +414,6 @@ six==1.16.0 # azure-core # catboost # databricks-cli - # google-auth # google-pasta # influxdb # isodate @@ -422,9 +421,9 @@ six==1.16.0 # python-dateutil # querystring-parser # tensorflow -smmap==5.0.0 +smmap==5.0.1 # via gitdb -sqlalchemy==2.0.20 +sqlalchemy==2.0.24 # via # alembic # mlflow @@ -438,15 +437,15 @@ tenacity==8.2.3 # via plotly tensorboard==2.12.3 # via tensorflow -tensorboard-data-server==0.7.1 +tensorboard-data-server==0.7.2 # via tensorboard tensorflow==2.12.1 # via -r requirements.in tensorflow-estimator==2.12.0 # via tensorflow -tensorflow-io-gcs-filesystem==0.33.0 +tensorflow-io-gcs-filesystem==0.35.0 # via tensorflow -termcolor==2.3.0 +termcolor==2.4.0 # via tensorflow threadpoolctl==3.2.0 # via scikit-learn @@ -460,20 +459,19 @@ typing-extensions==4.5.0 # typing-inspect typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.18 +urllib3==2.1.0 # via # azureml-core # databricks-cli # docker - # google-auth # requests -websocket-client==1.6.2 +websocket-client==1.7.0 # via docker -werkzeug==2.3.8 +werkzeug==3.0.1 # via # flask # tensorboard -wheel==0.41.2 +wheel==0.42.0 # via # astunparse # tensorboard @@ -481,9 +479,9 @@ wrapt==1.14.1 # via # gordo-client # tensorflow -xarray==2023.8.0 +xarray==2023.12.0 # via gordo-core -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: From 5e442d02b9b1c5fb4c1fc216afe328768d9588da Mon Sep 17 00:00:00 2001 From: Serhii Koropets Date: Tue, 23 Jan 2024 20:21:12 +0200 Subject: [PATCH 2/3] Update test dependencies --- requirements/test_requirements.txt | 156 +++++++++++++---------------- 1 file changed, 70 insertions(+), 86 deletions(-) diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 05ce9211c..74e614df9 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --no-emit-index-url --output-file=test_requirements.txt test_requirements.in # -anyio==4.0.0 +anyio==4.2.0 # via jupyter-server argon2-cffi==23.1.0 # via @@ -13,32 +13,30 @@ argon2-cffi==23.1.0 # notebook argon2-cffi-bindings==21.2.0 # via argon2-cffi -arrow==1.2.3 +arrow==1.3.0 # via isoduration -asttokens==2.3.0 +asttokens==2.4.1 # via stack-data -attrs==23.1.0 +attrs==23.2.0 # via # jsonschema # pytest-mypy # referencing -backcall==0.2.0 - # via ipython -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via nbconvert black==22.12.0 # via -r test_requirements.in -bleach==6.0.0 +bleach==6.1.0 # via nbconvert -certifi==2023.7.22 +certifi==2023.11.17 # via # -c full_requirements.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via # -c full_requirements.txt # argon2-cffi-bindings -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via # -c full_requirements.txt # requests @@ -46,11 +44,11 @@ click==8.1.7 # via # -c full_requirements.txt # black -comm==0.1.4 +comm==0.2.1 # via ipykernel -coverage[toml]==7.3.0 +coverage[toml]==7.4.0 # via pytest-cov -debugpy==1.6.7.post1 +debugpy==1.8.0 # via ipykernel decorator==5.1.1 # via ipython @@ -64,22 +62,17 @@ entrypoints==0.4 # via # -c full_requirements.txt # jupyter-client -exceptiongroup==1.1.3 - # via - # anyio - # ipython - # pytest execnet==2.0.2 # via pytest-xdist -executing==1.2.0 +executing==2.0.1 # via stack-data -fastjsonschema==2.18.0 +fastjsonschema==2.19.1 # via nbformat -filelock==3.12.2 +filelock==3.13.1 # via pytest-mypy fqdn==1.5.1 # via jsonschema -idna==3.4 +idna==3.6 # via # -c full_requirements.txt # anyio @@ -87,11 +80,11 @@ idna==3.4 # requests iniconfig==2.0.0 # via pytest -ipykernel==6.25.1 +ipykernel==6.29.0 # via # nbclassic # notebook -ipython==8.15.0 +ipython==8.20.0 # via ipykernel ipython-genutils==0.2.0 # via @@ -99,9 +92,9 @@ ipython-genutils==0.2.0 # notebook isoduration==20.11.0 # via jsonschema -jedi==0.19.0 +jedi==0.19.1 # via ipython -jinja2==3.1.2 +jinja2==3.1.3 # via # -c full_requirements.txt # jupyter-server @@ -110,11 +103,11 @@ jinja2==3.1.2 # notebook jsonpointer==2.4 # via jsonschema -jsonschema[format-nongpl]==4.19.0 +jsonschema[format-nongpl]==4.21.1 # via # jupyter-events # nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==7.4.9 # via @@ -123,7 +116,7 @@ jupyter-client==7.4.9 # nbclassic # nbclient # notebook -jupyter-core==5.3.1 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -133,17 +126,17 @@ jupyter-core==5.3.1 # nbconvert # nbformat # notebook -jupyter-events==0.7.0 +jupyter-events==0.9.0 # via jupyter-server -jupyter-server==2.7.3 +jupyter-server==2.12.5 # via # nbclassic # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert -markupsafe==2.1.3 +markupsafe==2.1.4 # via # -c full_requirements.txt # jinja2 @@ -152,11 +145,11 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -mistune==3.0.1 +mistune==3.0.2 # via nbconvert mock==5.1.0 # via -r test_requirements.in -mypy==1.5.1 +mypy==1.8.0 # via pytest-mypy mypy-extensions==1.0.0 # via @@ -165,9 +158,9 @@ mypy-extensions==1.0.0 # mypy nbclassic==1.0.0 # via notebook -nbclient==0.8.0 +nbclient==0.9.0 # via nbconvert -nbconvert==7.8.0 +nbconvert==7.14.2 # via # -r test_requirements.in # jupyter-server @@ -180,17 +173,17 @@ nbformat==5.9.2 # nbclient # nbconvert # notebook -nest-asyncio==1.5.7 +nest-asyncio==1.6.0 # via # ipykernel # jupyter-client # nbclassic # notebook -notebook==6.5.5 +notebook==6.5.6 # via -r test_requirements.in notebook-shim==0.2.3 # via nbclassic -overrides==7.4.0 +overrides==7.6.0 # via jupyter-server packaging==21.3 # via @@ -200,33 +193,31 @@ packaging==21.3 # jupyter-server # nbconvert # pytest -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi -pathspec==0.11.2 +pathspec==0.12.1 # via # -c full_requirements.txt # black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 +pexpect==4.9.0 # via ipython -platformdirs==3.10.0 +platformdirs==4.1.0 # via # black # jupyter-core pluggy==1.3.0 # via pytest -prometheus-client==0.17.1 +prometheus-client==0.19.0 # via # -c full_requirements.txt # jupyter-server # nbclassic # notebook -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.43 # via ipython -psutil==5.9.5 +psutil==5.9.8 # via ipykernel ptyprocess==0.7.0 # via @@ -240,18 +231,18 @@ pycparser==2.21 # via # -c full_requirements.txt # cffi -pyflakes==3.1.0 +pyflakes==3.2.0 # via pytest-flakes -pygments==2.16.1 +pygments==2.17.2 # via # -c full_requirements.txt # ipython # nbconvert -pyparsing==3.0.9 +pyparsing==3.1.1 # via # -c full_requirements.txt # packaging -pytest==7.4.1 +pytest==7.4.4 # via # -r test_requirements.in # pytest-benchmark @@ -267,13 +258,13 @@ pytest-cov==4.1.0 # via -r test_requirements.in pytest-flakes==4.0.5 # via -r test_requirements.in -pytest-mock==3.11.1 +pytest-mock==3.12.0 # via -r test_requirements.in pytest-mypy==0.10.3 # via -r test_requirements.in -pytest-timeout==2.1.0 +pytest-timeout==2.2.0 # via -r test_requirements.in -pytest-xdist==3.3.1 +pytest-xdist==3.5.0 # via -r test_requirements.in python-dateutil==2.8.2 # via @@ -294,7 +285,7 @@ pyzmq==24.0.1 # jupyter-server # nbclassic # notebook -referencing==0.30.2 +referencing==0.32.1 # via # jsonschema # jsonschema-specifications @@ -304,7 +295,7 @@ requests==2.31.0 # -c full_requirements.txt # docker # responses -responses==0.23.3 +responses==0.24.1 # via -r test_requirements.in rfc3339-validator==0.1.4 # via @@ -314,7 +305,7 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rpds-py==0.10.0 +rpds-py==0.17.1 # via # jsonschema # referencing @@ -334,9 +325,9 @@ sniffio==1.3.0 # via anyio soupsieve==2.5 # via beautifulsoup4 -stack-data==0.6.2 +stack-data==0.6.3 # via ipython -terminado==0.17.1 +terminado==0.18.0 # via # jupyter-server # jupyter-server-terminals @@ -344,13 +335,7 @@ terminado==0.17.1 # notebook tinycss2==1.2.1 # via nbconvert -tomli==2.0.1 - # via - # black - # coverage - # mypy - # pytest -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client @@ -358,7 +343,7 @@ tornado==6.3.3 # nbclassic # notebook # terminado -traitlets==5.9.0 +traitlets==5.14.1 # via # comm # ipykernel @@ -373,37 +358,36 @@ traitlets==5.9.0 # nbconvert # nbformat # notebook -types-mock==5.1.0.2 - # via -r test_requirements.in -types-python-dateutil==2.8.19.14 - # via -r test_requirements.in -types-pytz==2023.3.0.1 +types-mock==5.1.0.20240106 # via -r test_requirements.in -types-pyyaml==6.0.12.11 +types-python-dateutil==2.8.19.20240106 # via # -r test_requirements.in - # responses -types-requests==2.31.0.2 + # arrow +types-pytz==2023.3.1.1 + # via -r test_requirements.in +types-pyyaml==6.0.12.12 + # via -r test_requirements.in +types-requests==2.31.0.20240106 # via -r test_requirements.in -types-setuptools==68.1.0.1 +types-setuptools==69.0.0.20240115 # via -r test_requirements.in types-simplejson==3.19.0.2 # via -r test_requirements.in -types-urllib3==1.26.25.14 - # via types-requests typing-extensions==4.5.0 # via # -c full_requirements.txt # mypy uri-template==1.3.0 # via jsonschema -urllib3==1.26.18 +urllib3==2.1.0 # via # -c full_requirements.txt # docker # requests # responses -wcwidth==0.2.6 + # types-requests +wcwidth==0.2.13 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -411,7 +395,7 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.2 +websocket-client==1.7.0 # via # -c full_requirements.txt # docker From c05475c358e60613cbf99f478934729e6fb86f9e Mon Sep 17 00:00:00 2001 From: Serhii Koropets Date: Wed, 24 Jan 2024 09:51:05 +0200 Subject: [PATCH 3/3] Fix test_get_metrics_dict_scaler unit-test --- tests/gordo/builder/test_builder.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/gordo/builder/test_builder.py b/tests/gordo/builder/test_builder.py index b96198436..1b752663a 100644 --- a/tests/gordo/builder/test_builder.py +++ b/tests/gordo/builder/test_builder.py @@ -9,7 +9,7 @@ import pandas as pd import sklearn.compose import sklearn.ensemble -from sklearn.base import BaseEstimator +from sklearn.base import BaseEstimator, RegressorMixin from sklearn.multioutput import MultiOutputRegressor from sklearn.linear_model import LinearRegression from sklearn import metrics @@ -63,6 +63,17 @@ def machine_check(machine: Machine, check_history): ) +class CustomRegressor(BaseEstimator, RegressorMixin): + def __init__(self, multiplier): + self.multiplier = multiplier + + def fit(self, X, y): + return self + + def predict(self, X): + return X * self.multiplier + + @pytest.mark.parametrize("scaler", [None, "sklearn.preprocessing.MinMaxScaler"]) def test_get_metrics_dict_scaler(scaler): metrics_list = [sklearn.metrics.mean_squared_error] @@ -74,12 +85,10 @@ def test_get_metrics_dict_scaler(scaler): metrics_dict = ModelBuilder.build_metrics_dict(metrics_list, y, scaler=scaler) metric_func = metrics_dict["mean-squared-error"] - mock_model1 = MagicMock() - mock_model1.predict = lambda _y: _y * [0.8, 1] - mse_feature_one_wrong = metric_func(mock_model1, y, y) - mock_model2 = MagicMock() - mock_model2.predict = lambda _y: _y * [1, 0.8] - mse_feature_two_wrong = metric_func(mock_model2, y, y) + classifier1 = CustomRegressor(np.array([0.8, 1])) + mse_feature_one_wrong = metric_func(classifier1, y, y) + classifier2 = CustomRegressor(np.array([1, 0.8])) + mse_feature_two_wrong = metric_func(classifier2, y, y) if scaler: assert np.isclose(mse_feature_one_wrong, mse_feature_two_wrong)