From 57b9ee6cccf16353457fa0fab243e4d1feadb4fe Mon Sep 17 00:00:00 2001 From: Will Holtz Date: Mon, 9 Oct 2023 21:29:59 -0700 Subject: [PATCH 1/2] feat: upgrade notebooks to run under python 3.11.6 --- docker/Dockerfile.shifter | 8 +- docker/requirements.in | 34 ++ docker/requirements.txt | 623 +++++++++++++++++++++++++++++------- docker/top.requirements.txt | 34 -- 4 files changed, 542 insertions(+), 157 deletions(-) create mode 100644 docker/requirements.in delete mode 100644 docker/top.requirements.txt diff --git a/docker/Dockerfile.shifter b/docker/Dockerfile.shifter index 46d2d2b1..4c6fc1ae 100644 --- a/docker/Dockerfile.shifter +++ b/docker/Dockerfile.shifter @@ -1,10 +1,10 @@ -FROM python:3.8-slim-bullseye +FROM python:3.11-slim-bookworm RUN apt-get update \ && apt-get install --yes --no-install-recommends \ - "libxrender1=1:0.9.10-1" \ - "libxext6=2:1.3.3-1.1" \ - "git=1:2.30.2-1+deb11u2" \ + "libxrender1=1:0.9.10-1.1" \ + "libxext6=2:1.3.4-1+b1" \ + "git=1:2.39.2-1.1" \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt /requirements.txt diff --git a/docker/requirements.in b/docker/requirements.in new file mode 100644 index 00000000..61d3405b --- /dev/null +++ b/docker/requirements.in @@ -0,0 +1,34 @@ +alembic +banal +black +colorama +dataset +dill +gspread +h5py +humanize +ipykernel +ipympl==0.8.6 +ipysheet +ipywidgets +jupyter-client +jupyterlab +matchms +matplotlib +numpy +oauth2client +pandas +papermill +pip +pubchempy +pydantic +pymysql +pymzml +pyyaml +rdkit +scikit-learn +scipy +sqlalchemy +tables +tabulate +xlsxwriter diff --git a/docker/requirements.txt b/docker/requirements.txt index 204fedd7..592381b8 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,146 +1,531 @@ -alembic==1.7.7 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-build-isolation requirements.in +# +alembic==1.12.0 + # via + # -r requirements.in + # dataset +annotated-types==0.6.0 + # via pydantic ansiwrap==0.8.4 -anyio==3.6.1 -argon2-cffi==21.3.0 + # via papermill +anyio==4.0.0 + # via jupyter-server +appnope==0.1.3 + # via + # ipykernel + # ipython +argon2-cffi==23.1.0 + # via jupyter-server argon2-cffi-bindings==21.2.0 -asttokens==2.0.8 -attrs==22.1.0 -Babel==2.10.3 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +asttokens==2.4.0 + # via stack-data +async-lru==2.0.4 + # via jupyterlab +attrs==23.1.0 + # via + # jsonschema + # referencing +babel==2.13.0 + # via jupyterlab-server backcall==0.2.0 + # via ipython banal==1.0.6 -beautifulsoup4==4.11.1 -black==22.3.0 -bleach==5.0.1 -cachetools==5.2.0 -certifi==2022.6.15 -cffi==1.15.1 -charset-normalizer==2.1.1 -click==8.1.3 -colorama==0.4.4 -cycler==0.11.0 -dataset==1.5.2 -debugpy==1.6.3 + # via + # -r requirements.in + # dataset +beautifulsoup4==4.12.2 + # via nbconvert +black==23.9.1 + # via -r requirements.in +bleach==6.1.0 + # via nbconvert +blosc2==2.2.9 + # via tables +cachetools==5.3.1 + # via google-auth +certifi==2023.7.22 + # via requests +cffi==1.16.0 + # via argon2-cffi-bindings +charset-normalizer==3.3.0 + # via requests +click==8.1.7 + # via + # black + # papermill +colorama==0.4.6 + # via -r requirements.in +comm==0.1.4 + # via + # ipykernel + # ipywidgets +contourpy==1.1.1 + # via matplotlib +cycler==0.12.1 + # via matplotlib +dataset==1.6.2 + # via -r requirements.in +debugpy==1.8.0 + # via ipykernel decorator==5.1.1 + # via ipython defusedxml==0.7.1 -Deprecated==1.2.13 -dill==0.3.5.1 + # via nbconvert +deprecated==1.2.14 + # via matchms +dill==0.3.7 + # via -r requirements.in entrypoints==0.4 -executing==1.0.0 -fastjsonschema==2.16.1 -fonttools==4.37.1 -google-auth==2.11.0 -google-auth-oauthlib==0.5.2 -greenlet==1.1.3 -gspread==5.4.0 -h5py==3.7.0 -httplib2==0.20.4 -humanize==4.1.0 -idna==3.3 -importlib-metadata==4.12.0 -importlib-resources==5.9.0 -ipykernel==6.13.0 + # via papermill +executing==2.0.0 + # via stack-data +fastjsonschema==2.18.1 + # via nbformat +fonttools==4.43.1 + # via matplotlib +fqdn==1.5.1 + # via jsonschema +google-auth==2.23.3 + # via + # google-auth-oauthlib + # gspread +google-auth-oauthlib==1.1.0 + # via gspread +greenlet==3.0.0 + # via sqlalchemy +gspread==5.11.3 + # via -r requirements.in +h5py==3.10.0 + # via -r requirements.in +httplib2==0.22.0 + # via oauth2client +humanize==4.8.0 + # via -r requirements.in +idna==3.4 + # via + # anyio + # jsonschema + # requests +ipykernel==6.25.2 + # via + # -r requirements.in + # jupyterlab ipympl==0.8.6 -ipysheet==0.5.0 -ipython==8.5.0 + # via -r requirements.in +ipysheet==0.7.0 + # via -r requirements.in +ipython==8.16.1 + # via + # ipykernel + # ipympl + # ipywidgets ipython-genutils==0.2.0 -ipywidgets==7.7.0 -jedi==0.18.1 -Jinja2==3.1.2 -joblib==1.1.0 -json5==0.9.10 -jsonschema==4.15.0 -jupyter-core==4.11.1 -jupyter-server==1.18.1 -jupyter_client==7.3.5 -jupyterlab==3.4.2 + # via + # ipympl + # ipywidgets +ipywidgets==7.8.1 + # via + # -r requirements.in + # ipympl + # ipysheet +isoduration==20.11.0 + # via jsonschema +jedi==0.19.1 + # via ipython +jinja2==3.1.2 + # via + # jupyter-server + # jupyterlab + # jupyterlab-server + # nbconvert +joblib==1.3.2 + # via scikit-learn +json5==0.9.14 + # via jupyterlab-server +jsonpointer==2.4 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.7.1 + # via jsonschema +jsonschema[format-nongpl]==4.19.1 + # via jsonschema +jupyter-client==8.3.1 + # via + # -r requirements.in + # ipykernel + # jupyter-server + # nbclient +jupyter-core==5.3.2 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat +jupyter-events==0.7.0 + # via jupyter-server +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.7.3 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.4.4 + # via jupyter-server +jupyterlab==4.0.6 + # via + # -r requirements.in + # notebook jupyterlab-pygments==0.2.2 -jupyterlab-widgets==3.0.3 -jupyterlab_server==2.15.1 -kiwisolver==1.4.4 -llvmlite==0.39.1 -lxml==4.9.1 -Mako==1.2.2 -MarkupSafe==2.1.1 -matchms==0.15.0 -matplotlib==3.5.2 + # via nbconvert +jupyterlab-server==2.25.0 + # via + # jupyterlab + # notebook +jupyterlab-widgets==1.1.7 + # via ipywidgets +kiwisolver==1.4.5 + # via matplotlib +llvmlite==0.40.1 + # via numba +lxml==4.9.3 + # via matchms +mako==1.2.4 + # via alembic +markupsafe==2.1.3 + # via + # jinja2 + # mako + # nbconvert +matchms==0.22.0 + # via -r requirements.in +matplotlib==3.8.0 + # via + # -r requirements.in + # ipympl + # matchms matplotlib-inline==0.1.6 -mistune==2.0.4 -mypy-extensions==0.4.3 -nbclassic==0.4.3 -nbclient==0.6.7 -nbconvert==7.0.0 -nbformat==5.4.0 -nest-asyncio==1.5.5 -networkx==2.8.6 -notebook==6.4.12 -notebook-shim==0.1.0 -numba==0.56.2 -numexpr==2.8.3 -numpy==1.22.4 + # via + # ipykernel + # ipython +mistune==3.0.2 + # via nbconvert +msgpack==1.0.7 + # via blosc2 +mypy-extensions==1.0.0 + # via black +nbclient==0.8.0 + # via + # nbconvert + # papermill +nbconvert==7.9.2 + # via jupyter-server +nbformat==5.9.2 + # via + # jupyter-server + # nbclient + # nbconvert + # papermill +ndindex==1.7 + # via blosc2 +nest-asyncio==1.5.8 + # via ipykernel +networkx==3.1 + # via matchms +notebook==7.0.4 + # via widgetsnbextension +notebook-shim==0.2.3 + # via + # jupyterlab + # notebook +numba==0.57.1 + # via + # matchms + # sparsestack +numexpr==2.8.7 + # via tables +numpy==1.24.4 + # via + # -r requirements.in + # blosc2 + # contourpy + # h5py + # ipympl + # matchms + # matplotlib + # numba + # numexpr + # pandas + # pymzml + # rdkit + # scikit-learn + # scipy + # sparsestack + # tables oauth2client==4.1.3 -oauthlib==3.2.0 -packaging==21.3 -pandas==1.4.2 + # via -r requirements.in +oauthlib==3.2.2 + # via requests-oauthlib +overrides==7.4.0 + # via jupyter-server +packaging==23.2 + # via + # black + # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server + # matplotlib + # nbconvert + # tables +pandas==2.1.1 + # via + # -r requirements.in + # matchms pandocfilters==1.5.0 -papermill==2.3.4 + # via nbconvert +papermill==2.4.0 + # via -r requirements.in parso==0.8.3 -pathspec==0.10.1 + # via jedi +pathspec==0.11.2 + # via black pexpect==4.8.0 + # via ipython pickleshare==0.7.5 + # via ipython pickydict==0.4.0 -Pillow==9.2.0 -pkgutil_resolve_name==1.3.10 -platformdirs==2.5.2 -prometheus-client==0.14.1 -prompt-toolkit==3.0.31 -psutil==5.9.2 + # via matchms +pillow==10.0.1 + # via + # ipympl + # matchms + # matplotlib + # rdkit +platformdirs==3.11.0 + # via + # black + # jupyter-core +prometheus-client==0.17.1 + # via jupyter-server +prompt-toolkit==3.0.39 + # via ipython +psutil==5.9.5 + # via ipykernel ptyprocess==0.7.0 -PubChemPy==1.0.4 + # via + # pexpect + # terminado +pubchempy==1.0.4 + # via -r requirements.in pure-eval==0.2.2 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 + # via stack-data +py-cpuinfo==9.0.0 + # via + # blosc2 + # tables +pyasn1==0.5.0 + # via + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via + # google-auth + # oauth2client pycparser==2.21 -pydantic==1.9.1 -Pygments==2.13.0 -PyMySQL==1.0.2 -pymzml==2.5.1 -pyparsing==3.0.9 -pyrsistent==0.18.1 -pyteomics==4.5.5 + # via cffi +pydantic==2.4.2 + # via -r requirements.in +pydantic-core==2.10.1 + # via pydantic +pygments==2.16.1 + # via + # ipython + # nbconvert +pymysql==1.1.0 + # via -r requirements.in +pymzml==2.5.2 + # via -r requirements.in +pyparsing==3.1.1 + # via + # httplib2 + # matplotlib +pyteomics==4.6.2 + # via matchms python-dateutil==2.8.2 -pytz==2022.2.1 -PyYAML==6.0 -pyzmq==23.2.1 -rdkit-pypi==2022.3.2.1 -regex==2022.8.17 -requests==2.28.1 + # via + # arrow + # jupyter-client + # matplotlib + # pandas +python-json-logger==2.0.7 + # via jupyter-events +pytz==2023.3.post1 + # via pandas +pyyaml==6.0.1 + # via + # -r requirements.in + # jupyter-events + # matchms + # papermill +pyzmq==25.1.1 + # via + # ipykernel + # jupyter-client + # jupyter-server +rdkit==2023.3.3 + # via + # -r requirements.in + # matchms +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events +regex==2023.10.3 + # via pymzml +requests==2.31.0 + # via + # jupyterlab-server + # matchms + # papermill + # requests-oauthlib requests-oauthlib==1.3.1 + # via google-auth-oauthlib +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rpds-py==0.10.4 + # via + # jsonschema + # referencing rsa==4.9 -scikit-learn==1.1.1 -scipy==1.8.1 -Send2Trash==1.8.0 + # via + # google-auth + # oauth2client +scikit-learn==1.3.1 + # via -r requirements.in +scipy==1.10.1 + # via + # -r requirements.in + # matchms + # scikit-learn + # sparsestack +send2trash==1.8.2 + # via jupyter-server six==1.16.0 + # via + # asttokens + # bleach + # oauth2client + # python-dateutil + # rfc3339-validator sniffio==1.3.0 -soupsieve==2.3.2.post1 -SQLAlchemy==1.4.36 -stack-data==0.5.0 -tables==3.7.0 -tabulate==0.8.9 -tenacity==8.0.1 -terminado==0.15.0 + # via anyio +soupsieve==2.5 + # via beautifulsoup4 +sparsestack==0.4.1 + # via matchms +sqlalchemy==1.4.49 + # via + # -r requirements.in + # alembic + # dataset +stack-data==0.6.3 + # via ipython +tables==3.9.1 + # via -r requirements.in +tabulate==0.9.0 + # via -r requirements.in +tenacity==8.2.3 + # via papermill +terminado==0.17.1 + # via + # jupyter-server + # jupyter-server-terminals textwrap3==0.9.2 -threadpoolctl==3.1.0 -tinycss2==1.1.1 -tomli==2.0.1 -tornado==6.2 -tqdm==4.64.1 -traitlets==5.3.0 -typing_extensions==4.3.0 -urllib3==1.26.12 -wcwidth==0.2.5 + # via ansiwrap +threadpoolctl==3.2.0 + # via scikit-learn +tinycss2==1.2.1 + # via nbconvert +tornado==6.3.3 + # via + # ipykernel + # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado +tqdm==4.66.1 + # via + # matchms + # papermill +traitlets==5.11.2 + # via + # comm + # ipykernel + # ipympl + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +types-python-dateutil==2.8.19.14 + # via arrow +typing-extensions==4.8.0 + # via + # alembic + # pydantic + # pydantic-core +tzdata==2023.3 + # via pandas +uri-template==1.3.0 + # via jsonschema +urllib3==2.0.6 + # via requests +wcwidth==0.2.8 + # via prompt-toolkit +webcolors==1.13 + # via jsonschema webencodings==0.5.1 -websocket-client==1.4.1 -widgetsnbextension==3.6.1 -wrapt==1.14.1 -XlsxWriter==3.0.3 -zipp==3.8.1 + # via + # bleach + # tinycss2 +websocket-client==1.6.4 + # via jupyter-server +widgetsnbextension==3.6.6 + # via ipywidgets +wrapt==1.15.0 + # via deprecated +xlsxwriter==3.1.7 + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/docker/top.requirements.txt b/docker/top.requirements.txt deleted file mode 100644 index 24056b19..00000000 --- a/docker/top.requirements.txt +++ /dev/null @@ -1,34 +0,0 @@ -alembic==1.7.7 -banal==1.0.6 -black==22.3.0 -colorama==0.4.4 -dataset==1.5.2 -dill==0.3.5.1 -gspread==5.4.0 -h5py==3.7.0 -humanize==4.1.0 -ipykernel==6.13.0 -ipympl==0.8.6 -ipysheet==0.5.0 -ipywidgets==7.7.0 -jupyter-client==7.4.9 -jupyterlab==3.4.2 -matchms==0.15.0 -matplotlib==3.5.2 -numpy==1.22.4 -oauth2client==4.1.3 -pandas==1.4.2 -papermill==2.3.4 -pip==22.1.2 -pubchempy==1.0.4 -pydantic==1.9.1 -pymysql==1.0.2 -pymzml==2.5.1 -pyyaml==6.0 -rdkit-pypi==2022.3.2.1 -scikit-learn==1.1.1 -scipy==1.8.1 -sqlalchemy==1.4.36 -tables==3.7.0 -tabulate==0.8.9 -xlsxwriter==3.0.3 From 4adc22349796c00e407d30ba2689574d21508276 Mon Sep 17 00:00:00 2001 From: Will Holtz Date: Tue, 10 Oct 2023 14:40:16 -0700 Subject: [PATCH 2/2] chore: WIP get unit_test passing with py311 deps --- docker/requirements.in | 2 +- docker/requirements.txt | 2 +- metatlas/tools/cheminfo.py | 21 ++++++++++++--------- metatlas/tools/config.py | 2 +- noxfile.py | 17 ++++++----------- tests/unit/test_cheminfo.py | 11 ++++++----- 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/docker/requirements.in b/docker/requirements.in index 61d3405b..04fbf4a5 100644 --- a/docker/requirements.in +++ b/docker/requirements.in @@ -8,7 +8,7 @@ gspread h5py humanize ipykernel -ipympl==0.8.6 +ipympl==0.9.3 ipysheet ipywidgets jupyter-client diff --git a/docker/requirements.txt b/docker/requirements.txt index 592381b8..c6458033 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -117,7 +117,7 @@ ipykernel==6.25.2 # via # -r requirements.in # jupyterlab -ipympl==0.8.6 +ipympl==0.9.3 # via -r requirements.in ipysheet==0.7.0 # via -r requirements.in diff --git a/metatlas/tools/cheminfo.py b/metatlas/tools/cheminfo.py index 146b9c8a..618c62f8 100644 --- a/metatlas/tools/cheminfo.py +++ b/metatlas/tools/cheminfo.py @@ -7,7 +7,8 @@ import matchms import numpy as np -from matchms.filtering.load_adducts import load_adducts_dict +from matchms.filtering.filter_utils.load_known_adducts import ( + load_known_adducts) from rdkit import Chem from metatlas.interfaces.compounds import structure_cleaning as cleaning @@ -28,21 +29,23 @@ def get_parent_mass(precursor_mz: float, adduct: str) -> float: @functools.lru_cache def get_precursor_mz(parent_mass: float, adduct: str) -> float: """For an input molecule with parent_mass that generates adduct, return the resutling precursor_mz""" - adducts = load_adducts_dict() - if adduct not in adducts: + adducts = load_known_adducts() + if adduct not in adducts["adduct"].values: raise KeyError("Adduct '%s' is not supported") - multiplier = adducts[adduct]["mass_multiplier"] - correction_mass = adducts[adduct]["correction_mass"] + row = adducts.loc[adducts["adduct"] == adduct].iloc[0] + multiplier = row["mass_multiplier"] + correction_mass = row["correction_mass"] return (parent_mass + correction_mass) / multiplier @functools.lru_cache def is_positive_mode(adduct: str) -> bool: """Returns True if the MS mode for an adduct is positive""" - adducts = load_adducts_dict() - if adduct not in adducts: + adducts = load_known_adducts() + if adduct not in adducts["adduct"].values: raise KeyError("Adduct '%s' is not supported") - return adducts[adduct]["ionmode"] == "positive" + row = adducts.loc[adducts["adduct"] == adduct].iloc[0] + return row["ionmode"] == "positive" @functools.lru_cache @@ -149,7 +152,7 @@ def valid_adduct(value: str) -> bool: True if the value is an adduct listed supported by the matchms package This is not a comprehensive list, so it will return False for some uncommon adducts """ - adducts = load_adducts_dict() + adducts = load_known_adducts() return value in adducts diff --git a/metatlas/tools/config.py b/metatlas/tools/config.py index b843f963..64a58d2f 100644 --- a/metatlas/tools/config.py +++ b/metatlas/tools/config.py @@ -77,7 +77,7 @@ class BaseNotebookParameters(BaseModel): project_directory: Path = Path().home() / "metabolomics_data" max_cpus: int = 4 log_level: str = "INFO" - resolve_msms_matches_by = "distance" + resolve_msms_matches_by: str = "distance" def update(self, override_parameters: Dict) -> None: """update all parameters with any non-None values in override_parameters""" diff --git a/noxfile.py b/noxfile.py index a665f029..ab58bb4d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,18 +6,18 @@ import nox -py_versions = ["3.8", "3.9"] +py_versions = ["3.11"] nox.options.sessions = [ "flake8", "black", - "pylint-3.8", - "mypy-3.8", - "unit_tests-3.8", + "pylint-3.11", + "mypy-3.11", + "unit_tests-3.11", "flake8_nb", "black_nb", - "pylint_nb-3.8", - "system_tests-3.8", + "pylint_nb-3.11", + "system_tests-3.11", "update_git_hooks", ] @@ -69,15 +69,10 @@ ] pytest_deps = [ - "attrs==22.1.0", "coverage==7.0.5", "iniconfig==2.0.0", - "numpy==1.22.4", - "packaging==21.3", - "pandas==1.4.2", "pluggy==1.0.0", "py==1.11.0", - "pyparsing==3.0.9", "pytest==7.2.1", "pytest-cov==3.0.0", "pytest-mock==3.10.0", diff --git a/tests/unit/test_cheminfo.py b/tests/unit/test_cheminfo.py index 72e8a5b1..139894de 100644 --- a/tests/unit/test_cheminfo.py +++ b/tests/unit/test_cheminfo.py @@ -1,6 +1,7 @@ # pylint: disable=missing-function-docstring, missing-module-docstring, line-too-long -from matchms.filtering.load_adducts import load_adducts_dict +from matchms.filtering.filter_utils.load_known_adducts import ( + load_known_adducts) from rdkit import Chem from metatlas.tools import cheminfo @@ -11,11 +12,11 @@ def test_get_parent_mass01(): - adducts = load_adducts_dict() + adducts = load_known_adducts() original_parent = 100 - for name in adducts: - pre = cheminfo.get_precursor_mz(original_parent, name) - parent = cheminfo.get_parent_mass(pre, name) + for _, row in adducts.iterrows(): + pre = cheminfo.get_precursor_mz(original_parent, row["adduct"]) + parent = cheminfo.get_parent_mass(pre, row["adduct"]) assert abs(original_parent - parent) < 1e-7