Merge pull request #331 from biorack/oo_mads3

Object oriented wrapper for metatlas_dataset

.github/workflows/ci.yaml

@@ -10,13 +10,11 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
-      - name: Cache conda environment for unit tests
+      - name: Cache nox environment for unit tests
         uses: actions/cache@v2
         with:
           path: ~/work/metatlas/metatlas/.nox
           key: ${{ runner.os }}-nox
-      - name: Install miniconda
-        uses: conda-incubator/setup-miniconda@v2
       - name: Setup nox
         uses: excitedleigh/[email protected]
       - name: Run unit tests

.gitignore

@@ -40,6 +40,7 @@ htmlcov/
 .nox/
 .tox/
 .coverage
+.coverage.*
 .cache
 nosetests.xml
 coverage.xml
@@ -69,6 +70,7 @@ target/
 *.db
 *-shm
 *-wal
+dumps/
 
 # kbase installation directories
 bootstrap/
@@ -89,3 +91,10 @@ scratch/
 
 # editor swap files
 .*.swp
+.vscode/
+
+# pyenv
+.python-version
+
+# slurm
+slurm-*.out

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v3.4.0
+  rev: v4.0.1
   hooks:
   - id: check-added-large-files
   - id: check-ast

DEVELOP.rst

@@ -6,11 +6,10 @@ Setup
 
 1. Install Python 3.8+ (`pyenv <https://github.com/pyenv/pyenv>`_ and `pyenv intstaller <https://github.com/pyenv/pyenv-installer>`_ can help here)
 2. Install `Pip <https://pip.pypa.io/en/stable/installing/>`_
-3. Install a conda environment manager. You can get miniconda `here <https://docs.conda.io/en/latest/miniconda.html>`_.
-4. Install `Docker <https://docs.docker.com/get-docker/>`_.
-5. Install Nox with :code:`pip install --user --upgrade nox`
-6. :code:`git clone https://github.com/biorack/metatlas.git`
-7. Install git pre-commit hooks with :code:`cd metatlas && nox -s install_git_hooks`
+3. Install `Docker <https://docs.docker.com/get-docker/>`_.
+4. Install Nox with :code:`pip install --user --upgrade nox`
+5. :code:`git clone https://github.com/biorack/metatlas.git`
+6. Install git pre-commit hooks with :code:`cd metatlas && nox -s install_git_hooks`
 
 Local Development
 #################

conda/create_conda_env.sh

@@ -0,0 +1,37 @@
+#!/bin/bash --login
+set -ef -o pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+REPO_DIR="$(dirname "$SCRIPT_DIR")"
+NAME="metatlas-targeted-$(date --iso-8601)"
+BASE_DIR="/global/common/software/m2650"
+ENV_DIR="${BASE_DIR}/${NAME}"
+ENV_FILE="${SCRIPT_DIR}/env.yaml"
+
+echo "name: $NAME
+channels:
+- conda-forge
+dependencies:
+- python=3.8
+- pip
+- pip:" > "$ENV_FILE"
+awk '{ print  "  - " $0 }' requirements.txt >> "$ENV_FILE"
+
+conda env create \
+  --prefix "$ENV_DIR" \
+  --file "$ENV_FILE"
+rm "$ENV_FILE"
+
+cat >"${REPO_DIR}/notebooks/kernels/metatlas-targeted.kernel.json" <<EOL
+{
+ "argv": [
+  "${ENV_DIR}/bin/python",
+  "-m",
+  "ipykernel_launcher",
+  "-f",
+  "{connection_file}"
+ ],
+ "display_name": "Metatlas Targeted",
+ "language": "python"
+}
+EOL

conda/requirements.txt

@@ -0,0 +1 @@
+../docker/requirements.txt

docker/Dockerfile.ci01

@@ -0,0 +1,50 @@
+FROM python:3.8-slim-bullseye
+
+# https://portal.nersc.gov/cfs/m2650/metatlas/test_data
+# serves from /global/cfs/cdirs/m2650/www/metatlas/test_data
+ARG BASE_DATA_URL=https://portal.nersc.gov/cfs/m2650/metatlas/test_data/ci01
+ARG REFS_DIR=/global/project/projectdirs/metatlas/projects/spectral_libraries
+ARG H5_DIR=/project/projectdirs/metatlas/raw_data/akuftin/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583
+
+ENV METATLAS_LOCAL=True
+
+EXPOSE 8888
+
+ADD https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 /usr/local/bin/jq
+
+RUN chmod +x /usr/local/bin/jq
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install --quiet -r requirements.txt
+
+RUN mkdir -p /io /src /work $REFS_DIR $H5_DIR
+
+ADD $BASE_DATA_URL/msms_refs_v3.tab $REFS_DIR/
+
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_49_Cone-S1_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run34.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_57_Cone-S2_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run40.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_65_Cone-S3_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run16.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_73_Cone-S4_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run31.h5 $H5_DIR/
+
+# also get the mzML files, as these are used in matchms within add_msms_refs
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_49_Cone-S1_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run34.mzML $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_57_Cone-S2_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run40.mzML $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_65_Cone-S3_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run16.mzML $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_73_Cone-S4_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run31.mzML $H5_DIR/
+
+
+ADD $BASE_DATA_URL/meta_atlas.sqlite3 /work/root_workspace.db
+
+RUN mkdir -p /root/.local/share/jupyter/kernels/metatlas-targeted
+COPY kernel.json /root/.local/share/jupyter/kernels/metatlas-targeted/kernel.json
+
+WORKDIR /work
+
+RUN apt-get update && apt-get install -y \
+	libxrender1 \
+	nodejs \
+	npm && \
+    rm -rf /var/lib/apt/lists/*
+
+CMD ["/usr/local/bin/jupyter", "lab", "--ip=0.0.0.0", "--allow-root", "--ServerApp.token=''", "--ServerApp.root_dir=/"]

docker/Dockerfile.ci02

@@ -0,0 +1,46 @@
+FROM python:3.8-slim-buster
+
+# https://portal.nersc.gov/cfs/m2650/metatlas/test_data
+# serves from /global/cfs/cdirs/m2650/www/metatlas/test_data
+ARG BASE_DATA_URL=https://portal.nersc.gov/cfs/m2650/metatlas/test_data/ci02
+ARG REFS_DIR=/global/project/projectdirs/metatlas/projects/spectral_libraries
+ARG H5_DIR=/project/projectdirs/metatlas/raw_data/akuftin/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583
+
+ENV METATLAS_LOCAL=True
+
+EXPOSE 8888
+
+RUN apt-get update && apt-get install -y libxrender1 && \
+    rm -rf /var/lib/apt/lists/*
+
+ADD https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 /usr/local/bin/jq
+RUN chmod +x /usr/local/bin/jq
+
+RUN mkdir -p /io /src /work $REFS_DIR $H5_DIR
+ADD $BASE_DATA_URL/msms_refs_v3.tab $REFS_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_FPS_MS1_0_QC_Post_Rg70to1050-CE102040--QC_Run307.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_FPS_MS1_0_QC_Pre_Rg70to1050-CE102040--QC_Run6.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_NEG_MSMS_0_QC_Post_Rg70to1050-CE102040--QC_Run309.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_NEG_MSMS_0_QC_Pre_Rg70to1050-CE102040--QC_Run8.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_0_QC_Post_Rg70to1050-CE102040--QC_Run308.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_0_QC_Pre_Rg70to1050-CE102040--QC_Run7.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_NEG_MSMS_53_Cone-S1_5_Rg70to1050-CE102040-QlobataAkingi-S1_Run188.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_NEG_MSMS_57_Cone-S2_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run41.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_NEG_MSMS_58_Cone-S2_2_Rg70to1050-CE102040-QlobataAkingi-S1_Run56.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_NEG_MSMS_59_Cone-S2_3_Rg70to1050-CE102040-QlobataAkingi-S1_Run87.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_49_Cone-S1_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run34.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_53_Cone-S1_5_Rg70to1050-CE102040-QlobataAkingi-S1_Run187.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_54_Cone-S1_6_Rg70to1050-CE102040-QlobataAkingi-S1_Run221.h5 $H5_DIR/
+ADD $BASE_DATA_URL/20201106_JGI-AK_PS-KM_505892_OakGall_final_QE-HF_HILICZ_USHXG01583_POS_MSMS_57_Cone-S2_1_Rg70to1050-CE102040-QlobataAkingi-S1_Run40.h5 $H5_DIR/
+
+COPY requirements.txt /requirements.txt
+RUN pip install --quiet -r requirements.txt
+
+ADD $BASE_DATA_URL/meta_atlas_rt_predict.sqlite3 /work/root_workspace.db
+
+RUN mkdir -p /root/.local/share/jupyter/kernels/metatlas-targeted
+COPY kernel.json /root/.local/share/jupyter/kernels/metatlas-targeted/kernel.json
+
+WORKDIR /work
+
+CMD ["/usr/local/bin/jupyter", "nbclassic", "--ip=0.0.0.0", "--allow-root", "--ServerApp.token=''", "--ServerApp.root_dir=/"]

docker/build.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# catch some common errors, terminate if a command returns non-zero exit code
+set -euf -o pipefail
+
+SPIN_USER="$USER"
+PROJECT="metatlas_test"
+REGISTRY="registry.spin.nersc.gov"
+DOCKER="docker"
+DOCKERFILE=""
+
+IMAGE_NAME=""
+TAG=""
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    -d|--docker) DOCKER="$2"; shift ;;
+    -f|--dockerfile) DOCKERFILE="$2"; shift ;;
+    -i|--image) IMAGE_NAME="$2"; shift ;;
+    -r|--registry) REGISTRY="$2"; shift ;;
+    -p|--project) PROJECT="$2"; shift ;;
+    -t|--tag) TAG="$2"; shift ;;
+    -u|--user) SPIN_USER="$2"; shift ;;
+    -h|--help)
+        echo -e "$0 [options]"
+        echo ""
+        echo "   -h, --help              show this command reference"
+	echo "   -d, --docker            name of docker command (default ${DOCKER})"
+	echo "   -f, --dockerfile        name of Dockerfile"
+        echo "   -i, --image string      name of image to build"
+        echo "   -p, --project string    project name within the registry (default ${PROJECT})"
+        echo "   -r, --registry string   FQDN of container registry to push to"
+        echo "                           use 'NONE' to not push (default ${REGISTRY})"
+        echo "   -t, --tag string        image tag"
+        echo "   -u, --user string       username for ${REGISTRY} (default ${USER})"
+        exit 0
+        ;;
+    *)echo "Unknown parameter passed: $1"; exit 1 ;;
+  esac
+  shift
+done
+
+if [[ "$IMAGE_NAME" == "" ]]; then
+  >&2 echo "ERROR: no Dockerfile value given"
+  exit 10
+fi
+
+if [[ "$DOCKERFILE" == "" ]]; then
+  >&2 echo "ERROR: no Dockerfile value given"
+  exit 10
+fi
+
+if [[ "$TAG" == "" ]]; then
+  >&2 echo "ERROR: no tag value given"
+  exit 9
+fi
+
+SHORT_TAG="${IMAGE_NAME}:${TAG}"
+LONG_TAG="${REGISTRY}/${PROJECT}/${SHORT_TAG}"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+if [[ ! -r "${DOCKERFILE}" ]]; then
+  >&2 echo "ERROR: Could not find readable Dockerfile at ${DOCKERFILE}."
+  exit 1
+fi
+
+${DOCKER} image build --tag "${SHORT_TAG}" --file "${DOCKERFILE}" "$SCRIPT_DIR"
+
+if [[ "$REGISTRY" != "NONE" ]]; then
+  if [[ $(uname -s) == "Darwin" ]]; then
+    # no readlink on macOS...
+    if [[ $(basename $(which ${DOCKER})) == 'podman' ]]; then
+      PUSH_FLAGS="--format=docker"
+    fi
+  else
+    if [[ $(basename $(readlink -f $(which ${DOCKER}))) == 'podman' ]]; then
+      PUSH_FLAGS="--format=docker"
+    fi
+  fi
+  ${DOCKER} image tag "${SHORT_TAG}" "${LONG_TAG}"
+  ${DOCKER} image push ${PUSH_FLAGS:-} "${LONG_TAG}"
+  TAG="${LONG_TAG}"
+else
+  TAG="${SHORT_TAG}"
+fi

docker/build_x.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -euf -o pipefail
+
+SPIN_USER="$USER"
+PROJECT="metatlas_test"
+REGISTRY="registry.spin.nersc.gov"
+DOCKER="docker"
+TAG=""
+ID=""
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    -d|--docker) DOCKER="$2"; shift ;;
+    -i|--id) ID="$2"; shift ;;
+    -p|--project) PROJECT="$2"; shift ;;
+    -r|--registry) REGISTRY="$2"; shift ;;
+    -t|--tag) TAG="$2"; shift ;;
+    -u|--user) SPIN_USER="$2"; shift ;;
+    -h|--help)
+        echo -e "$0 [options]"
+        echo ""
+        echo "   -h, --help              show this command reference"
+	echo "   -d, --docker            name of docker command (default ${DOCKER})"
+	echo "   -i, --id                image id"
+        echo "   -p, --project string    project name within the registry (default ${PROJECT})"
+        echo "   -r, --registry string   FQDN of container registry to push to"
+        echo "                           use 'NONE' to not push (default ${REGISTRY})"
+        echo "   -t, --tag string        image tag"
+        echo "   -u, --user string       username for ${REGISTRY} (default ${USER})"
+        exit 0
+        ;;
+    *)echo "Unknown parameter passed: $1"; exit 1 ;;
+  esac
+  shift
+done
+
+if [[ "$TAG" == "" ]]; then
+  >&2 echo "ERROR: no tag value given"
+  exit 1
+fi
+
+if [[ "$ID" == "" ]]; then
+  >&2 echo "ERROR: no id value given"
+  exit 2
+fi
+
+DOCKERFILE="Dockerfile.$ID"
+IMAGE_NAME="metatlas_$ID"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+"${SCRIPT_DIR}/build.sh" --image "$IMAGE_NAME" --tag "$TAG" --docker "$DOCKER" --project "$PROJECT" --registry "$REGISTRY" --user "$USER" --dockerfile "${SCRIPT_DIR}/$DOCKERFILE"