script to build and test docker images

.github/workflows/gpu.yml

@@ -33,12 +33,10 @@
 #       run: |
 #         pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html
 #
-#     - name: Install package and test dependencies
+#     - name: Install dependencies
 #       run: |
-#         python -m pip install --upgrade "pip<24.0"
-#         pip install -r requirements.txt
-#         pip install pytest pytest-cov
-#         python setup.py develop
+#         python -m pip install --upgrade pip
+#         pip install -e .
 #
 #     - name: Setup test data
 #       run: |

.github/workflows/main.yml

@@ -48,12 +48,10 @@ jobs:
       run: |
         pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
 
-    - name: Install package and test dependencies
+    - name: Install dependencies
       run: |
-        python -m pip install --upgrade "pip<24.0"
-        pip install -r requirements.txt
-        pip install pytest pytest-cov
-        python setup.py develop
+        python -m pip install --upgrade pip
+        pip install .
 
     - name: Setup test data
       run: |

docker/Dockerfile-full

@@ -31,10 +31,9 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39
 
 # install
 RUN conda install python=3.7 -y
-RUN pip install setuptools --upgrade && pip install --upgrade "pip<24.0"
-RUN pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html
-
+RUN pip install --upgrade pip
+# Install deepethogram with dev dependencies
 ADD . /app/deepethogram
 WORKDIR /app/deepethogram
 ENV DEG_VERSION='full'
-RUN pip install -e .
+RUN pip install -e ".[dev]"

docker/Dockerfile-gui

@@ -35,13 +35,13 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39
 
 # install
 RUN conda install python=3.7 -y
-RUN pip install setuptools --upgrade && pip install --upgrade pip
+RUN pip install --upgrade pip
 
 # TODO: REFACTOR CODE SO IT'S POSSIBLE TO RUN GUI WITHOUT TORCH
 RUN conda install pytorch cpuonly -c pytorch
 
-# # needed for pandas for some reason
+# Install deepethogram with dev dependencies
 ADD . /app/deepethogram
 WORKDIR /app/deepethogram
 ENV DEG_VERSION='gui'
-RUN pip install -e .
+RUN pip install -e ".[dev]"

docker/Dockerfile-headless

@@ -33,11 +33,11 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39
 
 # install
 RUN conda install python=3.7 -y
-RUN pip install setuptools --upgrade && pip install --upgrade pip
-RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
+RUN pip install --upgrade pip
+# RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
 
-# # needed for pandas for some reason
+# Install deepethogram with dev dependencies
 ADD . /app/deepethogram
 WORKDIR /app/deepethogram
 ENV DEG_VERSION='headless'
-RUN pip install -e .
+RUN pip install -e ".[dev]"

docker/build_and_test.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+set -e  # Exit on any error
+
+# Colors for output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+BLUE='\033[0;34m'
+YELLOW='\033[1;33m'
+
+# Function to print section headers
+print_header() {
+    echo -e "\n${BLUE}=== $1 ===${NC}\n"
+}
+
+# Function to echo command before running
+echo_run() {
+    echo -e "${YELLOW}Running: $@${NC}"
+    "$@"
+}
+
+# Function to build an image
+build_image() {
+    local type=$1
+    print_header "Building $type image"
+    echo_run docker build -t deepethogram:$type -f docker/Dockerfile-$type .
+}
+
+# Function to verify GPU in container
+verify_gpu() {
+    local gpu_flag=$1
+    local type=$2
+    echo "Verifying GPU access in container..."
+    echo -e "${YELLOW}Running: docker run $gpu_flag --rm deepethogram:$type nvidia-smi${NC}"
+    if ! docker run $gpu_flag --rm deepethogram:$type nvidia-smi; then
+        echo -e "${RED}Failed to access GPU in container${NC}"
+        return 1
+    fi
+    echo -e "${YELLOW}Running: docker run $gpu_flag --rm deepethogram:$type python -c \"import torch; print('CUDA available:', torch.cuda.is_available())\"${NC}"
+    if ! docker run $gpu_flag --rm deepethogram:$type python -c "import torch; print('CUDA available:', torch.cuda.is_available())" | grep -q "CUDA available: True"; then
+        echo -e "${RED}Failed to access GPU through PyTorch${NC}"
+        return 1
+    fi
+    return 0
+}
+
+# Function to run tests in container
+test_container() {
+    local type=$1
+    local gpu_flag=$2
+    local has_gpu=$3
+
+    print_header "Testing $type container"
+
+    # Test basic import
+    echo "Testing Python import..."
+    echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type python -c \"import deepethogram\"${NC}"
+    docker run $gpu_flag -it deepethogram:$type python -c "import deepethogram" && \
+        echo -e "${GREEN}✓ Import test passed${NC}" || \
+        (echo -e "${RED}✗ Import test failed${NC}" && exit 1)
+
+    # For containers that should support tests
+    if [ "$type" = "full" ] || [ "$type" = "headless" ]; then
+        echo "Running CPU tests..."
+        echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type pytest -v -m \"not gpu\" tests/${NC}"
+        docker run $gpu_flag -it deepethogram:$type pytest -v -m "not gpu" tests/ && \
+            echo -e "${GREEN}✓ CPU tests passed${NC}" || \
+            (echo -e "${RED}✗ CPU tests failed${NC}" && exit 1)
+
+        # Run GPU tests if GPU is available
+        if [ "$has_gpu" = true ] && [ "$type" != "gui" ]; then
+            echo "Running GPU tests..."
+            # First verify CUDA is accessible
+            echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type python -c \"import torch; assert torch.cuda.is_available(), 'CUDA not available'; print('CUDA is available')\"${NC}"
+            docker run $gpu_flag -it deepethogram:$type python -c "import torch; assert torch.cuda.is_available(), 'CUDA not available'; print('CUDA is available')"
+            # Run the actual GPU tests
+            echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type bash -c \"export CUDA_VISIBLE_DEVICES=0 && pytest -v -m gpu tests/\"${NC}"
+            docker run $gpu_flag -it deepethogram:$type \
+                bash -c "export CUDA_VISIBLE_DEVICES=0 && pytest -v -m gpu tests/" && \
+                echo -e "${GREEN}✓ GPU tests passed${NC}" || \
+                (echo -e "${RED}✗ GPU tests failed${NC}" && exit 1)
+        fi
+    fi
+
+    # For containers that should support GUI
+    if [ "$type" = "full" ] || [ "$type" = "gui" ]; then
+        echo "Testing GUI import..."
+        echo -e "${YELLOW}Running: docker run $gpu_flag -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw -it deepethogram:$type python -c \"from deepethogram.gui import main\"${NC}"
+        docker run $gpu_flag -e DISPLAY=$DISPLAY \
+            -v /tmp/.X11-unix:/tmp/.X11-unix:rw \
+            -it deepethogram:$type python -c "from deepethogram.gui import main" && \
+            echo -e "${GREEN}✓ GUI import test passed${NC}" || \
+            (echo -e "${RED}✗ GUI import test failed${NC}" && exit 1)
+    fi
+}
+
+# Main execution
+main() {
+    # Ensure we're in the project root
+    if [[ ! -f "pyproject.toml" ]]; then
+        echo -e "${RED}Error: Must run from project root directory (where pyproject.toml is located)${NC}"
+        exit 1
+    fi
+
+    # Check if GPU is available by testing nvidia-smi
+    local has_gpu=false
+    if command -v nvidia-smi &> /dev/null && nvidia-smi &> /dev/null; then
+        GPU_FLAG="--gpus all"
+        has_gpu=true
+        echo -e "${GREEN}NVIDIA GPU detected, will use GPUs and run GPU tests${NC}"
+    else
+        GPU_FLAG=""
+        echo -e "${RED}No NVIDIA GPU detected, running without GPU${NC}"
+    fi
+
+    # Build and test each image type
+    for type in "headless" "gui" "full"; do
+        build_image $type
+        # Verify GPU access after building if we have a GPU
+        if [ "$has_gpu" = true ] && [ "$type" != "gui" ]; then
+            if ! verify_gpu "$GPU_FLAG" "$type"; then
+                echo -e "${RED}GPU detected on host but not accessible in container. Please check nvidia-docker installation.${NC}"
+                exit 1
+            fi
+        fi
+        test_container $type "$GPU_FLAG" $has_gpu
+    done
+
+    print_header "All builds and tests completed successfully!"
+}
+
+# Execute main function
+main

docs/installation.md

@@ -14,7 +14,7 @@
 * `conda env create -f environment.yml`
     * Be prepared to wait a long time!! On mechanical hard drives, this may take 5-10 minutes (or more). Interrupting here will cause installation to fail.
 * `conda activate deg`
-* `python setup.py develop`
+* `pip install -e .`
 
 ### Installing Anaconda
 For instructions on installing anaconda,
@@ -62,3 +62,9 @@ environment with `conda create --name deg python=3.8` before installation.
   * This is an issue where Shiboken and PySide2 are not playing nicely together. Please `pip uninstall pyside2` and `conda remove pyside2`. Don't manually install these packages; instead, let DeepEthogram install it for you via pip. Therefore, `pip uninstall deepethogram` and `pip install deepethogram`.
 *  `qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in ".../python3.8/site-packages/cv2/qt/plugins"  even though it was found. This application failed to start because no Qt platform plugin could be initialized. Reinstalling the application may fix this problem.`
    * This is an issue with a recent version of `opencv-python` not working well with Qt. Please do `pip install --force-reinstall opencv-python-headless==4.1.2.30`
+
+# Beta: Using UV
+
+* install astral's UV on your system: `pip install uv`
+* `uv venv --python 3.7`: make a virtual environment
+* `uv pip install -e .`

environment.yml

@@ -4,9 +4,10 @@ channels:
   - conda-forge
   - defaults
 dependencies:
+  - python=3.7
+  - pytorch
+  - torchvision
   - pip
-  - conda-forge::pyside2=5.13.2
-  - python>3.7, <3.9
-  - pytorch::pytorch
   - pip:
-    - -r requirements.txt
+    - -e .
+  - conda-forge::pyside2=5.13.2

pyproject.toml

@@ -1,3 +1,53 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "deepethogram"
+version = "0.2.0"
+description = "Temporal action detection for biology"
+readme = "README.md"
+authors = [
+    {name = "Jim Bohnslav", email = "[email protected]"},
+]
+requires-python = ">=3.7,<3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "chardet<4.0",
+    "h5py",
+    "kornia>=0.5",
+    "matplotlib",
+    "numpy",
+    "omegaconf>=2",
+    "opencv-python-headless",
+    "opencv-transforms",
+    "pandas<1.4",
+    "PySide2==5.13.2",
+    "scikit-learn<1.1",
+    "scipy<1.8",
+    "tqdm",
+    "vidio",
+    "pytorch_lightning==1.6.5",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.1.0",
+    "pre-commit>=2.20.0,<3.0.0",
+    "pytest",
+    "pytest-cov",
+    "gdown",
+]
+
+[project.scripts]
+deepethogram = "deepethogram.gui.main:entry"
+
+[tool.setuptools]
+packages = ["deepethogram"]
+
 [tool.ruff]
 # Python version compatibility
 target-version = "py37"
@@ -33,7 +83,6 @@ exclude = [
 ]
 
 [tool.ruff.lint]
-
 # Allow autofix for all enabled rules (when `--fix`) is provided.
 fixable = ["ALL"]
 unfixable = []

reset_venv.sh

@@ -14,17 +14,9 @@ uv venv --python 3.7
 echo "Activating virtual environment..."
 source .venv/bin/activate
 
-# Install requirements first
-echo "Installing requirements..."
-uv pip install -r requirements.txt
-
-# Install pytest
-echo "Installing pytest..."
-uv pip install pytest pytest-cov
-
-# Install package in editable mode
-echo "Installing package in editable mode..."
-uv pip install -e .
+# Install package in editable mode with dev dependencies
+echo "Installing package and dependencies..."
+uv pip install -e ".[dev]"
 
 # Setup test data
 echo "Setting up test data..."