diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index c78bd01..8a09cd8 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -33,12 +33,10 @@ # run: | # pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html # -# - name: Install package and test dependencies +# - name: Install dependencies # run: | -# python -m pip install --upgrade "pip<24.0" -# pip install -r requirements.txt -# pip install pytest pytest-cov -# python setup.py develop +# python -m pip install --upgrade pip +# pip install -e . # # - name: Setup test data # run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0bc5409..0ce29f0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,12 +48,10 @@ jobs: run: | pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - - name: Install package and test dependencies + - name: Install dependencies run: | - python -m pip install --upgrade "pip<24.0" - pip install -r requirements.txt - pip install pytest pytest-cov - python setup.py develop + python -m pip install --upgrade pip + pip install . - name: Setup test data run: | diff --git a/docker/Dockerfile-full b/docker/Dockerfile-full index d66833a..ae47ca4 100644 --- a/docker/Dockerfile-full +++ b/docker/Dockerfile-full @@ -31,10 +31,9 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39 # install RUN conda install python=3.7 -y -RUN pip install setuptools --upgrade && pip install --upgrade "pip<24.0" -RUN pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html - +RUN pip install --upgrade pip +# Install deepethogram with dev dependencies ADD . /app/deepethogram WORKDIR /app/deepethogram ENV DEG_VERSION='full' -RUN pip install -e . +RUN pip install -e ".[dev]" diff --git a/docker/Dockerfile-gui b/docker/Dockerfile-gui index b8b6487..83eff6b 100644 --- a/docker/Dockerfile-gui +++ b/docker/Dockerfile-gui @@ -35,13 +35,13 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39 # install RUN conda install python=3.7 -y -RUN pip install setuptools --upgrade && pip install --upgrade pip +RUN pip install --upgrade pip # TODO: REFACTOR CODE SO IT'S POSSIBLE TO RUN GUI WITHOUT TORCH RUN conda install pytorch cpuonly -c pytorch -# # needed for pandas for some reason +# Install deepethogram with dev dependencies ADD . /app/deepethogram WORKDIR /app/deepethogram ENV DEG_VERSION='gui' -RUN pip install -e . +RUN pip install -e ".[dev]" diff --git a/docker/Dockerfile-headless b/docker/Dockerfile-headless index ce92955..05f2f2b 100644 --- a/docker/Dockerfile-headless +++ b/docker/Dockerfile-headless @@ -33,11 +33,11 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39 # install RUN conda install python=3.7 -y -RUN pip install setuptools --upgrade && pip install --upgrade pip -RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch +RUN pip install --upgrade pip +# RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch -# # needed for pandas for some reason +# Install deepethogram with dev dependencies ADD . /app/deepethogram WORKDIR /app/deepethogram ENV DEG_VERSION='headless' -RUN pip install -e . +RUN pip install -e ".[dev]" diff --git a/docker/build_and_test.sh b/docker/build_and_test.sh new file mode 100755 index 0000000..71fc214 --- /dev/null +++ b/docker/build_and_test.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +set -e # Exit on any error + +# Colors for output +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' # No Color +BLUE='\033[0;34m' +YELLOW='\033[1;33m' + +# Function to print section headers +print_header() { + echo -e "\n${BLUE}=== $1 ===${NC}\n" +} + +# Function to echo command before running +echo_run() { + echo -e "${YELLOW}Running: $@${NC}" + "$@" +} + +# Function to build an image +build_image() { + local type=$1 + print_header "Building $type image" + echo_run docker build -t deepethogram:$type -f docker/Dockerfile-$type . +} + +# Function to verify GPU in container +verify_gpu() { + local gpu_flag=$1 + local type=$2 + echo "Verifying GPU access in container..." + echo -e "${YELLOW}Running: docker run $gpu_flag --rm deepethogram:$type nvidia-smi${NC}" + if ! docker run $gpu_flag --rm deepethogram:$type nvidia-smi; then + echo -e "${RED}Failed to access GPU in container${NC}" + return 1 + fi + echo -e "${YELLOW}Running: docker run $gpu_flag --rm deepethogram:$type python -c \"import torch; print('CUDA available:', torch.cuda.is_available())\"${NC}" + if ! docker run $gpu_flag --rm deepethogram:$type python -c "import torch; print('CUDA available:', torch.cuda.is_available())" | grep -q "CUDA available: True"; then + echo -e "${RED}Failed to access GPU through PyTorch${NC}" + return 1 + fi + return 0 +} + +# Function to run tests in container +test_container() { + local type=$1 + local gpu_flag=$2 + local has_gpu=$3 + + print_header "Testing $type container" + + # Test basic import + echo "Testing Python import..." + echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type python -c \"import deepethogram\"${NC}" + docker run $gpu_flag -it deepethogram:$type python -c "import deepethogram" && \ + echo -e "${GREEN}✓ Import test passed${NC}" || \ + (echo -e "${RED}✗ Import test failed${NC}" && exit 1) + + # For containers that should support tests + if [ "$type" = "full" ] || [ "$type" = "headless" ]; then + echo "Running CPU tests..." + echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type pytest -v -m \"not gpu\" tests/${NC}" + docker run $gpu_flag -it deepethogram:$type pytest -v -m "not gpu" tests/ && \ + echo -e "${GREEN}✓ CPU tests passed${NC}" || \ + (echo -e "${RED}✗ CPU tests failed${NC}" && exit 1) + + # Run GPU tests if GPU is available + if [ "$has_gpu" = true ] && [ "$type" != "gui" ]; then + echo "Running GPU tests..." + # First verify CUDA is accessible + echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type python -c \"import torch; assert torch.cuda.is_available(), 'CUDA not available'; print('CUDA is available')\"${NC}" + docker run $gpu_flag -it deepethogram:$type python -c "import torch; assert torch.cuda.is_available(), 'CUDA not available'; print('CUDA is available')" + # Run the actual GPU tests + echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type bash -c \"export CUDA_VISIBLE_DEVICES=0 && pytest -v -m gpu tests/\"${NC}" + docker run $gpu_flag -it deepethogram:$type \ + bash -c "export CUDA_VISIBLE_DEVICES=0 && pytest -v -m gpu tests/" && \ + echo -e "${GREEN}✓ GPU tests passed${NC}" || \ + (echo -e "${RED}✗ GPU tests failed${NC}" && exit 1) + fi + fi + + # For containers that should support GUI + if [ "$type" = "full" ] || [ "$type" = "gui" ]; then + echo "Testing GUI import..." + echo -e "${YELLOW}Running: docker run $gpu_flag -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw -it deepethogram:$type python -c \"from deepethogram.gui import main\"${NC}" + docker run $gpu_flag -e DISPLAY=$DISPLAY \ + -v /tmp/.X11-unix:/tmp/.X11-unix:rw \ + -it deepethogram:$type python -c "from deepethogram.gui import main" && \ + echo -e "${GREEN}✓ GUI import test passed${NC}" || \ + (echo -e "${RED}✗ GUI import test failed${NC}" && exit 1) + fi +} + +# Main execution +main() { + # Ensure we're in the project root + if [[ ! -f "pyproject.toml" ]]; then + echo -e "${RED}Error: Must run from project root directory (where pyproject.toml is located)${NC}" + exit 1 + fi + + # Check if GPU is available by testing nvidia-smi + local has_gpu=false + if command -v nvidia-smi &> /dev/null && nvidia-smi &> /dev/null; then + GPU_FLAG="--gpus all" + has_gpu=true + echo -e "${GREEN}NVIDIA GPU detected, will use GPUs and run GPU tests${NC}" + else + GPU_FLAG="" + echo -e "${RED}No NVIDIA GPU detected, running without GPU${NC}" + fi + + # Build and test each image type + for type in "headless" "gui" "full"; do + build_image $type + # Verify GPU access after building if we have a GPU + if [ "$has_gpu" = true ] && [ "$type" != "gui" ]; then + if ! verify_gpu "$GPU_FLAG" "$type"; then + echo -e "${RED}GPU detected on host but not accessible in container. Please check nvidia-docker installation.${NC}" + exit 1 + fi + fi + test_container $type "$GPU_FLAG" $has_gpu + done + + print_header "All builds and tests completed successfully!" +} + +# Execute main function +main diff --git a/docs/installation.md b/docs/installation.md index 313f148..5b26ecb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -14,7 +14,7 @@ * `conda env create -f environment.yml` * Be prepared to wait a long time!! On mechanical hard drives, this may take 5-10 minutes (or more). Interrupting here will cause installation to fail. * `conda activate deg` -* `python setup.py develop` +* `pip install -e .` ### Installing Anaconda For instructions on installing anaconda, @@ -62,3 +62,9 @@ environment with `conda create --name deg python=3.8` before installation. * This is an issue where Shiboken and PySide2 are not playing nicely together. Please `pip uninstall pyside2` and `conda remove pyside2`. Don't manually install these packages; instead, let DeepEthogram install it for you via pip. Therefore, `pip uninstall deepethogram` and `pip install deepethogram`. * `qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in ".../python3.8/site-packages/cv2/qt/plugins" even though it was found. This application failed to start because no Qt platform plugin could be initialized. Reinstalling the application may fix this problem.` * This is an issue with a recent version of `opencv-python` not working well with Qt. Please do `pip install --force-reinstall opencv-python-headless==4.1.2.30` + +# Beta: Using UV + +* install astral's UV on your system: `pip install uv` +* `uv venv --python 3.7`: make a virtual environment +* `uv pip install -e .` diff --git a/environment.yml b/environment.yml index bb3dccc..2ba47c7 100644 --- a/environment.yml +++ b/environment.yml @@ -4,9 +4,10 @@ channels: - conda-forge - defaults dependencies: + - python=3.7 + - pytorch + - torchvision - pip - - conda-forge::pyside2=5.13.2 - - python>3.7, <3.9 - - pytorch::pytorch - pip: - - -r requirements.txt + - -e . + - conda-forge::pyside2=5.13.2 diff --git a/pyproject.toml b/pyproject.toml index c099f91..cefd73a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,53 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "deepethogram" +version = "0.2.0" +description = "Temporal action detection for biology" +readme = "README.md" +authors = [ + {name = "Jim Bohnslav", email = "jbohnslav@gmail.com"}, +] +requires-python = ">=3.7,<3.8" +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +dependencies = [ + "chardet<4.0", + "h5py", + "kornia>=0.5", + "matplotlib", + "numpy", + "omegaconf>=2", + "opencv-python-headless", + "opencv-transforms", + "pandas<1.4", + "PySide2==5.13.2", + "scikit-learn<1.1", + "scipy<1.8", + "tqdm", + "vidio", + "pytorch_lightning==1.6.5", +] + +[project.optional-dependencies] +dev = [ + "ruff>=0.1.0", + "pre-commit>=2.20.0,<3.0.0", + "pytest", + "pytest-cov", + "gdown", +] + +[project.scripts] +deepethogram = "deepethogram.gui.main:entry" + +[tool.setuptools] +packages = ["deepethogram"] + [tool.ruff] # Python version compatibility target-version = "py37" @@ -33,7 +83,6 @@ exclude = [ ] [tool.ruff.lint] - # Allow autofix for all enabled rules (when `--fix`) is provided. fixable = ["ALL"] unfixable = [] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9626e8d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -chardet<4.0 -h5py -kornia>=0.5 -matplotlib -numpy -omegaconf>=2 -opencv-python-headless -opencv-transforms -pandas<1.4 -PySide2==5.13.2 -pytest -scikit-learn<1.1 -scipy<1.8 -tqdm -vidio -pytorch_lightning==1.6.5 -ruff>=0.1.0 -pre-commit>=2.20.0,<3.0.0 -setuptools -gdown diff --git a/reset_venv.sh b/reset_venv.sh index bfd5d8e..8ffab47 100755 --- a/reset_venv.sh +++ b/reset_venv.sh @@ -14,17 +14,9 @@ uv venv --python 3.7 echo "Activating virtual environment..." source .venv/bin/activate -# Install requirements first -echo "Installing requirements..." -uv pip install -r requirements.txt - -# Install pytest -echo "Installing pytest..." -uv pip install pytest pytest-cov - -# Install package in editable mode -echo "Installing package in editable mode..." -uv pip install -e . +# Install package in editable mode with dev dependencies +echo "Installing package and dependencies..." +uv pip install -e ".[dev]" # Setup test data echo "Setting up test data..." diff --git a/setup.py b/setup.py deleted file mode 100644 index 971ff08..0000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -import setuptools - -with open("README.md", "r") as f: - long_description = f.read() - - -def get_requirements(): - with open("requirements.txt") as f: - return f.read().splitlines() - - -setuptools.setup( - name="deepethogram", - version="0.2.0", - author="Jim Bohnslav", - author_email="jbohnslav@gmail.com", - description="Temporal action detection for biology", - long_description=long_description, - long_description_content_type="text/markdown", - include_package_data=True, - packages=setuptools.find_packages(), - classifiers=[ - "Programming Language :: Python :: 3", - "Operating System :: OS Independent", - ], - entry_points={"console_scripts": ["deepethogram = deepethogram.gui.main:entry"]}, - python_requires=">=3.7,<3.8", - install_requires=get_requirements(), - options={ - "ruff": { - "target-version": "py37", - }, - }, - setup_requires=["setuptools"], -)