From 284a8e8372340fb72a32c2bfdc048453ea80cc28 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:19:26 -0400 Subject: [PATCH 01/13] Pin Python version 3 now resolves to 3.11 which fails to build psycopg2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 159b98d..5d91dd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.10 ENV PYTHONUNBUFFERED 1 RUN mkdir /code From 21b53f6a60eaee534ce1be6cd5aa02b50bab0891 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:48:55 -0400 Subject: [PATCH 02/13] Pin to 3.9 instead 3.10 errors out: ``` catchpy-web-1 | File "/code/anno/models.py", line 32, in catchpy-web-1 | from django.db.models import JSONField catchpy-web-1 | ImportError: cannot import name 'JSONField' from 'django.db.models' (/usr/local/lib/python3.10/site-packages/django/db/models/__init__.py) ``` --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5d91dd2..37fea7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.9 ENV PYTHONUNBUFFERED 1 RUN mkdir /code From d96b195cf48518f60faab479f3b2fd99b6a6e445 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 11:45:21 -0400 Subject: [PATCH 03/13] Try updating Django, add CI - Two tests are failing locally: ``` FAILED anno/tests/test_annojs.py::test_to_annotatorjs - AssertionError: assert False FAILED anno/tests/test_crud.py::test_remove_in_2step - ValueError: 'Anno' instance needs to have a primary key value before this relationship can be used. ``` --- .github/workflows/ci-pytest.yml | 23 +++++++++++++++++++++++ catchpy/requirements/base.txt | 2 +- catchpy/settings/test.py | 15 +-------------- 3 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/ci-pytest.yml diff --git a/.github/workflows/ci-pytest.yml b/.github/workflows/ci-pytest.yml new file mode 100644 index 0000000..3bf8cc3 --- /dev/null +++ b/.github/workflows/ci-pytest.yml @@ -0,0 +1,23 @@ +name: CI - Pytest + +on: + workflow_dispatch: + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build Docker images + run: | + docker compose -f docker-compose.yml build + - name: Run Docker Compose containers + run: | + docker compose -f docker-compose.yml up -d + - name: Run Pytest unit tests within Compose using tox + run: | + docker compose -f docker-compose.yml exec web CATCHPY_DOTENV_PATH=docker_dotenv.env tox + - name: Stop Docker Compose containers + if: always() + run: docker compose -f docker-compose.yml down \ No newline at end of file diff --git a/catchpy/requirements/base.txt b/catchpy/requirements/base.txt index 088481f..d5465bc 100644 --- a/catchpy/requirements/base.txt +++ b/catchpy/requirements/base.txt @@ -1,4 +1,4 @@ -Django==3.2.18 +Django~=4.2 iso8601==1.1.0 jsonschema==4.17.3 psycopg2==2.9.5 diff --git a/catchpy/settings/test.py b/catchpy/settings/test.py index 5af22a6..14da3fe 100644 --- a/catchpy/settings/test.py +++ b/catchpy/settings/test.py @@ -17,17 +17,4 @@ "level": "INFO", "handlers": ["console"], "propagate": True, -} - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.postgresql", - "NAME": "catchpy2", - "USER": "catchpy", - "PASSWORD": "catchpy", - "HOST": "dbserver.vm", - "PORT": "5432", - "ATOMIC_REQUESTS": False, - "CONN_MAX_AGE": 500, # permanent connections - }, -} +} \ No newline at end of file From 2a9bd72cd8f6fb55cbdc647a7400287dd53378c0 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:00:05 -0400 Subject: [PATCH 04/13] Declare dotenv path in env --- .github/workflows/ci-pytest.yml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-pytest.yml b/.github/workflows/ci-pytest.yml index 3bf8cc3..26dcc10 100644 --- a/.github/workflows/ci-pytest.yml +++ b/.github/workflows/ci-pytest.yml @@ -4,20 +4,23 @@ on: workflow_dispatch: pull_request: +env: + CATCHPY_DOTENV_PATH: docker_dotenv.env + jobs: tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Build Docker images - run: | - docker compose -f docker-compose.yml build - - name: Run Docker Compose containers - run: | - docker compose -f docker-compose.yml up -d - - name: Run Pytest unit tests within Compose using tox - run: | - docker compose -f docker-compose.yml exec web CATCHPY_DOTENV_PATH=docker_dotenv.env tox - - name: Stop Docker Compose containers - if: always() - run: docker compose -f docker-compose.yml down \ No newline at end of file + - uses: actions/checkout@v3 + - name: Build Docker images + run: | + docker compose -f docker-compose.yml build + - name: Run Docker Compose containers + run: | + docker compose -f docker-compose.yml up -d + - name: Run Pytest unit tests within Compose using tox + run: | + docker compose -f docker-compose.yml exec web tox + - name: Stop Docker Compose containers + if: always() + run: docker compose -f docker-compose.yml down \ No newline at end of file From f76ab76b276fbd0c12dff81ee7d4968fefdf1c6b Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:22:41 -0400 Subject: [PATCH 05/13] Update ci-pytest.yml --- .github/workflows/ci-pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pytest.yml b/.github/workflows/ci-pytest.yml index 26dcc10..ca59b95 100644 --- a/.github/workflows/ci-pytest.yml +++ b/.github/workflows/ci-pytest.yml @@ -18,9 +18,9 @@ jobs: - name: Run Docker Compose containers run: | docker compose -f docker-compose.yml up -d - - name: Run Pytest unit tests within Compose using tox + - name: Run Pytest unit tests within Compose run: | - docker compose -f docker-compose.yml exec web tox + docker compose -f docker-compose.yml exec web pytest -v anno/tests - name: Stop Docker Compose containers if: always() run: docker compose -f docker-compose.yml down \ No newline at end of file From 0314d1165092d02f7728bd5dbc81c3190ea1e80c Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 12:25:50 -0400 Subject: [PATCH 06/13] Update ci-pytest.yml --- .github/workflows/ci-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pytest.yml b/.github/workflows/ci-pytest.yml index ca59b95..8f69565 100644 --- a/.github/workflows/ci-pytest.yml +++ b/.github/workflows/ci-pytest.yml @@ -20,7 +20,7 @@ jobs: docker compose -f docker-compose.yml up -d - name: Run Pytest unit tests within Compose run: | - docker compose -f docker-compose.yml exec web pytest -v anno/tests + docker compose -f docker-compose.yml exec web bash -c "tox" - name: Stop Docker Compose containers if: always() run: docker compose -f docker-compose.yml down \ No newline at end of file From c1b62e20e3cd035003196d87c80be79692d5037c Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:04:11 -0400 Subject: [PATCH 07/13] Add separate Dockerfile and docker-compose for tests Install test requirements --- .github/workflows/ci-pytest.yml | 6 +++--- docker-compose-test.yml | 31 +++++++++++++++++++++++++++++++ test.Dockerfile | 9 +++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 docker-compose-test.yml create mode 100644 test.Dockerfile diff --git a/.github/workflows/ci-pytest.yml b/.github/workflows/ci-pytest.yml index 8f69565..3f5bfe4 100644 --- a/.github/workflows/ci-pytest.yml +++ b/.github/workflows/ci-pytest.yml @@ -14,13 +14,13 @@ jobs: - uses: actions/checkout@v3 - name: Build Docker images run: | - docker compose -f docker-compose.yml build + docker compose -f docker-compose-test.yml build - name: Run Docker Compose containers run: | - docker compose -f docker-compose.yml up -d + docker compose -f docker-compose-test.yml up -d - name: Run Pytest unit tests within Compose run: | - docker compose -f docker-compose.yml exec web bash -c "tox" + docker compose -f docker-compose-test.yml exec web bash -c "tox" - name: Stop Docker Compose containers if: always() run: docker compose -f docker-compose.yml down \ No newline at end of file diff --git a/docker-compose-test.yml b/docker-compose-test.yml new file mode 100644 index 0000000..e4f583c --- /dev/null +++ b/docker-compose-test.yml @@ -0,0 +1,31 @@ +version: '3' +services: + db: + image: postgres:15 + ports: + - "8001:5432" + environment: + POSTGRES_USER: catchpy + POSTGRES_PASSWORD: catchpy + POSTGRES_DB: catchpy + web: + build: test.Dockerfile + image: hx/catchpy:test + command: ["./wait-for-it.sh", "db:5432", "--", "python", "manage.py", "runserver", "0.0.0.0:8000"] + volumes: + - .:/code + ports: + - "8000:8000" + depends_on: + - db + environment: + CATCHPY_DOTENV_PATH: "/code/docker_dotenv.env" + networks: + default: + public: + aliases: + - catchpy.localhost + +networks: + public: + diff --git a/test.Dockerfile b/test.Dockerfile new file mode 100644 index 0000000..a36f59c --- /dev/null +++ b/test.Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.9 +ENV PYTHONUNBUFFERED 1 + +RUN mkdir /code +WORKDIR /code +ADD . /code + +RUN pip install -r catchpy/requirements/test.txt + From 632e1a6ceb2470bfd3387f760c53037f64e3de66 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:08:26 -0400 Subject: [PATCH 08/13] Local Dockerfile build --- .gitignore | 2 ++ docker-compose-test.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5976d7b..8103f47 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ catch/settings/local.py # static root public/ + +venv \ No newline at end of file diff --git a/docker-compose-test.yml b/docker-compose-test.yml index e4f583c..114dcad 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -9,7 +9,7 @@ services: POSTGRES_PASSWORD: catchpy POSTGRES_DB: catchpy web: - build: test.Dockerfile + build: ./test.Dockerfile image: hx/catchpy:test command: ["./wait-for-it.sh", "db:5432", "--", "python", "manage.py", "runserver", "0.0.0.0:8000"] volumes: From 8381392223831da1e42cff19b9b6d555157c1198 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 13:20:54 -0400 Subject: [PATCH 09/13] Fix syntax --- docker-compose-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 114dcad..a7f26dd 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -9,7 +9,9 @@ services: POSTGRES_PASSWORD: catchpy POSTGRES_DB: catchpy web: - build: ./test.Dockerfile + build: + context: . + dockerfile: test.Dockerfile image: hx/catchpy:test command: ["./wait-for-it.sh", "db:5432", "--", "python", "manage.py", "runserver", "0.0.0.0:8000"] volumes: From 25e94fd0d6492059213880d91a4e30cc94d3a0cd Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Tue, 25 Jul 2023 15:57:54 -0400 Subject: [PATCH 10/13] Fix to pass anno/tests/test_crud/test_remove_in_2step Was failing - I think the queryset items are just pointers so if they are deleted you can't do much with them? ValueError: 'Anno' instance needs to have a primary key value before this relationship can be used. anno/crud.py:662: in delete_annos success.append(a.serialized) anno/models.py:138: in serialized s['totalReplies'] = self.total_replies anno/models.py:117: in total_replies return self.anno_set.all().filter(anno_deleted=False).count() /usr/local/lib/python3.9/site-packages/django/db/models/manager.py:164: in all return self.get_queryset() --- anno/crud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/anno/crud.py b/anno/crud.py index a2ab324..0dac202 100644 --- a/anno/crud.py +++ b/anno/crud.py @@ -638,6 +638,7 @@ def delete_annos( failure = [] success = [] for a in selected: + serialization = a.serialized try: if true_delete and a.anno_deleted: a.delete() @@ -648,7 +649,7 @@ def delete_annos( failure.append(a.serialized) logger.error("failed to delete annotation({}): {}".format(a.anno_id, e)) else: - success.append(a.serialized) + success.append(serialization) return { "failed": len(failure), From 284ba62a15fcff3cc5c953b8d39568c813bf774a Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Wed, 26 Jul 2023 12:59:05 -0400 Subject: [PATCH 11/13] Fix anno/tests/test_annojs/test_to_annotatorjs Equality test was failing between AnnotatorJS backcompat annos. There was an empty quote key in the JSON blob, which was because the `fortune` library wasn't installed. Fixed by adding to the Dockerfile. --- Dockerfile | 5 +++++ test.Dockerfile | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Dockerfile b/Dockerfile index 37fea7f..d939399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,11 @@ FROM python:3.9 ENV PYTHONUNBUFFERED 1 +RUN apt-get update +# Include fortune library for quote generation for text annotations +RUN apt-get install fortune-mod -y +ENV PATH "$PATH:/usr/games" + RUN mkdir /code WORKDIR /code ADD . /code diff --git a/test.Dockerfile b/test.Dockerfile index a36f59c..884ca70 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -1,6 +1,11 @@ FROM python:3.9 ENV PYTHONUNBUFFERED 1 +RUN apt-get update +# Include fortune library for quote generation for text annotations +RUN apt-get install fortune-mod -y +ENV PATH "$PATH:/usr/games" + RUN mkdir /code WORKDIR /code ADD . /code From 9917c4248e5de2446b1b94697733f76a346ede66 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:03:43 -0400 Subject: [PATCH 12/13] Update requirements Python 3.8 to 3.11 (3.8+ is needed for Django 4.2+). Dockerfile uses Python 3.11. Bump other dependencies, most notably psycopg2 to psycopg (version 3 but drops the tailing version number). In the test Dockerfile, installs 3.11 as base and also install 3.8:3.12 to test the full range with tox. Currently tests still run in Docker containers and are not parallelized (tox matrix). Maybe try to switch to that as this will likely take a long time to run. --- .dockerignore | 27 +++++++++++++++++++++++++++ Dockerfile | 5 +---- README.rst | 20 +++++++++++++------- catchpy/requirements/base.txt | 14 +++++++------- setup.py | 13 +++++++------ test.Dockerfile | 21 ++++++++++++++++++--- tox.ini | 8 ++++++-- 7 files changed, 79 insertions(+), 29 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bcc75aa --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +*.pyc +*.pyo +*.mo +*.db +*.css.map +*.egg-info +*.sql.gz +.cache +.project +.idea +.pydevproject +.idea/workspace.xml +.DS_Store +.git/ +.github/ +.tox/ +.sass-cache +.vagrant/ +__pycache__ +dist +docs +env +venv/ +logs +stats +Dockerfile +test.Dockerfile \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d939399..c668ef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,7 @@ -FROM python:3.9 +FROM python:3.11 ENV PYTHONUNBUFFERED 1 RUN apt-get update -# Include fortune library for quote generation for text annotations -RUN apt-get install fortune-mod -y -ENV PATH "$PATH:/usr/games" RUN mkdir /code WORKDIR /code diff --git a/README.rst b/README.rst index 43b3793..da041c9 100644 --- a/README.rst +++ b/README.rst @@ -69,13 +69,13 @@ the ``Authorize`` button at the top right of the page. Not So Quick Start ------------------ -For those who want to set up a local instance of catchpy, for tests or -developement. +For those who want to set up a local instance of Catchpy, for tests or +development. -Setting up catchpy locally requires: +Setting up Catchpy locally requires: - - postgres 9.6 or higher - - python 3.5 or higher + - Postgres 9.6 or higher + - Python 3.8 or higher (Django 4.2 requirement) :: @@ -135,14 +135,14 @@ Run unit tests unit tests require: - - a postgres 9.6 or higher running (and its config in + - Postgres 9.6 or higher (config in ``catchpy/settings/test.py``); this is hard to fake because it requires postgres jsonb data type - the fortune program, ex: ``brew install fortune`` if you're in macos. ``fortune`` is used to create content in test annotations. -tests are located under each django app: +tests are located under each Django app: :: @@ -156,6 +156,12 @@ tests are located under each django app: CATCHPY_DOTENV_PATH=/path/to/dotenv/file tox +Github Actions CI +--------------- +Github Actions is configured to run unit tests on every new PR. The tests are configured in +``.github/workflows/ci-pytest.yml``. The workflow is configured to run tests on Python3.8-3.12 using +``tox``. + ---eop diff --git a/catchpy/requirements/base.txt b/catchpy/requirements/base.txt index d5465bc..94b24fd 100644 --- a/catchpy/requirements/base.txt +++ b/catchpy/requirements/base.txt @@ -1,12 +1,12 @@ Django~=4.2 -iso8601==1.1.0 -jsonschema==4.17.3 -psycopg2==2.9.5 -PyJWT==2.6.0 +iso8601~=2.0.0 +jsonschema==4.18.4 +psycopg>=3.1.8 +PyJWT==2.8.0 PyLD==2.0.3 python-dateutil==2.8.2 python-dotenv==1.0.0 -pytz==2022.7.1 -requests==2.28.2 +pytz==2023.3 +requests~=2.31.0 django-log-request-id==2.1.0 -django-cors-headers==3.14.0 +django-cors-headers~=4.2.0 diff --git a/setup.py b/setup.py index b74fcee..97d2074 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def get_version(*file_paths): "Django", "iso8601", "jsonschema", - "psycopg2", + "psycopg", "pyjwt", "pyld", "python-dateutil", @@ -64,12 +64,13 @@ def get_version(*file_paths): classifiers=[ 'Development Status :: 3 - Alpha', 'Framework :: Django', - 'Framework :: Django :: 1.11', + 'Framework :: Django :: 4.2', 'Intended Audience :: Developers', 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ) diff --git a/test.Dockerfile b/test.Dockerfile index 884ca70..8503cd0 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -1,14 +1,29 @@ -FROM python:3.9 +FROM python:3.11 ENV PYTHONUNBUFFERED 1 RUN apt-get update + # Include fortune library for quote generation for text annotations RUN apt-get install fortune-mod -y ENV PATH "$PATH:/usr/games" +# Install all other versions of Python we want to test with tox +RUN git clone https://github.com/pyenv/pyenv /root/.pyenv +RUN for PYTHON_VERSION in 3.8.17 3.9.17 3.10.12 3.11.4 3.12.0b4; do \ + set -ex \ + && /root/.pyenv/bin/pyenv install ${PYTHON_VERSION} \ + && /root/.pyenv/versions/${PYTHON_VERSION}/bin/python -m pip install --upgrade pip \ + ; done + +# Add to PATH, in order of lowest precedence to highest. +ENV PATH /root/.pyenv/versions/3.8.17/bin:${PATH} +ENV PATH /root/.pyenv/versions/3.9.17/bin:${PATH} +ENV PATH /root/.pyenv/versions/3.10.12/bin:${PATH} +ENV PATH /root/.pyenv/versions/3.12.0b4/bin:${PATH} +ENV PATH /root/.pyenv/versions/3.11.4/bin:${PATH} + RUN mkdir /code WORKDIR /code ADD . /code -RUN pip install -r catchpy/requirements/test.txt - +RUN pip install -r catchpy/requirements/test.txt \ No newline at end of file diff --git a/tox.ini b/tox.ini index b71169b..5f81f56 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,10 @@ [tox] -envlis = py35, py36 - +env_list = + py38 + py39 + py310 + py311 + py312 [testenv] deps = From 5bdc1ba07bd6e3ba1d4bb61818f04c67bf832739 Mon Sep 17 00:00:00 2001 From: Cole Crawford <16374762+ColeDCrawford@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:16:27 -0400 Subject: [PATCH 13/13] Run GH Actions workflow if PR edited --- .github/workflows/ci-pytest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pytest.yml b/.github/workflows/ci-pytest.yml index 3f5bfe4..3a9b5b5 100644 --- a/.github/workflows/ci-pytest.yml +++ b/.github/workflows/ci-pytest.yml @@ -3,6 +3,7 @@ name: CI - Pytest on: workflow_dispatch: pull_request: + types: [opened, reopened, edited, synchronize] env: CATCHPY_DOTENV_PATH: docker_dotenv.env