From 2dda4ee92eb808e26ebc59d44afe5b0c787ba42e Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 6 May 2024 09:02:26 -0400 Subject: [PATCH 1/5] Split docker files and requirements. --- Dockerfile | 24 +++++------------------- deps.txt | 7 +++++++ requirements.txt | 7 +------ scriptshifter_base.Dockerfile | 21 +++++++++++++++++++++ test.Dockerfile | 7 +++++++ 5 files changed, 41 insertions(+), 25 deletions(-) create mode 100644 deps.txt create mode 100644 scriptshifter_base.Dockerfile create mode 100644 test.Dockerfile diff --git a/Dockerfile b/Dockerfile index aabab1b..5d3e99c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,15 @@ -FROM python:3.10-slim-bookworm +FROM lcnetdev/scriptshifter_base:latest +ARG WORKROOT "/usr/local/scriptshifter/src" -RUN apt update -RUN apt install -y build-essential tzdata gfortran libopenblas-dev libboost-all-dev libpcre2-dev - -ENV TZ=America/New_York -ENV _workroot "/usr/local/scriptshifter/src" - -RUN addgroup --system www -RUN adduser --system www -RUN gpasswd -a www www - -WORKDIR ${_workroot} +# Copy core application files. +WORKDIR ${WORKROOT} COPY entrypoint.sh uwsgi.ini wsgi.py ./ -COPY ext ./ext/ COPY scriptshifter ./scriptshifter/ - COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -# Remove development packages. -RUN apt remove -y build-essential git -RUN apt autoremove -y - RUN chmod +x ./entrypoint.sh -RUN chown -R www:www ${_workroot} . +#RUN chown -R www:www ${WORKROOT} . EXPOSE 8000 diff --git a/deps.txt b/deps.txt new file mode 100644 index 0000000..67b0a05 --- /dev/null +++ b/deps.txt @@ -0,0 +1,7 @@ +# External dependencies. +aksharamukha>=2.1,<3 +camel-tools>=1.5 +funcy>=1.15,<2 +pymarc>=4.0,<5 +repackage>=0.7.3 +./ext/yiddish diff --git a/requirements.txt b/requirements.txt index ccd5d4c..6f1a221 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,5 @@ -aksharamukha>=2.1,<3 -camel-tools>=1.5 +# Core application dependencies. flask>=2.3,<3 -funcy>=1.15,<2 -pymarc>=4.0,<5 python-dotenv>=1.0,<2 pyyaml>=6.0,<7 -repackage>=0.7.3 uwsgi>=2.0,<2.1 -./ext/yiddish diff --git a/scriptshifter_base.Dockerfile b/scriptshifter_base.Dockerfile new file mode 100644 index 0000000..9cd06c3 --- /dev/null +++ b/scriptshifter_base.Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.10-slim-bookworm + +RUN apt update +RUN apt install -y build-essential tzdata gfortran libopenblas-dev libboost-all-dev libpcre2-dev + +ENV TZ=America/New_York +ARG WORKROOT "/usr/local/scriptshifter/src" + +RUN addgroup --system www +RUN adduser --system www +RUN gpasswd -a www www + +# Copy external dependencies. +WORKDIR ${WORKROOT} +COPY ext ./ext/ +COPY deps.txt ./ +RUN pip install --no-cache-dir -r deps.txt + +# Remove development packages. +RUN apt remove -y build-essential git +RUN apt autoremove -y diff --git a/test.Dockerfile b/test.Dockerfile new file mode 100644 index 0000000..0f310c4 --- /dev/null +++ b/test.Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.10-slim-bookworm + +RUN apt update +RUN apt install -y build-essential libpcre2-dev + +RUN pip install uwsgi + From e4449ff796702fc11d0f01a1e87f3b5f182808a8 Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 6 May 2024 09:02:50 -0400 Subject: [PATCH 2/5] Add bad request debug handler. --- scriptshifter/rest_api.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scriptshifter/rest_api.py b/scriptshifter/rest_api.py index ccf5d0e..5ed4206 100644 --- a/scriptshifter/rest_api.py +++ b/scriptshifter/rest_api.py @@ -8,6 +8,7 @@ from smtplib import SMTP from flask import Flask, jsonify, render_template, request +from werkzeug.exceptions import BadRequest from scriptshifter import EMAIL_FROM, EMAIL_TO, SMTP_HOST, SMTP_PORT from scriptshifter.exceptions import ApiError @@ -46,6 +47,20 @@ def handle_exception(e: ApiError): }, e.status_code) +@app.errorhandler(BadRequest) +def handle_400(e): + if logging.DEBUG >= logging.root.level: + body = { + "debug": { + "form_data": request.form, + } + } + else: + body = "" + + return body, 400 + + @app.route("/", methods=["GET"]) def index(): return render_template( @@ -108,9 +123,7 @@ def transliterate_req(): except (NotImplementedError, ValueError) as e: return (str(e), 400) - return { - "output": out, "warnings": warnings, - "debug": {"form_data": request.form}} + return {"output": out, "warnings": warnings} @app.route("/feedback", methods=["POST"]) From b4eb47d2a918256aa41f2654e25adfd6da650536 Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 6 May 2024 18:37:29 -0400 Subject: [PATCH 3/5] Adjust CI workflows. --- .github/workflows/push-app-image.yml | 34 +++++++++++++++++++ ...h-docker-image.yml => push-base-image.yml} | 9 ++--- .github/workflows/push-test-image.yml | 17 +++------- 3 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/push-app-image.yml rename .github/workflows/{push-docker-image.yml => push-base-image.yml} (81%) diff --git a/.github/workflows/push-app-image.yml b/.github/workflows/push-app-image.yml new file mode 100644 index 0000000..5d61aec --- /dev/null +++ b/.github/workflows/push-app-image.yml @@ -0,0 +1,34 @@ +name: Push image to Docker Hub. +on: + push: + tags: + - "v*.*.*" + +env: + DOCKER_USER: lcnetdev + DOCKER_PASSWORD: ${{secrets.DOCKER_HUB}} + REPO_NAME: scriptshifter + +jobs: + push-image-to-docker-hub: + runs-on: ubuntu-latest + steps: + - name: checkout repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Build the Docker image + run: > + docker build -f Dockerfile . + --tag $DOCKER_USER/$REPO_NAME:${{ github.ref_name }} + --tag $DOCKER_USER/$REPO_NAME:latest + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: lcnetdev + password: ${{ secrets.DOCKER_HUB }} + + - name: Push to Docker Hub + run: docker push $DOCKER_USER/$REPO_NAME --all-tags diff --git a/.github/workflows/push-docker-image.yml b/.github/workflows/push-base-image.yml similarity index 81% rename from .github/workflows/push-docker-image.yml rename to .github/workflows/push-base-image.yml index 06d2709..e9721e5 100644 --- a/.github/workflows/push-docker-image.yml +++ b/.github/workflows/push-base-image.yml @@ -1,13 +1,13 @@ -name: Push image to Docker Hub. +name: Push base image to Docker Hub. Only on minor/major version update. on: push: tags: - - "v*.*.*" + - "v*.*.0" env: DOCKER_USER: lcnetdev DOCKER_PASSWORD: ${{secrets.DOCKER_HUB}} - REPO_NAME: scriptshifter + REPO_NAME: scriptshifter-base jobs: push-image-to-docker-hub: @@ -32,7 +32,8 @@ jobs: - name: Build the Docker image run: > - docker build . --tag $DOCKER_USER/$REPO_NAME:${{ github.ref_name }} + docker build -f scriptshifter_base.Dockerfile . + --tag $DOCKER_USER/$REPO_NAME:${{ github.ref_name }} --tag $DOCKER_USER/$REPO_NAME:latest - name: Login to Docker Hub diff --git a/.github/workflows/push-test-image.yml b/.github/workflows/push-test-image.yml index 538544a..c7502e3 100644 --- a/.github/workflows/push-test-image.yml +++ b/.github/workflows/push-test-image.yml @@ -18,20 +18,11 @@ jobs: with: submodules: recursive - - name: checkout yiddish submodules (1/2) - uses: actions/checkout@v4 - with: - repository: ibleaman/loshn-koydesh-pronunciation - path: ext/yiddish/yiddish/submodules/loshn-koydesh-pronunciation - - - name: checkout yiddish submodules (2/2) - uses: actions/checkout@v4 - with: - repository: ibleaman/hasidify_lexicon - path: ext/yiddish/yiddish/submodules/hasidify_lexicon - - name: Build the Docker image - run: docker build . --tag $DOCKER_USER/$REPO_NAME:test + run: > + docker build -f Dockerfile . + --tag $DOCKER_USER/$REPO_NAME:${{ github.ref_name }} + --tag $DOCKER_USER/$REPO_NAME:test - name: Login to Docker Hub uses: docker/login-action@v3 From 779b0c6fe1bc4613f68521d8702053f2898c5142 Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 6 May 2024 18:50:25 -0400 Subject: [PATCH 4/5] Fix image name typo. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5d3e99c..779cfe4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lcnetdev/scriptshifter_base:latest +FROM lcnetdev/scriptshifter-base:latest ARG WORKROOT "/usr/local/scriptshifter/src" # Copy core application files. From 29d53092fa56631c778d91614502d2baa40a5020 Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 6 May 2024 18:59:47 -0400 Subject: [PATCH 5/5] Refine triggers. --- .github/workflows/push-app-image.yml | 9 +++++++-- .github/workflows/push-base-image.yml | 2 +- .github/workflows/push-test-image.yml | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push-app-image.yml b/.github/workflows/push-app-image.yml index 5d61aec..1c704ae 100644 --- a/.github/workflows/push-app-image.yml +++ b/.github/workflows/push-app-image.yml @@ -1,8 +1,13 @@ -name: Push image to Docker Hub. +name: Push app image on: + # This runs on v *.*.0 after the base image has been + # built and pushed, or on v push: tags: - - "v*.*.*" + - "v*.*.[1-9]*" + workflow_run: + workflows: ["Push base image"] + types: ["completed"] env: DOCKER_USER: lcnetdev diff --git a/.github/workflows/push-base-image.yml b/.github/workflows/push-base-image.yml index e9721e5..db26243 100644 --- a/.github/workflows/push-base-image.yml +++ b/.github/workflows/push-base-image.yml @@ -1,4 +1,4 @@ -name: Push base image to Docker Hub. Only on minor/major version update. +name: Push base image on: push: tags: diff --git a/.github/workflows/push-test-image.yml b/.github/workflows/push-test-image.yml index c7502e3..07389a0 100644 --- a/.github/workflows/push-test-image.yml +++ b/.github/workflows/push-test-image.yml @@ -1,4 +1,4 @@ -name: Push test image to Docker Hub. +name: Push test image on: push: branch: