Cleanup job system, add feature flags

.development.env

@@ -1,3 +1,5 @@
+HOST=localhost
+
 
 LTS_FOLDER=./lts
 MOCKARCHIVER_IMAGE=ghcr.io/swissopenem/scopemarchiver-mockarchiver:latest

.github/workflows/build-docker-images.yml

@@ -7,9 +7,9 @@ env:
     # github.repository as <account>/<repo>
     MOCKARCHIVER_IMAGE_NAME: ${{ github.repository }}-mockarchiver
     WEBUPLOADER_IMAGE_NAME: ${{ github.repository }}-webuploader
-    CELERY_WORKER_IMAGE_NAME: ${{ github.repository }}-worker
-    CELERY_SERVER_IMAGE_NAME: ${{ github.repository }}-server
-    CELERY_FLOWER_IMAGE_NAME: ${{ github.repository }}-flower
+    CELERY_WORKER_IMAGE_NAME: ${{ github.repository }}-openem-worker
+    OPENEM_BACKEND_IMAGE_NAME: ${{ github.repository }}-openem-backend
+    CELERY_FLOWER_IMAGE_NAME: ${{ github.repository }}-openem-flower
 
 on:
   push:
@@ -28,7 +28,7 @@ jobs:
     outputs:
       mockarchiver: ${{ steps.filter.outputs.mockarchiver }}
       webuploader: ${{ steps.filter.outputs.webuploader }}
-      jobsystem: ${{ steps.filter.outputs.jobsystem }}
+      backend: ${{ steps.filter.outputs.backend }}
     steps:
     # For pull requests it's not necessary to checkout the code
     - uses: actions/checkout@v4
@@ -41,8 +41,8 @@ jobs:
             - 'mockarchiver/**'
           webuploader:
             - 'webuploader/**'
-          jobsystem:
-            - 'jobsystem/**'
+          backend:
+            - 'backend/**'
 
   build_mockarchiver:
     runs-on: ubuntu-latest
@@ -93,7 +93,7 @@ jobs:
       contents: read
 
     needs: detect_changes
-    if: ${{ needs.detect_changes.outputs.jobsystem == 'true' }} || github.event_name != 'pull_request'
+    if: ${{ needs.detect_changes.outputs.backend == 'true' }} || github.event_name != 'pull_request'
 
     steps:
     - uses: actions/checkout@v3
@@ -121,21 +121,21 @@ jobs:
       id: build-and-push
       uses: docker/build-push-action@v5
       with:
-          context: ./jobsystem
-          file: ./jobsystem/worker.Dockerfile
+          context: ./backend
+          file: ./backend/worker.Dockerfile
           push: true # ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
-  build_celery_server:
+  build_openem_backend:
     runs-on: ubuntu-latest
 
     permissions:
       packages: write
       contents: read
 
     needs: detect_changes
-    if: ${{ needs.detect_changes.outputs.jobsystem == 'true' }} || github.event_name != 'pull_request'
+    if: ${{ needs.detect_changes.outputs.backend == 'true' }} || github.event_name != 'pull_request'
 
     steps:
     - uses: actions/checkout@v3
@@ -155,16 +155,16 @@ jobs:
       id: meta
       uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
       with:
-        images: ${{ env.REGISTRY }}/${{ env.CELERY_SERVER_IMAGE_NAME }}
+        images: ${{ env.REGISTRY }}/${{ env.OPENEM_BACKEND_IMAGE_NAME }}
 
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
     - name: Build and push Docker image
       id: build-and-push
       uses: docker/build-push-action@v5
       with:
-          context: ./jobsystem
-          file: ./jobsystem/web.Dockerfile
+          context: ./backend
+          file: ./backend/web.Dockerfile
           push: true # ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -177,7 +177,7 @@ jobs:
       contents: read
 
     needs: detect_changes
-    if: ${{ needs.detect_changes.outputs.jobsystem == 'true' }} || github.event_name != 'pull_request'
+    if: ${{ needs.detect_changes.outputs.backend == 'true' }} || github.event_name != 'pull_request'
 
     steps:
     - uses: actions/checkout@v3
@@ -205,8 +205,8 @@ jobs:
       id: build-and-push
       uses: docker/build-push-action@v5
       with:
-          context: ./jobsystem
-          file: ./jobsystem/flower.Dockerfile
+          context: ./backend
+          file: ./backend/flower.Dockerfile
           push: true # ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.env → .production.env

@@ -1,28 +1,33 @@
 # Global
 HOST=openem-dev.ethz.ch
+ARCHIVER_ENABLED=True
+INGESTER_ENABLED=True
 
-# TUSD
-TUSD_HOOKS_FOLDER=./tusd/hooks
 
-# Minio
+# Backend
+CELERY_FLOWER_IMAGE_NAME=ghcr.io/swissopenem/openem-celery-flower
+CELERY_WORKER_IMAGE_NAME=ghcr.io/swissopenem/openem-celery-worker
+OPENEM_BACKEND_IMAGE_NAME=ghcr.io/swissopenem/openem-backend
+OPENEM_IMAGE_TAG=main-local
+
+# Archiver
+CELERY_ARCHIVING_DIR=/data/LTS
+
+## Minio
 MINIO_STORAGE_FOLDER=/data/minio
 MINIO_REGION="eu-west-1"
 MINIO_USER="minioadmin"
 MINIO_PASS="minioadmin"
 MINIO_ARCHIVAL_BUCKET="archival.openem-dev.ethz.ch"
 MINIO_RETRIEVAL_BUCKET="retrieval.openem-dev.ethz.ch"
 
-# Mockarchiver
+## TUSD
+TUSD_HOOKS_FOLDER=./tusd/hooks
+
+## Mockarchiver
 LTS_FOLDER=/data/lts
 MOCKARCHIVER_IMAGE=ghcr.io/swissopenem/scopemarchiver-mockarchiver:latest
 
-# Webuploader
+## Webuploader
 WEBUPLOADER_IMAGE_NAME=ghcr.io/swissopenem/scopemarchiver-webuploader
-WEBUPLOADER_IMAGE_TAG=main
-
-# Celery
-CELERY_FLOWER_IMAGE_NAME=ghcr.io/swissopenem/scopemarchiver-flower
-CELERY_WORKER_IMAGE_NAME=ghcr.io/swissopenem/scopemarchiver-worker
-CELERY_SERVER_IMAGE_NAME=ghcr.io/swissopenem/scopemarchiver-server
-CELERY_IMAGE_TAG=main
-CELERY_ARCHIVING_DIR=/data/LTS
+WEBUPLOADER_IMAGE_TAG=main

README.md

@@ -1,18 +1,56 @@
-# ScopeMArchiver
+# OpenEM Service
 
-A archiving servce that allows uploading data and registering it with [SciCat](https://scicatproject.github.io).
+A ingester and archiver service that allows uploading data and registering it with [SciCat](https://scicatproject.github.io).
 
-## Mockarchiver
+## Development
+
+```bash
+docker compose --env_file .production.env --env-file .development.env up -d
+```
+
+> **Note:** .env files are picked up by VSCode and variables defined there are added to the shell that is used. This can lead to confusion as the files is not reloaded after changing values and the values in the session of the shell take precedence.
+
+## Archiver
+
+The archiver functionality can be enabled by setting the following env variable:
+
+```bash
+ARCHIVER_ENABLED=True
+```
+
+### Mockarchiver
 
 Python based service that mocks behavior of the LTS at ETH.
 See its [Readme](./mockarchiver/README.me) for details.
 
+## Ingester
+
+The ingester functionality can be enabled by setting the following env variable:
+
+```bash
+INGESTER_ENABLED=True
+```
+
 ## Deployment
 
-All the services can be deployed using docker compose:
+Production:
+
+```bash
+docker compose --env_file .production.env up -d
+```
+
+Development:
 
 ```bash
-docker compose up -d
+docker compose --env-file .production.env --env-file .development.env --profile archiver up -d
 ```
 
-> Note: Individual image tags need to be passed as env variables
+## Endpoints
+
+Deploying it locally for development provide the following endpoints
+
+| Service | Endpoint |
+|-- |-- |
+| Archiver Frontend | <http://localhost> |
+| Traefik | <http://traefik.localhost/dashboard/> |
+| Jobs API | <http://localhost/api/v1/docs> |

backend/.vscode/launch.json

@@ -0,0 +1,50 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Backend API",
+      "type": "debugpy",
+      "request": "launch",
+      "env": {
+        "ARCHIVER_ENABLED": "True",
+        "INGESTER_ENABLED": "True"
+      },
+      "module": "uvicorn",
+      "args": [
+        "main:app",
+        "--reload"
+      ],
+      "jinja": true
+    },
+    {
+      "name": "Worker",
+      "type": "debugpy",
+      "request": "launch",
+      "env": {
+        "ARCHIVER_ENABLED": "True",
+        "INGESTER_ENABLED": "True"
+      },
+      "module": "celery",
+      "args": [
+        "-A",
+        "jobsystem.celery",
+        "worker"
+      ],
+      "jinja": true
+    }
+  ],
+  "compounds": [
+    {
+      "name": "Backend",
+      "configurations": [
+        "Backend API",
+        "Worker"
+      ],
+      "preLaunchTask": "${defaultBuildTask}",
+      "stopAll": true
+    }
+  ]
+}

jobsystem/Pipfile → backend/Pipfile

@@ -13,6 +13,7 @@ uvicorn = "*"
 celery = {extras = ["redis"], version = "*"}
 pytest-celery = "*"
 pytest = "*"
+fastapi-featureflags = "*"
 
 [dev-packages]