✨(emails) use mjml to generate html and text emails

Introducing MJML, a responsive email framework, to reduce the pain of writing
responsive emails.

.circleci/config.yml

@@ -149,13 +149,15 @@ jobs:
           password: $DOCKER_HUB_PASSWORD
     working_directory: ~/fun
     steps:
-      - checkout
+      - checkout:
+          path: ~/fun
       - restore_cache:
           keys:
             - v1-dependencies-{{ .Revision }}
       - run:
           name: Install development dependencies
           command: pip install --user .[dev]
+          working_directory: src/app
       - save_cache:
           paths:
             - ~/.local
@@ -167,18 +169,19 @@ jobs:
         auth:
           username: $DOCKER_HUB_USER
           password: $DOCKER_HUB_PASSWORD
-    working_directory: ~/fun
+    working_directory: ~/fun/src/app
     steps:
-      - checkout
+      - checkout:
+          path: ~/fun
       - restore_cache:
           keys:
             - v1-dependencies-{{ .Revision }}
       - run:
           name: Lint code with black
-          command: ~/.local/bin/black src/mork tests --check
+          command: ~/.local/bin/black mork --check
       - run:
           name: Lint code with Ruff check
-          command: ~/.local/bin/ruff check .
+          command: ~/.local/bin/ruff check mork
 
   test:
     docker:
@@ -188,6 +191,7 @@ jobs:
           password: $DOCKER_HUB_PASSWORD
         environment:
           MORK_DB_HOST: localhost
+          PYTHONPATH: /home/circleci/fun
       - image: cimg/postgres:12.14
         auth:
           username: $DOCKER_HUB_USER
@@ -196,12 +200,16 @@ jobs:
           POSTGRES_DB: mork-db
           POSTGRES_USER: fun
           POSTGRES_PASSWORD: pass
-    working_directory: ~/fun
+    working_directory: ~/fun/src/app
     steps:
-      - checkout
+      - checkout:
+          path: ~/fun
+      - attach_workspace:
+          at: ~/fun
       - restore_cache:
           keys:
             - v1-dependencies-{{ .Revision }}
+      # Create test database
       - run: sudo apt-get update
       - run: sudo apt-get install postgresql-client
       - run: whoami
@@ -213,16 +221,48 @@ jobs:
           name: Run tests
           command: ~/.local/bin/pytest
 
+  # ---- Email jobs ----
+  build-mails:
+    docker:
+      - image: cimg/node:20.16.0
+        auth:
+          username: $DOCKER_HUB_USER
+          password: $DOCKER_HUB_PASSWORD
+    working_directory: ~/fun/src/mail
+    steps:
+      - checkout:
+          path: ~/fun
+      - restore_cache:
+          keys:
+            - v1-mail-dependencies-{{ .Revision }}
+      # If the yarn.lock file is not up-to-date with the package.json file,
+      # using the --frozen-lockfile should fail.
+      - run:
+          name: Install node dependencies
+          command: yarn install --frozen-lockfile
+      - run:
+          name: Build mails
+          command: yarn build
+      - persist_to_workspace:
+          root: ~/fun
+          paths:
+            - src/app/mork/templates
+      - save_cache:
+          paths:
+            - ./node_modules
+          key: v1-mail-dependencies-{{ .Revision }}
+
   # ---- Packaging jobs ----
   package:
     docker:
       - image: cimg/python:3.12
         auth:
           username: $DOCKER_HUB_USER
           password: $DOCKER_HUB_PASSWORD
-    working_directory: ~/fun
+    working_directory: ~/fun/src/app
     steps:
-      - checkout
+      - checkout:
+          path: ~/fun
       - restore_cache:
           keys:
             - v1-dependencies-{{ .Revision }}
@@ -235,7 +275,7 @@ jobs:
       - persist_to_workspace:
           root: ~/fun
           paths:
-            - dist
+            - src/app/dist
       # Store packages as artifacts to download/test them
       - store_artifacts:
           path: ~/fun/dist
@@ -402,6 +442,11 @@ workflows:
           filters:
             tags:
               only: /.*/
+      # Build mails
+      - build-mails:
+          filters:
+            tags:
+              only: /.*/
 
       # Docker jobs
       #
@@ -428,6 +473,7 @@ workflows:
       - test:
           requires:
             - build
+            - build-mails
           filters:
             tags:
               only: /.*/

.env.dist

@@ -29,6 +29,20 @@ MORK_CELERY_RESULT_BACKEND=redis://redis:6379/0
 MORK_CELERY_BROKER_TRANSPORT_OPTIONS={}
 MORK_CELERY_TASK_DEFAULT_QUEUE=celery
 
+# Emails
+MORK_EMAIL_HOST=mailcatcher
+MORK_EMAIL_HOST_USER=
+MORK_EMAIL_HOST_PASSWORD=
+MORK_EMAIL_PORT=1025
+MORK_EMAIL_USE_TLS=False
+[email protected]
+MORK_EMAIL_RATE_LIMIT=100/m
+MORK_EMAIL_MAX_RETRIES=3
+MORK_EMAIL_SITE_NAME="France Université Numérique"
+MORK_EMAIL_SITE_BASE_URL=https://fun-mooc.fr
+MORK_EMAIL_SITE_LOGIN_URL=https://lms.fun-mooc.fr/login
+
+
 # Python
 PYTHONPATH=/app
 
@@ -48,13 +62,3 @@ MYSQL_PASSWORD=password
 POSTGRES_DB=mork-db
 POSTGRES_USER=fun
 POSTGRES_PASSWORD=pass
-
-# Emails
-EMAIL_HOST=mailcatcher
-EMAIL_HOST_USER=
-EMAIL_HOST_PASSWORD=
-EMAIL_PORT=1025
-EMAIL_USE_TLS=False
-[email protected]
-EMAIL_RATE_LIMIT=100/m
-EMAIL_MAX_RETRIES=3

.gitignore

@@ -40,6 +40,12 @@ ENV/
 env.bak/
 venv.bak/
 
+# npm
+node_modules
+
+# Mails
+src/app/mork/templates/
+
 # Logs
 *.log
 

Dockerfile

@@ -9,31 +9,44 @@ RUN apt-get update && \
     apt-get -y upgrade && \
     rm -rf /var/lib/apt/lists/*
 
-
 # -- Builder --
 FROM base AS builder
 
 WORKDIR /build
 
-COPY . /build/
+# Copy required python dependencies
+COPY ./src/app /build
+
+RUN mkdir /install && \
+    pip install --prefix=/install .
+
+# ---- mails ----
+FROM node:20 AS mail-builder
 
-RUN pip install .
+COPY ./src/mail /mail/app
 
+WORKDIR /mail/app
 
+RUN yarn install --frozen-lockfile && \
+    yarn build
+
 # -- Core --
 FROM base AS core
 
-COPY --from=builder /usr/local /usr/local
+# Copy installed python dependencies
+COPY --from=builder /install /usr/local
+
+# Copy mork application (see .dockerignore)
+COPY ./src/app /app/
 
 WORKDIR /app
 
 
 # -- Development --
 FROM core AS development
 
-# Copy all sources, not only runtime-required files
-COPY . /app/
-
+# Switch to privileged user to uninstall app
+USER root:root
 
 # Uninstall mork and re-install it in editable mode along with development
 # dependencies
@@ -52,6 +65,9 @@ FROM core AS production
 ARG DOCKER_USER=1000
 USER ${DOCKER_USER}
 
+# Copy mork mails
+COPY --from=mail-builder /mail/app/mork/templates /app/src/app/mork/templates
+
 CMD ["uvicorn", \
      "mork.api:app", \
      "--proxy-headers", \

Makefile

@@ -2,10 +2,11 @@
 SHELL := /bin/bash
 
 # -- Docker
-COMPOSE         = bin/compose
-COMPOSE_EXEC    = $(COMPOSE) exec
-COMPOSE_RUN     = $(COMPOSE) run --rm --no-deps
-COMPOSE_RUN_API = $(COMPOSE_RUN) api
+COMPOSE          = bin/compose
+COMPOSE_EXEC     = $(COMPOSE) exec
+COMPOSE_RUN      = $(COMPOSE) run --rm --no-deps
+COMPOSE_RUN_API  = $(COMPOSE_RUN) api
+COMPOSE_RUN_MAIL = $(COMPOSE_RUN) mail-generator
 
 # -- MySQL
 EDX_DB_HOST = mysql
@@ -25,6 +26,8 @@ MORK_TEST_DB_NAME       ?= test-mork-db
 # -- Celery
 MORK_CELERY_SERVER_PORT		?= 5555
 
+# -- Mail
+MAIL_YARN = $(COMPOSE_RUN_MAIL) yarn
 
 # ==============================================================================
 # RULES
@@ -48,7 +51,10 @@ bootstrap: \
   .env \
   build \
   run \
-  seed-edx-database
+  migrate \
+  seed-edx-database \
+  mails-install \
+  mails-build
 .PHONY: bootstrap
 
 build: ## build the app containers
@@ -147,17 +153,17 @@ lint: \
 
 lint-black: ## lint python sources with black
 	@echo 'lint:black started…'
-	@$(COMPOSE_RUN_API) black --config pyproject.toml src/mork tests
+	@$(COMPOSE_RUN_API) black mork
 .PHONY: lint-black
 
 lint-ruff: ## lint python sources with ruff
 	@echo 'lint:ruff started…'
-	@$(COMPOSE_RUN_API) ruff check --config pyproject.toml .
+	@$(COMPOSE_RUN_API) ruff check .
 .PHONY: lint-ruff
 
 lint-ruff-fix: ## lint and fix python sources with ruff
 	@echo 'lint:ruff-fix started…'
-	@$(COMPOSE_RUN_API) ruff check --config pyproject.toml . --fix
+	@$(COMPOSE_RUN_API) ruff check . --fix
 .PHONY: lint-ruff-fix
 
 ## -- Tests
@@ -170,6 +176,25 @@ test: \
 .PHONY: test
 
 
+# -- Mail generator
+
+mails-build: ## Convert mjml files to html and text
+	@$(MAIL_YARN) build
+.PHONY: mails-build
+
+mails-build-html-to-plain-text: ## Convert html files to text
+	@$(MAIL_YARN) build-html-to-plain-text
+.PHONY: mails-build-html-to-plain-text
+
+mails-build-mjml-to-html:	## Convert mjml files to html and text
+	@$(MAIL_YARN) build-mjml-to-html
+.PHONY: mails-build-mjml-to-html
+
+mails-install: ## mail-generator yarn install
+	@$(MAIL_YARN) install
+.PHONY: mails-install
+
+
 # -- Misc
 help:
 	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

bin/alembic

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-bin/compose run --rm api alembic -c src/mork/alembic.ini "$@"
+bin/compose run --rm api alembic -c /app/mork/alembic.ini "$@"

docker-compose.yml

@@ -33,7 +33,7 @@ services:
       - "${MORK_SERVER_PORT:-8100}"
       - "--reload"
     volumes:
-      - .:/app
+      - ./src/app:/app
 
   celery:
     image: mork:development
@@ -43,7 +43,7 @@ services:
     ports:
       - "5555:5555"
     volumes:
-      - ./src:/app
+      - ./src/app:/app
       - ./bin/seed_edx_database.py:/opt/src/seed_edx_database.py
     depends_on:
       - api
@@ -60,6 +60,15 @@ services:
     ports:
       - "1081:1080"
 
+  mail-generator:
+    image: node:20
+    user: "${DOCKER_USER:-1000}"
+    environment:
+      HOME: /tmp
+    volumes:
+      - ".:/app"
+    working_dir: /app/src/mail
+
   mysql:
     image: mysql:5.7
     ports: