Merge pull request #711 from NBISweden/feature/yearly-report2

Yearly report module with admin interface

.docker/frontend

@@ -1,4 +1,4 @@
-FROM node:17.6-slim as build
+FROM node:18-slim AS build
 
 USER node
 RUN mkdir /home/node/app

.github/workflows/create_packages.yml

@@ -1,13 +1,47 @@
+---
 name: Create packages
-on: push
+on:
+  push:
+    paths:
+      - "frontend/**" # Frontend changes
+      - "app/**" # Main app changes
+      - "r-api/**" # R API changes
+      - "docker/**" # Docker related changes
+      - ".docker/**" # Docker config changes
+      - "docker-compose.yml" # Docker compose changes
+      - ".github/workflows/create_packages.yml" # Workflow changes
 
 jobs:
-  main:
+  changes:
     runs-on: ubuntu-latest
+    outputs:
+      frontend: ${{ steps.filter.outputs.frontend }}
+      main: ${{ steps.filter.outputs.main }}
+      r-api: ${{ steps.filter.outputs.r-api }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            frontend:
+              - 'frontend/**'
+              - 'docker/frontend.Dockerfile'
+            main:
+              - 'app/**'
+              - 'docker/main.Dockerfile'
+              - '.docker/main'
+            r-api:
+              - 'r-api/**'
+              - 'docker/r-api.Dockerfile'
 
+  build:
+    needs: changes
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Update tags.json version
         uses: jossef/[email protected]
@@ -16,36 +50,76 @@ jobs:
           field: gitBranch
           value: ${{github.ref_name}}
 
-      - name: Build container
-        env:
-          GHCR_PAT: ${{ secrets.GHCR_PAT }}
-          GHCR_USER: ${{ secrets.GHCR_USER }}
-          DOCKER_BUILDKIT: 1
-          COMPOSE_DOCKER_CLI_BUILD: 1
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USER }}
+          password: ${{ secrets.GHCR_PAT }}
+
+      - name: Prepare .docker files
         run: |
-          echo "${GHCR_PAT}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin
+          for p in .docker/*.default ; do
+            mv "$p" "${p%.default}"
+          done
 
-          branch=${GITHUB_REF##*/}
+      # Pull existing images for layer caching
+      - name: Pull existing images
+        run: |
+          # Sanitize branch name for docker tags
+          branch="${{ github.ref_name }}"
+          safe_branch="${branch//\//-}"  # Replace / with -
 
           for image in herdbook_r-api herdbook_main herdbook_frontend; do
             docker pull "ghcr.io/nbisweden/$image:latest" || true
-            docker pull "ghcr.io/nbisweden/$image:$branch" || true
+            docker pull "ghcr.io/nbisweden/$image:$safe_branch" || true
           done
 
-          for p in .docker/*.default ; do
-            mv "$p" "${p%.default}"
-          done
+      # Build images using docker compose
+      - name: Build images
+        env:
+          DOCKER_BUILDKIT: 1
+          COMPOSE_DOCKER_CLI_BUILD: 1
+        run: |
+          # Only build changed images and their dependents
+          if [[ "${{ needs.changes.outputs.frontend }}" == "true" ]]; then
+            docker compose build --build-arg BUILDKIT_INLINE_CACHE=1 herdbook-frontend main
+          elif [[ "${{ needs.changes.outputs.main }}" == "true" ]]; then
+            docker compose build --build-arg BUILDKIT_INLINE_CACHE=1 main
+          fi
 
-          docker-compose build --build-arg BUILDKIT_INLINE_CACHE=1
+          if [[ "${{ needs.changes.outputs.r-api }}" == "true" ]]; then
+            docker compose build --build-arg BUILDKIT_INLINE_CACHE=1 r-api
+          fi
 
-          for image in herdbook_r-api herdbook_main herdbook_frontend; do
-            docker tag "$image:latest" "ghcr.io/nbisweden/$image:$branch"
-            docker push "ghcr.io/nbisweden/$image:$branch"
+      # Push images
+      - name: Push images
+        run: |
+          # Sanitize branch name for docker tags
+          branch="${{ github.ref_name }}"
+          safe_branch="${branch//\//-}"  # Replace / with -
 
-            for mb in main master develop; do
-              if [ "$mb" = "$branch" ]; then
+          # Function to push an image if it was built
+          push_if_built() {
+            local image="$1"
+            if docker image inspect "$image:latest" >/dev/null 2>&1; then
+              docker tag "$image:latest" "ghcr.io/nbisweden/$image:$safe_branch"
+              docker push "ghcr.io/nbisweden/$image:$safe_branch"
+              
+              if [[ "$branch" == "main" || "$branch" == "master" || "$branch" == "develop" ]]; then
                 docker tag "$image:latest" "ghcr.io/nbisweden/$image:latest"
                 docker push "ghcr.io/nbisweden/$image:latest"
               fi
-            done
-          done
+            fi
+          }
+
+          # Push only the images that were built
+          if [[ "${{ needs.changes.outputs.frontend }}" == "true" ]]; then
+            push_if_built "herdbook_frontend"
+          fi
+          if [[ "${{ needs.changes.outputs.main }}" == "true" || "${{ needs.changes.outputs.frontend }}" == "true" ]]; then
+            push_if_built "herdbook_main"
+          fi
+          if [[ "${{ needs.changes.outputs.r-api }}" == "true" ]]; then
+            push_if_built "herdbook_r-api"
+          fi

.github/workflows/lint.yml

@@ -16,41 +16,39 @@ name: Lint Code Base
 #############################
 on:
   push:
-    branches-ignore: [master,develop]
+    branches-ignore: [master, develop]
     # Remove the line above to run when pushing to master
   pull_request:
-    branches: [master,develop]
+    branches: [master, develop]
 
 ###############
 # Set the Job #
 ###############
+permissions: {}
+
 jobs:
   build:
-    # Name the Job
-    name: Lint Code Base
-    # Set the agent to run on
+    name: Lint
     runs-on: ubuntu-latest
 
-    ##################
-    # Load all steps #
-    ##################
+    permissions:
+      contents: read
+      packages: read
+      # To report GitHub Actions status checks
+      statuses: write
+
     steps:
-      ##########################
-      # Checkout the code base #
-      ##########################
-      - name: Checkout Code
-        uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v4
         with:
-          # Full git history is needed to get a proper list of changed files within `super-linter`
+          # super-linter needs the full git history to get the
+          # list of files that changed across commits
           fetch-depth: 0
 
-      ################################
-      # Run Linter against code base #
-      ################################
-      - name: Lint Code Base
-        uses: github/super-linter/slim@v4
+      - name: Super-linter
+        uses: super-linter/super-linter/slim@v7 # x-release-please-version
         env:
+          # To report GitHub Actions status checks
           VALIDATE_ALL_CODEBASE: false
           DEFAULT_BRANCH: develop
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-

.gitignore

@@ -13,6 +13,7 @@ ENV/
 env.bak/
 venv.bak/
 .vscode
+*.code-workspace
 .tox/
 .docker/database-variables.env
 .docker/r-api-variables.env
@@ -37,6 +38,7 @@ __pycache__/
 # database files
 postgres-data/
 db_dumps/
+db-dump*
 
 # config files
 config.ini
@@ -51,3 +53,9 @@ s3data/*
 *.csv
 *.dump
 *.xlsx
+
+# Logs
+logs/*
+
+#
+cache/*