[WIP] Deploy Jobs Release (local branch)

.github/workflows/scicat-be-next.yml

@@ -1,59 +1,59 @@
-# name: scicat-be-next
+name: scicat-be-next
 
-# on:  
-#   workflow_dispatch: 
-#     inputs:
-#       submodule_commit:
-#         description: 'Commit of the submodule to deploy'     
-#         required: false
-#       commit: 
-#         description: 'Commit of the CI repo to deploy'     
-#         required: false
-#   pull_request:
-#     branches: [ main ]
-#   push:
-#     branches: [ main ]
-#   release:
-#     types: [ published ]
+on:
+  workflow_dispatch:
+    inputs:
+      submodule_commit:
+        description: 'Commit of the submodule to deploy'
+        required: false
+      commit:
+        description: 'Commit of the CI repo to deploy'
+        required: false
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+  release:
+    types: [ published ]
 
-# jobs:
+jobs:
 
-#   set_env: 
-#     uses: ./.github/workflows/reusable.environment.yml
-#     with: 
-#       commit: ${{ github.event.inputs.commit }}
-#       submodule_commit: ${{ github.event.inputs.submodule_commit }}
+  set_env:
+    uses: ./.github/workflows/reusable.environment.yml
+    with:
+      commit: ${{ github.event.inputs.commit }}
+      submodule_commit: ${{ github.event.inputs.submodule_commit }}
 
-#   check_changed:
-#     needs: set_env
-#     uses: ./.github/workflows/reusable.changes.yml
-#     with: 
-#       files: | 
-#         .github/workflows/scicat-be-next.yml
-#         helm/configs/backend-next/${{ needs.set_env.outputs.environment }}/**
-#         helm/configs/backend-next/values.yaml
-#         scicat-backend-next/**
-#       commit: ${{ needs.set_env.outputs.commit }}
-#       submodule_commit: ${{ github.event.inputs.submodule_commit }}
-#       submodule: scicat-backend-next
+  check_changed:
+    needs: set_env
+    uses: ./.github/workflows/reusable.changes.yml
+    with:
+      files: |
+        .github/workflows/scicat-be-next.yml
+        helm/configs/backend-next/${{ needs.set_env.outputs.environment }}/**
+        helm/configs/backend-next/values.yaml
+        scicat-backend-next/**
+      commit: ${{ needs.set_env.outputs.commit }}
+      submodule_commit: ${{ github.event.inputs.submodule_commit }}
+      submodule: scicat-backend-next
 
-#   build_deploy_scicat_backend_next:
-#     if: (needs.check_changed.outputs.changed == 'true' && !needs.set_env.outputs.component) || needs.set_env.outputs.component == 'bn'
-#     needs: 
-#      - check_changed
-#      - set_env
-#     uses: ./.github/workflows/reusable.build-deploy.yml
-#     with: 
-#       context: scicat-backend-next/.
-#       image_name: ${{ github.repository }}/backend-next
-#       release_name: backend-next
-#       tag: ${{ needs.set_env.outputs.tag }}
-#       environment: ${{ needs.set_env.outputs.environment }}
-#       commit: ${{ needs.set_env.outputs.commit }}
-#       submodule_commit: ${{ github.event.inputs.submodule_commit }}
-#       submodule: scicat-backend-next
-#     secrets:
-#       KUBECONFIG: ${{ secrets.KUBECONFIG }}
-#       JSON_SECRETS: ${{ toJSON(secrets) }}
-#       BASTION_USER: ${{ secrets.BASTION_USER }}
-#       BASTION_SSH_KEY: ${{ secrets.BASTION_SSH_KEY }}
+  build_deploy_scicat_backend_next:
+    if: (needs.check_changed.outputs.changed == 'true' && !needs.set_env.outputs.component) || needs.set_env.outputs.component == 'bn'
+    needs:
+     - check_changed
+     - set_env
+    uses: ./.github/workflows/reusable.build-deploy.yml
+    with:
+      context: scicat-backend-next/.
+      image_name: ${{ github.repository }}/backend-next
+      release_name: backend-next
+      tag: ${{ needs.set_env.outputs.tag }}
+      environment: ${{ needs.set_env.outputs.environment }}
+      commit: ${{ needs.set_env.outputs.commit }}
+      submodule_commit: ${{ github.event.inputs.submodule_commit }}
+      submodule: scicat-backend-next
+    secrets:
+      KUBECONFIG: ${{ secrets.KUBECONFIG }}
+      JSON_SECRETS: ${{ toJSON(secrets) }}
+      BASTION_USER: ${{ secrets.BASTION_USER }}
+      BASTION_SSH_KEY: ${{ secrets.BASTION_SSH_KEY }}

.github/workflows/scicat-fe-next.yml

@@ -0,0 +1,63 @@
+name: scicat-fe-next
+
+on:
+  workflow_dispatch:
+    inputs:
+      submodule_commit:
+        description: 'Commit of the submodule to deploy'
+        required: false
+      commit:
+        description: 'Commit of the CI repo to deploy'
+        required: false
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+  release:
+    types: [ published ]
+
+jobs:
+
+  set_env:
+    uses: ./.github/workflows/reusable.environment.yml
+    with:
+      commit: ${{ github.event.inputs.commit }}
+      submodule_commit: ${{ github.event.inputs.submodule_commit }}
+
+  check_changed:
+    needs: set_env
+    uses: ./.github/workflows/reusable.changes.yml
+    with:
+      files: |
+        .github/workflows/scicat-fe-next.yml
+        helm/configs/frontend-next/values.yaml
+        helm/configs/frontend-next/login.component.html
+        helm/configs/frontend-next/${{ needs.set_env.outputs.environment }}/**
+        frontend/**
+      commit: ${{ needs.set_env.outputs.commit }}
+      submodule_commit: 6a220b1d8b7b257c215b2f2ef2a24e5b702e3723
+      submodule: frontend
+
+  build_deploy_scicat_fe_next:
+    if: (needs.check_changed.outputs.changed == 'true' && !needs.set_env.outputs.component) || needs.set_env.outputs.component == 'fe'
+    needs:
+     - check_changed
+     - set_env
+    uses: ./.github/workflows/reusable.build-deploy.yml
+    with:
+      context: frontend/.
+      image_name: ${{ github.repository }}/fe
+      release_name: frontend-next
+      tag: ${{ needs.set_env.outputs.tag }}
+      environment: ${{ needs.set_env.outputs.environment }}
+      commit: ${{ needs.set_env.outputs.commit }}
+      submodule_commit: 6a220b1d8b7b257c215b2f2ef2a24e5b702e3723
+      submodule: frontend
+      helm_set_files: >-
+        KEYCLOAK_ICON=helm/configs/frontend-next/keycloak_icon_256px.svg
+        ENVIRONMENT=helm/configs/frontend-next/${{ needs.set_env.outputs.environment }}/config.json
+    secrets:
+      KUBECONFIG: ${{ secrets.KUBECONFIG }}
+      BASTION_USER: ${{ secrets.BASTION_USER }}
+      BASTION_SSH_KEY: ${{ secrets.BASTION_SSH_KEY }}
+      JSON_SECRETS: ${{ toJSON(secrets) }}

dev/config/backend_next/.env

@@ -16,5 +16,6 @@ CREATE_DATASET_GROUPS=group1,group2,group3
 DELETE_GROUPS=archivemanager
 ACCESS_GROUPS_STATIC_VALUES="ess"
 CREATE_DATASET_WITH_PID_GROUPS="group2,group3"
+JOB_CONFIGURATION_FILE=jobConfig.yaml
 DATASET_CREATION_VALIDATION_ENABLED=true
 DATASET_CREATION_VALIDATION_REGEX="^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$"

dev/config/backend_next/jobConfig.yaml

@@ -0,0 +1,70 @@
+configVersion: v1.0 2024-03-01 6f3f38
+jobs:
+  - jobType: archive
+    create:
+      auth: "#datasetOwner"
+      actions:
+        - actionType: log
+        - actionType: url
+          url: http://localhost:3000/api/v3/health?jobid={{id}}
+          headers:
+            accept: application/json
+        - actionType: rabbitmq
+          hostname: rabbitmq
+          port: 5672
+          username: guest
+          password: guest
+          exchange: jobs.write
+          queue: client.jobs.write
+          key: jobqueue
+        - actionType: email
+          auth:
+            user: user
+            password: password
+          to: "{{contactEmail}}"
+          from: from
+          subject: "[SciCat] Your {{type}} job was submitted successfully"
+          bodyTemplateFile: src/common/email-templates/job-template-simplified.html
+    statusUpdate:
+      auth: archivemanager
+      actions:
+        - actionType: rabbitmq
+          hostname: rabbitmq
+          port: 5672
+          username: guest
+          password: guest
+          exchange: jobs.write
+          queue: client.jobs.write
+          key: jobqueue
+  - jobType: public
+    create:
+      auth: "#datasetPublic"
+      actions:
+        - actionType: validate
+          request:
+            jobParams.datasetList[*]:
+              type: object
+              required:
+                - pid
+                - files
+    statusUpdate:
+      auth: archivemanager
+  - jobType: validate
+    create:
+      auth: admin
+      actions:
+        - actionType: validate
+          request:
+            jobParams.requiredParam:
+              type: string
+            jobParams.arrayOfStrings:
+              type: array
+              items:
+                type: string
+    statusUpdate:
+      auth: admin
+      actions:
+        - actionType: validate
+          request:
+            jobResultObject.requiredString:
+              type: string

dev/config/frontend/config.json

@@ -114,7 +114,7 @@
         {
             "displayText": "PSI account",
             "displayImage": "../../../assets/images/keycloak_icon_256px.svg",
-            "authURL": "auth/keycloak"
+            "authURL": "api/v3/auth/oidc"
         }
     ],
     "helpMessages": {

dev/docker-compose.yaml

@@ -78,7 +78,7 @@ services:
     profiles:
       - fe
 
-  be_next_base: 
+  be_next_base:
     build:
       context: ../scicat-backend-next/.
       target: dev
@@ -89,10 +89,11 @@ services:
       - /home/node/app/node_modules
       - /home/node/app/dist
       - ../scicat-backend-next/functionalAccounts.json.minimal.example:/home/node/app/functionalAccounts.json
+      - ./config/backend_next/jobConfig.yaml:/home/node/app/jobConfig.yaml
     env_file: ./config/backend_next/.env
     command: /bin/sh -c "while true; do sleep 600; done"
     profiles:
-      - never 
+      - never
 
   test_be_next:
     extends: be_next_base

helm/configs/backend-next/development/values.yaml

@@ -1,3 +1,7 @@
 envValue: development
 
 host: scicat.development.psi.ch
+
+ingress:
+  annotations: 
+    b64/nginx.ingress.kubernetes.io/whitelist-source-range: "{{ .Values.secretsJson.OPENEM_WHITELISTED_IPS }}"

helm/configs/backend-next/values.yaml

@@ -16,22 +16,23 @@ volumes:
     secret:
       secretName: "{{ .Release.Name }}-s"
 
-secrets: 
+secrets:
   "{{ .Release.Name }}-s":
     type: Opaque
-    data: 
+    data:
       .env: "{{ .Values.secretsJson.BENEXT_ENV }}"
       functionalAccounts.json: "{{ .Values.secretsJson.BENEXT_FUNCTIONAL_ACCOUNTS }}"
+      jobConfig.yaml: "{{ .Values.secretsJson.BENEXT_JOBCONFIG }}"
 
 ingress:
   enabled: true
-  annotations: 
+  annotations:
     kubernetes.io/ingress.class: nginx
     cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/proxy-body-size: 50m
   hosts:
     - host: "{{ .Values.host }}"
-      paths: 
+      paths:
         - path: "/"
           pathType: Prefix
   tls:
@@ -46,6 +47,8 @@ env:
     value: "20.500.11935"
   - name: SITE
     value: "PSI"
+  - name: JOB_CONFIGURATION_FILE
+    value: /home/node/app/jobConfig.yaml
 
 volumeMounts:
   - name: secrets-volume
@@ -54,3 +57,6 @@ volumeMounts:
   - name: secrets-volume
     mountPath: /home/node/app/functionalAccounts.json
     subPath: functionalAccounts.json
+  - name: secrets-volume
+    mountPath: /home/node/app/jobConfig.yaml
+    subPath: jobConfig.yaml