new deployment structure

.gitignore

@@ -73,3 +73,4 @@ docker/local-docker-compose-2.yml
 data/**
 fcrepo-import-export-*
 solr_db_initialized
+ops/*-deploy.yaml

.gitlab-ci.yml

@@ -160,6 +160,28 @@ hyku.staging:
   tags:
     - kubernetes
 
+hyku.production:
+  stage: go
+  only:
+    refs:
+      - main
+  when: manual
+  variables:
+    DEPLOY_IMAGE: $CI_REGISTRY_IMAGE
+    DEPLOY_TAG: $CI_COMMIT_SHORT_SHA
+    WORKER_IMAGE: $CI_REGISTRY_IMAGE/worker
+    HELM_EXPERIMENTAL_OCI: 1
+    HELM_RELEASE_NAME: hyku-production
+    KUBE_NAMESPACE: hyku-production
+    HELM_EXTRA_ARGS: >
+      --values ops/production-deploy.yaml
+  script:
+    - export KUBECONFIG=$KUBECONFIG_BL
+    - envsubst < ops/production-deploy.tmpl.yaml > ops/production-deploy.yaml
+    - ./bin/helm_deploy hyku-production hyku-production
+  tags:
+    - kubernetes
+
 hyku.staging.stop:
   stage: go
   extends:

.sops.yaml

@@ -0,0 +1,3 @@
+---
+creation_rules:
+  - pgp: "40B3DE7A02CAC8D1DE76223483DA3B754DD29AF9"

bin/decrypt-secrets

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+# require 'byebug'
+
+parent_dir = File.dirname(__dir__)
+Dir.chdir(File.join(parent_dir, 'ops', 'provision'))
+[
+  ".env.*",
+  "kube_config.yml",
+  ".backend",
+  "k8s/*-values.yaml"
+].each do |files|
+  Dir.glob(files).each do |file|
+    next if file.match(/enc/)
+    next if !File.exists?("#{file}.enc")
+    cmd = "sops --decrypt #{file}.enc > #{file}"
+    puts cmd
+    %x{#{cmd}}
+  end
+end

bin/encrypt-secrets

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+# require 'byebug'
+
+parent_dir = File.dirname(__dir__)
+Dir.chdir(File.join(parent_dir, 'ops', 'provision'))
+[
+  ".env.*",
+  "kube_config.yml",
+  ".backend",
+  "k8s/*-values.yaml"
+].each do |files|
+  Dir.glob(files).each do |file|
+    next if file.match(/enc/)
+    cmd = "sops --encrypt #{file} > #{file}.enc"
+    puts cmd
+    %x{#{cmd}}
+  end
+end

ops/production-deploy.tmpl.yaml

@@ -0,0 +1,203 @@
+replicaCount: 2
+
+livenessProbe:
+  enabled: false
+readinessProbe:
+  enabled: false
+
+brandingVolume:
+  storageClass: efs-sc
+derivativesVolume:
+  storageClass: efs-sc
+uploadsVolume:
+  storageClass: efs-sc
+  size: 200Gi
+
+extraVolumeMounts: &volMounts
+  - name: uploads
+    mountPath: /app/samvera/hyrax-webapp/tmp/imports
+    subPath: imports
+  - name: uploads
+    mountPath: /app/samvera/hyrax-webapp/tmp/exports
+    subPath: exports
+  - name: uploads
+    mountPath: /app/samvera/hyrax-webapp/public/system
+    subPath: public-system
+  - name: uploads
+    mountPath: /app/samvera/hyrax-webapp/public/uploads
+    subPath: public-uploads
+  - name: uploads
+    mountPath: /app/samvera/hyrax-webapp/tmp/network_files
+    subPath: network-files
+
+ingress:
+  enabled: true
+  hosts:
+    - host: oar.notch8.cloud
+      paths:
+        - path: /
+    - host: "*.oar.notch8.cloud"
+      paths:
+        - path: /
+  annotations: {
+    kubernetes.io/ingress.class: "nginx",
+    nginx.ingress.kubernetes.io/proxy-body-size: "0",
+    cert-manager.io/cluster-issuer: letsencrypt-production-dns
+  }
+  tls:
+    - hosts:
+        - oar.notch8.cloud
+        - "*.oar.notch8.cloud"
+      secretName: notch8cloud
+
+extraEnvVars: &envVars
+  - name: CONFDIR
+    value: "/app/samvera/hyrax-webapp/solr/config"
+  - name: DATABASE_ADAPTER
+    value: postgresql
+  - name: DATABASE_HOST
+    value: postgresql.default.svc.cluster.local
+  - name: DATABASE_NAME
+    value: hyku
+  - name: DATABASE_PASSWORD
+    value: $PROD_DATABASE_PASSWORD
+  - name: DATABASE_USER
+    value: postgres
+  - name: FCREPO_BASE_PATH
+    value: /bl
+  - name: FCREPO_HOST
+    value: fcrepo.default.svc.cluster.local:8080
+  - name: FCREPO_PATH
+    value: /rest
+  - name: FEDORA_URL
+    value: http://fcrepo.default.svc.cluster.local:8080/rest
+  - name: INITIAL_ADMIN_EMAIL
+    value: [email protected]
+  - name: INITIAL_ADMIN_PASSWORD
+    value: testing123
+  - name: IN_DOCKER
+    value: "true"
+  - name: LD_LIBRARY_PATH
+    value: /app/fits/tools/mediainfo/linux
+  - name: PASSENGER_APP_ENV
+    value: production
+  - name: RAILS_CACHE_STORE_URL
+    value: redis://:production@hyku-production-redis-master:6379/bl
+  - name: RAILS_ENV
+    value: production
+  - name: RAILS_LOG_TO_STDOUT
+    value: "true"
+  - name: RAILS_MAX_THREADS
+    value: "5"
+  - name: RAILS_SERVE_STATIC_FILES
+    value: "true"
+  - name: REDIS_HOST
+    value: hyku-production-redis-master
+  - name: REDIS_URL
+    value: redis://:production@hyku-production-redis-master:6379/bl
+  - name: HYRAX_ACTIVE_JOB_QUEUE
+    value: sidekiq
+  - name: HYKU_BULKRAX_ENABLED
+    value: "true"
+  - name: HYKU_CONTACT_EMAIL
+    value: [email protected]
+  - name: HYKU_FILE_ACL
+    value: "false"
+  - name: HYRAX_FITS_PATH
+    value: /app/fits/fits.sh
+  - name: HYKU_ADMIN_HOST
+    value: iro.bl.uk
+  - name: HYKU_ADMIN_ONLY_TENANT_CREATION
+    value: "true"
+  - name: HYKU_ALLOW_SIGNUP
+    value: "false"
+  - name: HYKU_DEFAULT_HOST
+    value: "%{tenant}.iro.bl.uk"
+  - name: HYKU_MULTITENANT
+    value: "true"
+  - name: HYKU_ROOT_HOST
+    value: iro.bl.uk
+  - name: HYKU_SMTP_SETTINGS
+    value: '{"from":"[email protected]","user_name":"apikey","password":"***REMOVED***","address":"smtp.sendgrid.net","domain":"bl.uk","port":"587","authentication":"plain","enable_starttls_auto":true}'
+  - name: SMTP_ADDRESS
+    value: smtp.sendgrid.net
+  - name: SMTP_DOMAIN
+    value: "bl.uk"
+  - name: SMTP_ENABLED
+    value: "true"
+  - name: SMTP_PASSWORD
+    value: "***REMOVED***"
+  - name: SMTP_PORT
+    value: "587"
+  - name: SMTP_USER_NAME
+    value: apikey
+  - name: SMTP_TYPE
+    value: plain
+  - name: SOLR_ADMIN_USER
+    value: admin
+  - name: SOLR_COLLECTION_NAME
+    value: hyku
+  - name: SOLR_CONFIGSET_NAME
+    value: hyku
+  - name: SOLR_HOST
+    value: solr.default.svc.cluster.local
+  - name: SOLR_PORT
+    value: "8983"
+  - name: SOLR_URL
+    value: http://admin:[email protected]:8983/solr/
+  - name: SECRET_KEY_BASE
+    value: 2b989efef38672467771269e8e430afebf55faf20da72e302e1893a3a5de22d9967b30b18d4ef369c9d34a6ca7315f99cb13fb98825aea4ee347a21be70f917e
+
+worker:
+  replicaCount: 1
+  extraVolumeMounts: *volMounts
+  extraEnvVars: *envVars
+  podSecurityContext:
+    runAsUser: 1001
+    runAsGroup: 101
+    fsGroup: 101
+    fsGroupChangePolicy: "OnRootMismatch"
+podSecurityContext:
+  runAsUser: 1001
+  runAsGroup: 101
+  fsGroup: 101
+  fsGroupChangePolicy: "OnRootMismatch"
+
+embargoRelease:
+  enabled: false
+leaseRelease:
+  enabled: false
+
+redis:
+  cluster:
+    enabled: true
+  password: production
+
+imagePullSecrets:
+  - name: gitlab-registry
+
+solr:
+  enabled: false
+  replicaCount: 2
+  collectionReplicas: 1
+  zookeeper:
+    replicaCount: 1
+
+fcrepo:
+  enabled: false
+  storage:
+    size: 105Gi
+
+postgresql:
+  enabled: false
+
+externalSolrHost: solr.default.svc.cluster.local
+externalSolrUser: admin
+externalSolrPassword: $PROD_SOLR_PASSWORD
+externalSolrCollection: "hyku"
+externalFcrepoHost: fcrepo.default.svc.cluster.local
+externalPostgresql:
+  username: postgres
+  password: $PROD_DATABASE_PASSWORD
+  database: hyku
+  host: postgresql.default.svc.cluster.local

ops/provision/.backend.enc

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:+FavHueQv9mq/CCZKOZu1O9hAJsr490MndPAIj0dapUUJj9US4wuT9j9zy1uZgV7YxdAw6HluwBfGZSRps9VYP+9EVxFX0N98Ghykpe9MWZG/8aiHtu7wIMPyGTUUnAN/WLdNrumxBr77KL67II2J4n85cXd,iv:r3wOvzDRDWrDt6P3HkIIJHB4r9ns0gDTM/XxOUPAIPo=,tag:vjBcaHDZF4HjBKfLUCLrow==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2022-02-21T20:48:08Z",
+		"mac": "ENC[AES256_GCM,data:48+JeAKvcvFqErSruhWSJcRZaLLQ29u7NyLez/SyAd4ohvJaK7aJfN1jbOBEu6Q2pskbB+f5zC26zcgHgvqziQDPUz45ITib+wvLYTB470XpKMmMFL6ciC3j1kyRrYMTcGyqz2MbpWaUsS2eYootnnEzobnMrJEatjvMlxjf9rY=,iv:pYXhib6tsGjtmxXyCzvL/dhIn90ZluorA3Hp6iumYCs=,tag:2/Gjjrn/hW6QmCS/D2rEYA==,type:str]",
+		"pgp": [
+			{
+				"created_at": "2022-02-21T20:48:07Z",
+				"enc": "-----BEGIN PGP MESSAGE-----\n\nwcFMA7doaBdjNJH+ARAAeaeLJEYR27okMjBTqQCi2/8MQlP6YEaetjCsGvLD4L2e\nCL5mnqTBBJw2NgP7THKqHhc+1Om92FQ0yzsQgvJ4fxIhs2lICICcaXEn8Taoeq5e\nPPKukSmdJrI+HwwtYL4kva4Qkxf72d62LIaHWd9Ul0lpawqdVoroUrfwhtGfG1iD\nb05a4jffwRColozopU80oXk/qSy9P9CLkhGNbKHNZVJg/8zc1WPS6z3wOYZas+bt\nr1HT35Bgb82Bpke/2d+7RIJE12l9doio2jnHYr2DolGopwbcIZZZLap4/yOK1qie\n3MpSbQ3Q9xOQfCNqRkRDFnHqoaa4kGB34mEYjopwKyd5cmMa4JYLkd+0T+bp+iJo\n0i0/YS1w0LwrahYRBGh08Nmofwgw5Ys9mWQFR5qIs5S199Yg7kR5f0VfGlBpx3tO\niQGvk6yIDmIQsZiQXCJAgTuKw7TVQYuZcUuBNoqffT3dKEdB0cJpLhGINWF4NCmy\nkYWXrPIjNBmnO6wLWlFc0azwvDhEfP4wnmvjBxAVD/lE9Pp0Dq2IRXZL/ZZXqOp2\nvzS4t/Z/kVWNNDVpZfBgqcVU/OshBpKqQpn6kLV3EYV4KDSMaQOBnB9yTmtkcs/H\nfgAVaXZ8hTYzAr5gHkDX5CytAp9wOOde9SWuuXx/00JkWDK7vk9iGGSXGrPED7DS\n5gGrdqbT+uzx4yYxwB59vEdnSyw+IBNKYHyC+ouQyxBpZOR9Y4wXtEaxHUgj0Zs5\nNgcYv0nXw5YSw5dNkX0gpxDkgguKg7wD2jN2R/W4pEZN0+J3JOVKAA==\n=8Flw\n-----END PGP MESSAGE-----",
+				"fp": "40B3DE7A02CAC8D1DE76223483DA3B754DD29AF9"
+			}
+		],
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.1"
+	}
+}

ops/provision/.gitignore

@@ -0,0 +1,4 @@
+kube_config.yml
+k8s/*-values.yaml
+.backend
+.terraform*

ops/provision/efs_name

@@ -0,0 +1 @@
+fs-0f75c2b025a620200

ops/provision/k8s/fcrepo-values.yaml.enc

@@ -0,0 +1,58 @@
+replicaCount: ENC[AES256_GCM,data:iA==,iv:veTqWCc7z0UCRzdYoT7AAghEs6t/H9h8mucZuybWdYw=,tag:opJPFPSUdT39SQKcsuQ+Zw==,type:int]
+storage:
+    enabled: ENC[AES256_GCM,data:wtCyeQ==,iv:co3c4yz4U15JLWXkMP34E4ABSLV4RUVquyC3gDPPNHE=,tag:tiIoQm7NQAd78ejAqiUAEg==,type:bool]
+    size: ENC[AES256_GCM,data:SsFvlw==,iv:a06VOxc+kkROY6vIZcE5Im/84X1WMZJRz5Nn7iafY18=,tag:2b2pzeV242qSATTcLP3a+g==,type:str]
+    className: ENC[AES256_GCM,data:+BjW,iv:dDrlmvZrmyoqhe4JWlDuVGiTpbSm6FdAA+LSsDYouUE=,tag:OAPRB53/3hneBwg/fjD+PQ==,type:str]
+image:
+    repository: ENC[AES256_GCM,data:0ajssKMOOlaTcILEuDbO,iv:eZz5FEoEn2VzFRiNLPki2J+QfdwqmMxbhHFD5DnmioE=,tag:xlTskKO7EUagzNt4ZBMzLg==,type:str]
+    pullPolicy: ENC[AES256_GCM,data:rMNzR9nChM/HwQQ2,iv:uhnJAvJuQD6jSdXCT5eVnQHNY9ZxKETq0uOF6TfPkCg=,tag:59i4BD0EyibED3UgrAbx6w==,type:str]
+    tag: ENC[AES256_GCM,data:gX+72og=,iv:GuFaddMuV+3qSgis1Y88WecMl7DEtERVw4m3dtxcpQo=,tag:AVEZnZX/nxtzIH3fe/4S5g==,type:str]
+postgresql:
+    enabled: ENC[AES256_GCM,data:a0lmXX8=,iv:qF8VWOTRmRvEkRz0Rk87mBCmwauhUjIknSqCKK1czRg=,tag:4v6prqCEcXvfbBxmMJioJQ==,type:bool]
+externalPostgresql:
+    host: ENC[AES256_GCM,data:DP+T/iDGQm/4COOkIRW0byee,iv:sqLCmzg5iaGdh/Vr6JjOadtOyStgeN370vGnP1XMXew=,tag:cPx6mCnucLW+fop2m76pWQ==,type:str]
+    #ENC[AES256_GCM,data:qfwMWU3U+5zr5A8Tte+7dxaNijtW8JQAr34brmK/iHFOqA==,iv:lcH+r4HE5mQsv2iFGZi6B7BoqUr3Y4W8B1j2MvCtdIg=,tag:Pj3a0IrhFgtPI2VphVZGFA==,type:comment]
+    username: ENC[AES256_GCM,data:xz1M5w1H,iv:MRMo7/arY7A7LLi+946/RRK2r4tzOjK7eMVftDx+djY=,tag:jFZIle/K+JZEf4G9khlSfg==,type:str]
+    password: ENC[AES256_GCM,data:1DpDz87UZ7vtwg/6YOobSD4C,iv:pw3GKeQrq+8qczGOKMxvJkWuqFvbkKvH/0bj6qkexCM=,tag:SPCKacvEKJKAPgZ4uoiAqw==,type:str]
+s3:
+    enabled: ENC[AES256_GCM,data:kQfOvgM=,iv:ID9fz46mlh8UAksT/hBjN3Udl3nJfm3dhYlokoqJD48=,tag:+PYwx/80yAiO3o+Y+WCbyA==,type:bool]
+    bucket: ENC[AES256_GCM,data:dR7RbW01dopBKBys0ct9eIM=,iv:T8Zd7yXemLlgugqt9KRjlByTkYCzZeffh9Ip8XlpFaM=,tag:Lq2sMiCD7zizlY2UERvVMQ==,type:str]
+    access_key: ENC[AES256_GCM,data:fUDfLB4f3ZNBnB0NAHYMhARBD20=,iv:lqwIgND47P8ZLaH82VP//08fYjAi8p2UUqiVQZGWKJE=,tag:8LPiiwhSOmMzx+0z9QCkbg==,type:str]
+    secret_key: ENC[AES256_GCM,data:ugLr/k4RWXmpd5GEaYFEMgugkhDvIH/ft9wbcaDDiqfQjM/kY4L09Q==,iv:ydVu2D88zf5hY08pFrzkcdVb8UP7iyCFzBUhzTnwZq4=,tag:FMeQ2bLaDzEqvOPIKzdzxA==,type:str]
+resources:
+    limits:
+        memory: ENC[AES256_GCM,data:KI2N,iv:vsifSno0RviAGhGfk4afncmmn9/O9XV2xM4qq9+6m5k=,tag:d+EWYTp9eMhHDoahPNcR9Q==,type:str]
+    requests:
+        memory: ENC[AES256_GCM,data:i296,iv:jAu/Yc1WphLsj7c85Zibw0HRb3QLsfHIDeoFtopDYdI=,tag:SJXuhBLEfHF/VIBKoNqPCQ==,type:str]
+additional_java_options: ENC[AES256_GCM,data:bGWhqGkXGXVuIKBOqNpBr+rAoa/WiPdYIBws3nCm4kB3uHUyGEl1F69xWmjkR302vSonF0dDyooVpIgok7Dw2TU3DaRZuRsG5ao+jvgz2Fnhe0K34qLpeMzVySSl2y3G9+I4eRy+DY7CFhrwlErvCDe9HMgNSwoZ9C+nWro3FZMy1DDKX/nxMhBgR84769U1eNQMDwDBehVq/ZUhXymR6sczmw6x+CN5m7oz2pBSjlnY03A=,iv:ra/GqZMDtA0hK4wVuIB5REOyC4f7u/OcZubh3qtlHwY=,tag:NJS2k9L+zMcqRLfTyMjetg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2022-02-21T20:48:12Z"
+    mac: ENC[AES256_GCM,data:Td1C3qH/5mM7wqRJ2SJg8b77lNR3pro2M+m7YORjo8nn7FaYEVsjWhASGrRvLzLCm+M+VXxUV4rCj/H9OymqCuoc/6i11i6x07JFyQCgx3IdRgcsbvTCY8Q42LJ1l+nc+immZ0UWBN8Nux/gByRf48rV/+DRtJaZktXIcB71gSQ=,iv:0woVclaYa2F3It9ZnCg5mO8gYBKeKWWl2Xo0C13p9jo=,tag:Hs46y+MBM+OpZ0Rpsmg1Ew==,type:str]
+    pgp:
+        - created_at: "2022-02-21T20:48:11Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA7doaBdjNJH+ARAAeDYsetszKHoCO7zkA/heD9cSqDCyaKqe/HtpHVRWc2wm
+            OtFV8/eacPBiyubvrsbwmC5dsPl+8S9PTvPDUbXNtuMUeEZglUEE7KkMkgXlgxpE
+            AvUBGqQz2aOR9psK3Cn5tpD/5vXGt+/oUFaIvOs4zVazZENV01pZdl/SSMc2PHeu
+            d/wfVSvCbyOsX3tIhOqYrDLRu1q+KqG7Hw0JEpsrZuuvRPZ/36H/h3+sVUDxIlmC
+            NvKhryQDSKRSx2BKcG8cCoSFuXJwB+x+aanxATzzCUuXVaXPwwJQlZX3PRX2OrXj
+            szC0PvotZEjTnWJQMIXvt39E0dHZyT3+FIGdAUuDnRMNRR/ntWFFYHXEjyzCHz39
+            6BbUuN1c7SbBSECLwuiaCXeeQA3cxFu2sO4fbeE249OKYiYKskAQX1e6ucC7SNEl
+            ha0/YDPW4DGmW9i7rk0OoN2y5XjSSHONa2sffjjXVXxLo8wyCQ/heheKuVwYILBu
+            uKSTgFrDap2HkhyrnYyrL4eMV3OEpFRoOxnlfqaaGfqAMt+4dAYpD+6HtaHXMvtv
+            lkppBqzIiZ5w6zy0cJGPQaj2BSMCNM4nWSsQ/wF2ZNnswlvYBXU75dowPZCo/lTu
+            tjSfdH+Q72CHUe0fS08Hiam6Btz0Km1kfvUgFX6MFUU9BWzPsFX/XYutAvFGRvnS
+            5gGQCxNPpGj0kxxIr0oPvLeKuluj4KIlKPjMOEwZZl4LXWoGVXHGcE6R4ctHHqSV
+            ak5rWbt5tzB11EAEk82axSHki/3oo2jBg8c3uI6GfgPL9OJ/qr0YAA==
+            =WNIJ
+            -----END PGP MESSAGE-----
+          fp: 40B3DE7A02CAC8D1DE76223483DA3B754DD29AF9
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1