Merge branch 'main' into main

Dockerfile

@@ -1,9 +1,24 @@
 FROM python:3.10
-COPY drp_aa_mvp/requirements.txt /requirements.txt
-RUN pip install -r /requirements.txt
-COPY docker-entrypoint.sh /entrypoint.sh
+
+ARG USER=osiraa
+ENV user=${USER}
+
+RUN groupadd -g 1000 ${USER} && \
+    useradd -m -u 1000 -g ${USER} -s /bin/bash ${USER} && \
+    mkdir -p /code && \
+    chown -R ${USER}:${USER} \
+    /code 
+
+
+COPY --chown=${USER}:${USER} drp_aa_mvp/requirements.txt /requirements.txt
+COPY --chown=${USER}:${USER} docker-entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
-COPY drp_aa_mvp/ /code
+COPY --chown=${USER}:${USER} drp_aa_mvp/ /code
+
+USER ${USER}
 WORKDIR /code
+
+RUN pip install -r /requirements.txt
+
 CMD "/entrypoint.sh"
 EXPOSE 8000:8000

Jenkinsfile

@@ -0,0 +1,183 @@
+@Library(['k8s-jenkins-il-shared-lib@master', 'k8s-jenkins-common-shared-lib@master']) _
+import com.consumerreports.common.Config
+import com.consumerreports.common.Kubectl
+import com.consumerreports.common.ContainerImage
+import com.consumerreports.common.Slack
+import com.consumerreports.common.Utility
+import com.consumerreports.common.Kustomize
+
+pipelineConfig = loadVariables(path: 'com/consumerreports/il/osiraa/osiraa.yaml')
+
+pipeline {
+  agent {
+    kubernetes {
+      label "build-${pipelineConfig.application.name}-${BUILD_NUMBER}"
+      yaml devopsPodTemplate.kanikoDeploy()
+    }
+  }
+  options {
+    timeout(time: 20, unit: 'MINUTES')
+    disableConcurrentBuilds()
+    ansiColor('xterm')
+    timestamps()
+    skipDefaultCheckout()
+  }
+  stages {
+    stage('Checkout code') {
+      steps {
+        container('build'){
+          script {
+            // Checkout jenkinsfiles-il from stash.consumer.org
+            checkoutCodeInFolder(codeDir: WORKSPACE, repositoryUrl: pipelineConfig['buildCode']['repositoryUrl'], branch: pipelineConfig['buildCode']['repositoryBranch'])
+            // Checkout application code from github.com
+            dir(pipelineConfig['application']['codeDir']){
+                checkout scm
+            }
+          }
+        }
+      }
+    }
+    stage('Set environment variables'){
+      steps {
+        container('deploy'){
+          script {
+            switch (env.BRANCH_NAME) {
+              case ~/(?i)^develop$/:
+                  env.DEPLOYMENT_ENVIRONMENT = 'dev'
+                  env.DEPLOYMENT_ENABLED = true
+                  break
+              case ~/(?i)^main$/:
+                  env.DEPLOYMENT_ENVIRONMENT = 'stage'
+                  env.DEPLOYMENT_ENABLED = true
+                  break
+              // case ~/(?i)^production$/:
+              //     env.DEPLOYMENT_ENVIRONMENT = 'prod'
+              //     env.DEPLOYMENT_ENABLED = true
+              //     break
+              default:
+                  env.DEPLOYMENT_ENVIRONMENT = 'dev'
+                  env.DEPLOYMENT_ENABLED = false
+                  break
+            }
+            // Pipeline config: https://stash.consumer.org/projects/K8S/repos/k8s-jenkins-il-shared-lib/browse/resources/com/consumerreports/il/datarightsprotocol-website
+            env.KUBECTL_CONTEXT         = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['kubectlContext']
+            env.DOCKER_IMAGE_NAME       = pipelineConfig['application']['name']
+            env.BUILD_CODE_SUB_DIR      = pipelineConfig['application']['name']
+            env.SLACK_WEBHOOK           = pipelineConfig['slack']['webhook']
+            env.SLACK_CHANNEL           = pipelineConfig['slack']['channel']
+            env.SLACK_TOKEN             = pipelineConfig['slack']['token']
+            env.KUBECTL_CONTEXT         = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['kubectlContext']
+            env.KUBECTL_NAMESPACE       = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['kubectlNamespace']
+            env.CERTIFICATE_NAME        = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['certificateName']
+            env.DEPLOYMENT_TIMEOUT      = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['deployment']['timeout']
+            env.DEPLOYMENT_DRY_RUN      = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['deployment']['dryRun']['enabled']
+            env.NOTIFICATIONS_DEPLOYMENT_SLACK = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['notifications']['deployment']['slack']['enabled']
+            env.CONTAINER_SCANNER_ENABLED = pipelineConfig['environments'][env.DEPLOYMENT_ENVIRONMENT]['containerScanner']['enabled']
+            env.APPLICATION_CODE_DIR    = pipelineConfig['application']['codeDir']
+            dir(env.APPLICATION_CODE_DIR) {
+              sh "git config --global --add safe.directory '*'"
+              env.DOCKER_IMAGE_TAG = getCommitHash()
+            }
+            // If we re-run the job we want to redeploy but we don't want to rebuild the image
+            if (ContainerImage.isPresentDocker(this, env.DOCKER_IMAGE_NAME, env.DOCKER_IMAGE_TAG)){
+                env.BUILD_ENABLED = false
+            } else {
+                env.BUILD_ENABLED = true
+            }
+          }
+        }
+      }
+    }
+    stage('Test build container image') {
+      when {
+        anyOf {
+          expression { env.BRANCH_NAME ==~ /(?i)^pr-(\d)*$/ }
+          expression { env.BRANCH_NAME ==~ /(?i)^feature\/.*$/ }
+        }
+      }
+      environment { 
+        PATH = "/busybox:/kaniko:$PATH"
+      }
+      steps {
+        container('kaniko'){
+          script {
+            // Try to build the image without pusing it to the docker registry
+            ContainerImage.buildKanikoDockerfile(this, [buildMode: "no-push", contextPath: "`pwd`/${env.APPLICATION_CODE_DIR}", dockerfilePath: "`pwd`/${env.APPLICATION_CODE_DIR}/Dockerfile", imageName: env.DOCKER_IMAGE_NAME, imageTag: env.DOCKER_IMAGE_TAG, args: ""])
+          }
+        }
+      }
+    }
+    stage('Build container image') {
+      when {
+        allOf {
+          expression { env.BUILD_ENABLED.toBoolean() }
+          expression { env.DEPLOYMENT_ENABLED.toBoolean() }
+        }
+      }
+      environment { 
+        PATH = "/busybox:/kaniko:$PATH"
+      }
+      steps {
+        container('kaniko'){
+          script {
+            ContainerImage.buildKanikoDockerfile(this, [contextPath: "`pwd`/${env.APPLICATION_CODE_DIR}", dockerfilePath: "`pwd`/${env.APPLICATION_CODE_DIR}/Dockerfile", imageName: env.DOCKER_IMAGE_NAME, imageTag: env.DOCKER_IMAGE_TAG, args: ""])
+          }
+        }
+      }
+    }
+    stage('Deploy') {
+      when { 
+        expression { env.DEPLOYMENT_ENABLED.toBoolean() }
+      }        
+      steps {
+        container('deploy'){
+          script {
+            Slack.slackSendDeployment(this, [status: 'start'])
+            Kubectl.showClusterInfo(this, env.KUBECTL_CONTEXT)
+            Kustomize.deployKustomize(
+              this, [
+              kubectlContext        : env.KUBECTL_CONTEXT,
+              deploymentEnvironment : env.DEPLOYMENT_ENVIRONMENT,
+              kustomizeManifestsPath: "${env.BUILD_CODE_SUB_DIR}/kustomize",
+              baseManifestFilename  : 'deployment.yaml',
+              basePath              : 'base',
+              overlayPath           : 'overlays',
+              imageTag              : env.DOCKER_IMAGE_TAG,
+              imageTagPlaceholder   : 'APP_CONTAINER_TAG',
+              dryRun                : env.DEPLOYMENT_DRY_RUN
+            ])
+          }
+        }
+      }
+      post {
+        success {
+          script {
+            Slack.slackSendDeployment(this, [status: 'finish'])
+          }
+        }
+        failure {
+          script {
+            Slack.slackSendDeployment(this, [status: 'failed'])
+          }
+        }
+      }
+    }
+    stage('Validate Deployment') {
+      when { 
+        expression { env.DEPLOYMENT_ENABLED.toBoolean() }
+      }      
+      steps {
+        container('deploy'){
+          script {
+            validateDeployment(deploymentEnvironment: env.DEPLOYMENT_ENVIRONMENT, deploymentTimeout: env.DEPLOYMENT_TIMEOUT)
+          }
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      cleanWs()
+    }
+  }
+}

README.md

@@ -1,5 +1,4 @@
 ```
-cd drp_aa_mvp
 pip install -r requirements.txt
 sudo apt update
 sudo apt install postgresql
@@ -9,7 +8,8 @@ python manage.py migrate
 
 Create superuser named vscode with password `vscode`
 ```
-python manage.py createsuperuser
+python3 manage.py createsuperuser
+python3 manage.py collectstatic
 ```
 
 Run/deploy the app:

deploy/drp.yml

@@ -5,8 +5,8 @@
   roles:
   - role: docker-ce
     become: yes
-  - role: fidesdemo
-    become: no
+  # - role: fidesdemo
+  #   become: no
 
   - role: osiraa
     become: no
@@ -16,15 +16,17 @@
       osiraa_service_id: osiraa
       osiraa_service_domain: osiraa.datarightsprotocol.org
       osiraa_port: 8000
+      osiraa_version: 'main'
+      # osiraa_version: 1954a9d6e09c8d15c1a4fe185b8e9874c99bdbed
 
-  - role: osiraa
-    become: no
-    vars:
-      osiraa_source_dir: "~/Code/osiraa05"
-      osiraa_remote_dir: "/home/ubuntu/osiraa05"
-      osiraa_service_id: "osiraa05"
-      osiraa_service_domain: "osiraa05.datarightsprotocol.org"
-      osiraa_port: 8001
+  # - role: osiraa
+  #   become: no
+  #   vars:
+  #     osiraa_source_dir: "~/Code/osiraa05"
+  #     osiraa_remote_dir: "/home/ubuntu/osiraa05"
+  #     osiraa_service_id: "osiraa05"
+  #     osiraa_service_domain: "osiraa05.datarightsprotocol.org"
+  #     osiraa_port: 8001
 
   pre_tasks:
   - name: install SSH keys

deploy/files/keys/john.pub

@@ -1 +1,2 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/vQLIAe2suhSPEzPQyrRfzLkhqst96Y3MtTTb0kG3GqpbHi+ezVHgQTJ0zPzSyAKazC4JLSmpVwHRri4M+4c7Xl+j9VWhh1ECN60gthzzPDHDhWfp49pOFDK6PmnrMoCp7ZudyNgv/+DGc4Cf1DsLaaTKIdRdSASppZTJqCy1rxQZakMpOnuyEprUfRVz7o3sEWyLAjfwB2aEUq/ejd8Ner5uTkx4V1vM5FrP8B/PVJktmFGeY1C0rlDlbHD6tppDEQz36fJbKAswcuuotXAx3ZJ/yqIwnJBGrQiFW/ZwWvmnzxgxm7pJlJ4uuwQJPfRGZp2Z4CqmGDHWqqocO4R/IjfDDU/rzsOIIl50pL3ZTB66fnq1SJSdF9tpa3CVAd0hGtP6hrJlPxh+NDK/EKhO7abuok7/LxULIVBw7M4IXB7e2Ol/LJy2pNrG9EiSndD1VSkCxmuOGEDV4K49UhbAONpL6AvW6lvF2m3cV28TVnxBaFZoDxgasC2L9E39EM0= [email protected]
+ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYsG4P3zVWVzXmf3BQ85EOjYzCLHcNfPWF5Ajr1NvN7wI0StebrVXwyQZ7onq/2wLT0Vk5rqy3Gn2y9AxPLHsp9LfXFlK/IS2DmocScA/uKL4k0wke4Q3XRjETULHZC/1KqrliLosWSfjo3Zga0lP+YqggCjjBfv/ycH636DdNs4CK7D5Dzv715LDTO1d/Slevn7pM+/QhyDEkKsACW+iDDWHi73bzI2O2s7Dign3vI5VAgrBnQu01zR0y3WjZOuMlGzWs7PBCZDgCdevsh8iQK2IT/pjBvophTG9J0Hkk/qVmRy/0JUGP3VECJXd4z2upVXLU0pggW55efUJP2fqH6dGzPRON5r/Q/tUzngEHvT+dBLhUl4dPLkp85MSQwQOAhdhCklhmlYGehgPg45eGz3azrGa44lZ6EmRfVDzHARHXU6IHZ7c5/GPQCt5vDVRw+6aL+3AYaoNKiBz9alqU432IcKHKAjU6lffEFi5EpwoEmx1L1b893zhHmJKazTE= szinjo@MACL12300
+

deploy/inventory

@@ -1,5 +1,5 @@
 all:
   hosts:
     drp-test:
-      ansible_host: 44.209.94.186
+      ansible_host: osiraa.datarightsprotocol.org
       ansible_user: ubuntu

deploy/roles/osiraa/tasks/main.yml

@@ -5,11 +5,18 @@
     state: directory
     path: '{{osiraa_remote_dir}}'
 
-- name: copy local osiraa checkout to instance host
-  synchronize:
-    src: '{{osiraa_source_dir}}/'
+# - name: copy local osiraa checkout to instance host
+#   synchronize:
+#     src: '{{osiraa_source_dir}}/'
+#     dest: '{{osiraa_remote_dir}}/'
+#     delete: yes
+#   register: copy_osiraa
+
+- name: clone osiraa from github
+  git:
+    repo: https://github.com/consumer-reports-innovation-lab/osiraa
     dest: '{{osiraa_remote_dir}}/'
-    delete: yes
+    version: '{{osiraa_version}}'
   register: copy_osiraa
 
 # TODO: install an environment file with secrets, override settings.py SECURITY WARNINGs

deploy/roles/osiraa/templates/osiraa.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=github.com/consumer-reports-digital-lab/osiraa
+Description=github.com/consumer-reports-innovation-lab/osiraa
 
 [Service]
 Type=simple

docker-compose.prod.yml

@@ -1,7 +1,13 @@
+volumes:
+  osiraa:
 services:
   web:
     ports:
       - "8000:8000"
+    environment:
+      - OSIRAA_KEY_FILE=/var/lib/osiraa/keys.json
+    volumes:
+      - osiraa:/var/lib/osiraa
   db:
     ports:
       - "15432:5432"

drp_aa_mvp/agent_keys/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

drp_aa_mvp/agent_keys/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AgentKeysConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'agent_keys'

drp_aa_mvp/agent_keys/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

drp_aa_mvp/agent_keys/templates/auth_keys.html

@@ -0,0 +1,17 @@
+<div style="margin:60px;">
+    <h2>OSIRAA - Open Source Implementer's Reference Authorized Agent</h2>
+    <br/>
+
+    <p><b>New Auth Agent Signing Key (64-bit encoded):</b><br>
+        {{agent_signing_key_b64}}</p>
+
+    <p><b>New Auth Agent Verify Key (64-bit encoded):</b><br>
+        {{agent_verify_key_b64}}</p>
+
+    <p>You can copy these keys into the config settings of your app.  If you update your app's keys, you must notify the DRP team so we can update your entry in the Service Directory.</p>
+
+    <form action="generate_auth_agent_keys_return" method="POST" >{% csrf_token %}
+        <input type="submit" value="Okay">
+    </form>
+
+</div>

drp_aa_mvp/agent_keys/templates/index.html

@@ -0,0 +1,10 @@
+<div style="margin:60px;">
+    <h2>OSIRAA - Open Source Implementer's Reference Authorized Agent</h2>
+    <br/>
+
+    <p><b>Generate Auth Agent Signing and Verify Keys</b></p>
+    <form action="generate_auth_agent_keys" method="POST" >{% csrf_token %}
+        <input type="submit" value="Generate Keys">
+    </form>
+
+</div>

drp_aa_mvp/agent_keys/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

drp_aa_mvp/agent_keys/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path('', views.index, name='index'),
+    path('generate_auth_agent_keys', views.generate_auth_agent_keys, name='generate_auth_agent_keys'),
+    path('generate_auth_agent_keys_return', views.generate_auth_agent_keys_return, name='generate_auth_agent_keys_return'),
+]