First major release (#18)

Co-authored-by: Joshua Jones <[email protected]>

.github/workflows/ci-build-deploy.yaml

@@ -80,23 +80,6 @@ jobs:
     - name: Push
       run: docker push ${{ steps.docker_meta.outputs.tags }} 
 
-    - name: 'Setup Network Security Policy'
-      run: |
-        echo """
-        kind: NetworkSecurityPolicy
-        apiVersion: security.devops.gov.bc.ca/v1alpha1
-        metadata:
-          name: proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}
-        spec:
-          description: |
-            allow the backend to talk to mongodb
-          source:
-            - - app.kubernetes.io/instance=proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}
-          destination:
-            - - app.kubernetes.io/instance=proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}-db
-        """ > nsp.yaml
-        oc apply -f nsp.yaml
-
     - name: 'Get Helm'
       run: |
         curl -L -O https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz
@@ -107,6 +90,14 @@ jobs:
         export PATH=$PATH:`pwd`/linux-amd64
 
         echo '
+        image:
+          registry: ${{ env.REGISTRY }}
+          repository: bcgov-dss/api-serv-infra/mongodb
+          tag: 4.4-ea993071
+          pullPolicy: IfNotPresent
+          pullSecrets:
+            - dev-github-read-packages-creds
+
         auth:
             rootPassword: "s3cr3t"
 
@@ -179,8 +170,8 @@ jobs:
             client-secret: ${{ secrets.OIDC_CLIENT_SECRET }}
             oidc-issuer-url: ${{ secrets.OIDC_ISSUER }}
             redirect-url: https://aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}.apps.silver.devops.gov.bc.ca/oauth2/callback
-            skip-auth-regex: '/home|/public|/docs|/_next|/images|/feed'
-            whitelist-domain: authz-apps-gov-bc-ca.dev.apsgw.xyz
+            skip-auth-regex: '/home|/public|/docs|/_next|/images|/feed|/signout'
+            whitelist-domain: authz-apps-gov-bc-ca.dev.api.gov.bc.ca
             skip-provider-button: 'true'
 
 
@@ -200,6 +191,8 @@ jobs:
           MONGO_PASSWORD:
             value: s3cr3t
             secure: true
+          FEEDER_URL:
+            value: 'http://proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}-feeder-generic-api'
           GITHUB_API_TOKEN:
             value: '${{ secrets.GH_TOKEN_FOR_CONTENT}}'
             secure: true
@@ -211,6 +204,10 @@ jobs:
             value: 'https://aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}.apps.silver.devops.gov.bc.ca'
           NEXT_PUBLIC_API_ROOT:
             value: 'https://aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}.apps.silver.devops.gov.bc.ca'          
+          GWA_API_URL:
+            value: 'https://gwa-api-gov-bc-ca.dev.api.gov.bc.ca'
+          GWA_RES_SVR_CLIENT_ID:
+            value: 'gwa-api'
 
         " > values.yaml
         

.github/workflows/ci-build-feeders.yaml

@@ -56,3 +56,83 @@ jobs:
 
     - name: Push
       run: docker push ${{ steps.docker_meta.outputs.tags }} 
+
+    - name: Install oc
+      uses: redhat-actions/oc-installer@v1
+      with:
+        version: '4.6'
+
+    - name: Authenticate and set context
+      uses: redhat-actions/oc-login@v1
+      with:
+        openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }}
+        openshift_token: ${{ secrets.OPENSHIFT_TOKEN }}
+
+        # Disables SSL cert checking. Use this if you don't have the certificate authority data.
+        insecure_skip_tls_verify: true
+
+        namespace: ${{ env.OPENSHIFT_NAMESPACE }}
+
+    - name: 'Get Helm'
+      run: |
+        curl -L -O https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz
+        tar -xf helm-v3.4.2-linux-amd64.tar.gz
+
+    - name: Set DEPLOY_ID which will deploy a custom deploy to 'dev' environment
+      run: echo '::set-output name=DEPLOY_ID::${{ steps.docker_meta.outputs.version }}'
+      id: set-deploy-id
+
+    - name: Get deploy ID
+      run: echo "The DEPLOY_ID is ${{ steps.set-deploy-id.outputs.DEPLOY_ID }}"
+
+    - name: 'Deploy Feeder'
+      run: |
+        export PATH=$PATH:`pwd`/linux-amd64
+
+        echo "
+        podAnnotations:
+          sha: $GITHUB_SHA
+
+        replicaCount: 1
+
+        rollingUpdate:
+          maxUnavailable: 50%
+          maxSurge: 50%
+
+        image:
+          repository: ${{ env.REGISTRY }}/bcgov/aps-portal/data-feeder
+          tag: ${{ steps.set-deploy-id.outputs.DEPLOY_ID }}
+          pullPolicy: Always
+
+        imagePullSecrets:
+          - name: dev-github-read-packages-creds
+
+        podSecurityContext:
+          fsGroup: ${{ secrets.RUNNING_UID_GID }}
+
+        securityContext:
+          runAsUser: ${{ secrets.RUNNING_UID_GID }}
+
+        containerPort: 6000
+
+        serviceAccount:
+          create: true
+          name: proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}
+
+        env:
+          LOG_FEEDS:
+            value: 'false'
+          WORKING_PATH:
+            value: '/tmp'
+          KONG_ADMIN_URL:
+            value: 'http://kong-kong-admin:8001'
+          CKAN_URL:
+            value: 'https://catalog.data.gov.bc.ca'
+          PROM_URL:
+            value: 'http://prometheus-server'
+          DESTINATION_URL:
+            value: 'http://proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}-generic-api'
+        " > values.yaml
+        
+        helm repo add bcgov http://bcgov.github.io/helm-charts
+        helm upgrade --install proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}-feeder -f values.yaml bcgov/generic-api

.github/workflows/ci-build-only.yaml

@@ -0,0 +1,62 @@
+name: Build and Deploy Refactor
+
+on:
+  push:
+    branches: [ main ]
+
+env:
+  REGISTRY: docker.pkg.github.com
+  # REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+  # REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_PASSWORD }}
+  #REGISTRY: ${{ secrets.CONTAINER_REGISTRY }}
+  REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }}
+  REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_PASSWORD }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1 
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.REGISTRY_USERNAME }}
+        password: ${{ env.REGISTRY_PASSWORD }}
+
+    - uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+    
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+
+    - id: release
+      uses: rymndhng/[email protected]
+      with:
+        bump_version_scheme: patch
+
+    - name: Check Output Parameters
+      run: |
+        echo "Got tag name ${{ steps.release.outputs.tag_name }}"
+        echo "Got release version ${{ steps.release.outputs.version }}"
+
+    - name: Build
+      uses: docker/build-push-action@v2
+      with:
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache
+        context: .
+        file: Dockerfile
+        tags: ${{ steps.release.outputs.tag_name }}
+        load: true
+        build-args: |
+          GITHUB_API_TOKEN=${{ secrets.CONTAINER_REGISTRY_PASSWORD }}
+
+    - name: Push
+      run: docker push ${{ steps.release.outputs.tag_name }}

.github/workflows/ci-remove.yaml

@@ -35,16 +35,6 @@ jobs:
         insecure_skip_tls_verify: true
 
         namespace: ${{ env.OPENSHIFT_NAMESPACE }}
-
-    - name: 'Delete Network Security Policy'
-      run: |
-        echo """
-        kind: NetworkSecurityPolicy
-        apiVersion: security.devops.gov.bc.ca/v1alpha1
-        metadata:
-          name: proto-aps-portal-${{ steps.set-deploy-id.outputs.DEPLOY_ID }}
-        """ > nsp.yaml
-        oc delete -f nsp.yaml
 
     - name: 'Get Helm'
       run: |

.gitignore

@@ -108,4 +108,6 @@ dist
 
 LOCAL.md
 _data
-_tmp
+_tmp
+
+kc.js

Dockerfile

@@ -1,8 +1,8 @@
-FROM node:alpine
+FROM node:15.14.0-alpine3.13
 
 WORKDIR /app
 
-COPY src/package*.json ./
+COPY src/*.json ./
 RUN npm install
 
 COPY src ./

README.md

@@ -1,4 +1,8 @@
-# API Developer and Management Portal
+# API Services Portal
+
+## Introduction
+
+The `API Services Portal` is a frontend for API Providers to manage the lifecycle of their APIs and for Developers to discover and access these APIs.  It works in combination with the Kong Community Edition Gateway and Keycloak IAM solution.
 
 ## Running the Project.
 
@@ -21,25 +25,102 @@ export MONGO_URL=mongodb://$hostip:17017/keystonedb3
 export MONGO_USER=""
 export MONGO_PASSWORD=""
 
+export FEEDER_URL=http://localhost:6000
+
 export KONG_URL=""
 export OIDC_ISSUER=""
 export JWKS_URL=${OIDC_ISSUER}/protocol/openid-connect/certs
 
+export NEXT_PUBLIC_API_ROOT=http://localhost:4180
 export EXTERNAL_URL="http://localhost:4180"
 
+export GWA_API_URL=http://localhost:2000
+
 npm run dev
 ```
 
-Once running, the `aps portal` application is reachable via `localhost:4180`.
+Once running, the `api services portal` application is reachable via `localhost:4180`.
+
 
 ## Design
 
+The `API Services Portal` is a React application using the Chakra UI component library, and using two frameworks: KeystoneJS V5, and NextJS.
+
+The application is divided up into the following six components:
+### 1. Data Model
+
+The KeystoneJS lists define the aggregated data model that makes up this application.
+
+Source: `src/lists/*`
+### 2. UI
+
+The actual pages and components for the `API Services Portal`.
+
+Source: `src/nextapp/*`
+### 3. Authentication
+
+Support for an OAuth2-Proxy was added to allow authenticating with an OAuth2 flow.  A Token is passed on to the KeystoneJS backend and our middleware verifies the token and starts a session.
+
+Source: `src/auth/auth-oauth2-proxy.js`
+
+### 4. Authorization
+
+A decision matrix and authorization rules engine is implemented to centralize the rules around access to data.
+
+It uses Permissions retrieved for the logged in user and a particular `Namespace` Resource.  The Requesting Party Token (RPT) holding the permissions will be maintained in the KeystoneJS Session and refreshed accordingly.
+
+Switching namespaces will result in getting a new RPT that has the relevant permission for the user for the `Namespace`.
+
+| Function                                   | Access                                                                              |
+|--------------------------------------------|-------------------------------------------------------------------------------------|
+| Discover APIs                              | Guest                                                                               |
+| API Access (Request Access)                | Authenticated                                                                       |
+| API Access (Revoke, Documentation)         | Authenticated and Service Access (by Consumer for user or app)                      |
+| Documentation (public)                     | Guest                                                                               |
+| Documentation (private)                    | Authenticated and Service Access (by Consumer for user or app)                      |
+| My Resources (Grant/Revoke/Approve/Reject) | Authenticated and `Resource Owner` for UMA `Namespace` Resource                     |
+| My Resources (Create Service Account)      | Authenticated and `Resource Owner` for UMA `Namespace` Resource                     |
+| Applications (Ministry)                    | Authenticated with IDIR or Github                                                   |
+| Applications (Business)                    | Authenticated with BCeID                                                            |
+| Namespaces                                 | Authenticated and `any` UMA `Namespace` Resource Permission                         |
+| Namespaces (Create Namespace)              | Authenticated                                                                       |
+| Namespaces (Delete Namespace)              | Authenticated and UMA `Namespace` Resource Permission `Namespace.Manage` (or Owner) |
+| Products (and Environments)                | UMA `Namespace` Resource Permission `Namespace.Manage` or `Namespace.View`          |
+| Services (View Config and Metrics)         | UMA `Namespace` Resource Permission `Namespace.Manage` or `Namespace.View`          |
+| Consumers (Pending Approval)               | UMA `Namespace` Resource Permission `Access.Manage`                                 |
+| Consumers (Service Access)                 | UMA `Namespace` Resource Permission `Access.Manage`                                 |
+| Authorization Profiles (Credential Issuer) | UMA `Namespace` Resource Permission `Namespace.Manage`                              |
+| Activity                                   | UMA `Namespace` Resource Permission `Namespace.Manage` or `Namespace.View`          |
+| Publish Gateway Config                     | UMA `Namespace` Resource Permission `GatewayConfig.Publish`                         |
+| Delete Gateway Config                      | UMA `Namespace` Resource Permission `GatewayConfig.Publish`                         |
+| Namespace Profile (Org and Contacts)       | UMA `Namespace` Resource Permission `Namespace.Admin`                               |
+
+Source: `src/authz`
+### 5. Ingestor
+
+An ingestion framework for adding content from external sources.
+
+Source: `src/batch/feedWorker.js`
+
+### 6. Feeders
+
+A set of feeders that live close to the external sources for reading and sending data to the Ingestor
+
+Currently support feeders:
+* CKAN (Comprehensive Knowledge Archive Network)
+* Kong
+* Prometheus
+
+Source: `feeds`
+
+## User Journeys
 Roles:
 
-- **Credential Admin**: Application for authenticating with an OIDC Auth provider for the purposes of client registration. The Credential Issuer will generate the new credentials and provide a mechanism for the Developer to retrieve them.
-- **API Manager**: The API Manager makes APIs available for consumption with supporting documentation. They approve requests for access.
 - **API Owner**: Does the technical deployment of the API on the Gateway under a particular Namespace - Gateway Services.
 - **Developer**: A Developer discovers APIs, requests access if required and consumes them.
+- **Credential Admin**: Application for authenticating with an OIDC Auth provider for the purposes of client registration. The Credential Issuer will generate the new credentials and provide a mechanism for the Developer to retrieve them.
+- **API Manager**: The API Manager makes APIs available for consumption with supporting documentation. They approve requests for access.
+- **Pilot Tester**: This role enables features that are still being reviewed and not quite ready for broader use.
 
 Typical Flow: