Merge pull request #779 from bcgov/test

Create Latest Release
bcgov · Mar 13, 2023 · 0844132 · 0844132
2 parents 2b17589 + c227da7
commit 0844132
Show file tree

Hide file tree

Showing 32 changed files with 669 additions and 941 deletions.
diff --git a/.github/workflows/scripts/feeder-init/platform-authz-profile.yaml b/.github/workflows/scripts/feeder-init/platform-authz-profile.yaml
@@ -18,7 +18,7 @@ record:
     - GatewayConfig.Publish
     - Content.Publish
     - CredentialIssuer.Admin
-  owner: [email protected]
+  owner: api-owner@local
   environmentDetails:
     - environment: prod
       issuerUrl: '{OIDC_ISSUER}'

diff --git a/feeds/ckan/index.js b/feeds/ckan/index.js
@@ -47,7 +47,7 @@ async function scopedSync(
     loadDatasetProducer(xfer, scopedDir, destinationUrl)
   );
 
-  fs.rmdirSync(scopedDir, { recursive: true });
+  fs.rmSync(scopedDir, { recursive: true });
 }
 
 async function syncOrgs({ url, workingPath, destinationUrl }) {

diff --git a/feeds/kong/index.js b/feeds/kong/index.js
@@ -70,6 +70,8 @@ async function scopedSyncByNamespace(
     loadGroupsProducer(xfer, destinationUrl, '/feed/GatewayGroup')
   );
 
+  xfer.resultCollector().output();
+
   // remove any GatewayService or GatewayRoutes that no longer exist in Kong
   const destination = portal(destinationUrl);
 
@@ -94,7 +96,7 @@ async function scopedSyncByNamespace(
     }
   }
 
-  fs.rmdirSync(scopedDir, { recursive: true });
+  fs.rmSync(scopedDir, { recursive: true });
 }
 
 async function scopedSyncByConsumer(
@@ -122,7 +124,9 @@ async function scopedSyncByConsumer(
     )
   );
 
-  fs.rmdirSync(scopedDir, { recursive: true });
+  xfer.resultCollector().output();
+
+  fs.rmSync(scopedDir, { recursive: true });
 }
 
 async function sync({ url, workingPath, destinationUrl }) {
@@ -166,9 +170,13 @@ async function sync({ url, workingPath, destinationUrl }) {
       '/feed/GatewayConsumer'
     )
   );
+
   await xfer.concurrentWork(
     loadGroupsProducer(xfer, destinationUrl, '/feed/GatewayGroup')
   );
+
+  xfer.resultCollector().output();
+
   //await xfer.concurrentWork(loadProducer(xfer, destinationUrl, 'gw-products', 'name', 'product', '/feed/Product'))
   //await xfer.concurrentWork(loadServiceAccessProducer(xfer, destinationUrl, 'gw-consumers', '/feed/ServiceAccess'))
 }
@@ -181,26 +189,11 @@ function loadProducer(xfer, destinationUrl, file, name, type, feedPath) {
     type == 'consumer' ? xfer.get_json_content('gw-acls')['data'] : null;
   let index = 0;
 
-  log.info('[producer] %s : %d records', file, items.length);
-
-  const results = {
-    'no-change': 0,
-    created: 0,
-    'created-failed': 0,
-    updated: 0,
-    deleted: 0,
-    'updated-failed': 0,
-    'deleted-failed': 0,
-  };
+  log.debug('[producer] %s : %d records', file, items.length);
 
   return () => {
     if (index == items.length) {
-      Object.keys(results)
-        .filter((r) => results[r] != 0)
-        .forEach((r) => {
-          log.info('[%s] %d', String(r).padStart(15, ' '), results[r]);
-        });
-      log.info('Finished producing ' + index + ' records.');
+      log.info('[producer] %s : Finished %d records', file, items.length);
       return null;
     }
     const item = items[index];
@@ -244,12 +237,14 @@ function loadProducer(xfer, destinationUrl, file, name, type, feedPath) {
     return destination
       .fireAndForget(feedPath, item)
       .then((result) => {
-        results[result['result']]++;
+        xfer.resultCollector().inc(result['result']);
 
         log.debug('%s -> %s OK', file, result);
       })
       .catch((err) => {
-        log.error(`[${nm}] ERR ${err}`);
+        xfer.resultCollector().inc('exception');
+        log.error('%s', err);
+        log.error('%j', item);
       });
   };
 }
@@ -279,28 +274,17 @@ function loadGroupsProducer(xfer, destinationUrl, feedPath) {
       });
     });
 
-  log.info('[loadGroupsProducer] (%s) %d records', feedPath, items.length);
+  log.debug('[loadGroupsProducer] (%s) %d records', feedPath, items.length);
 
   let index = 0;
 
-  const results = {
-    'no-change': 0,
-    created: 0,
-    'created-failed': 0,
-    updated: 0,
-    deleted: 0,
-    'updated-failed': 0,
-    'deleted-failed': 0,
-  };
-
   return () => {
     if (index == items.length) {
-      Object.keys(results)
-        .filter((r) => results[r] != 0)
-        .forEach((r) => {
-          log.info('[%s] %d', String(r).padStart(15, ' '), results[r]);
-        });
-      log.info('Finished producing ' + index + ' records.');
+      log.info(
+        '[loadGroupsProducer] %s : Finished %d records',
+        feedPath,
+        items.length
+      );
       return null;
     }
     const item = items[index];
@@ -316,10 +300,11 @@ function loadGroupsProducer(xfer, destinationUrl, feedPath) {
     return destination
       .fireAndForget(feedPath, item)
       .then((result) => {
-        results[result['result']]++;
+        xfer.resultCollector().inc(result['result']);
         log.debug('%s -> %s OK', feedPath, result);
       })
       .catch((err) => {
+        xfer.resultCollector().inc('exception');
         log.error(`[${nm}] ERR ${err}`);
       });
   };

diff --git a/feeds/logger.js b/feeds/logger.js
@@ -1,4 +1,5 @@
 const winston = require('winston');
+const { createLogger, format, transports } = require('winston');
 
 const enumerateErrorFormat = winston.format((info) => {
   if (info instanceof Error) {
@@ -14,12 +15,16 @@ function Logger(category) {
     level: process.env.LOG_LEVEL || 'debug',
     format: winston.format.combine(
       enumerateErrorFormat(),
+      format.timestamp({
+        format: 'YYYY-MM-DD HH:mm:ss',
+      }),
       process.env.NODE_ENV === 'production'
         ? winston.format.uncolorize()
         : winston.format.colorize(),
       winston.format.splat(),
       winston.format.printf(
-        ({ level, message, stack }) => `${level}: [${category}] ${message}`
+        ({ timestamp, level, message, stack }) =>
+          `${timestamp} ${level}: [${category}] ${message}`
       )
     ),
     transports: [

diff --git a/feeds/prometheus/index.js b/feeds/prometheus/index.js
@@ -91,6 +91,8 @@ async function syncQueryRanges(
   await xfer.concurrentWork(
     producer(xfer, 'query_range', params.numDays, destinationUrl)
   );
+
+  xfer.resultCollector().output();
 }
 
 async function syncQueries(
@@ -126,6 +128,8 @@ async function syncQueries(
   await xfer.concurrentWork(
     producer(xfer, 'query', params.numDays, destinationUrl)
   );
+
+  xfer.resultCollector().output();
 }
 
 function producer(xfer, queryType, numDays, destinationUrl) {
@@ -139,7 +143,7 @@ function producer(xfer, queryType, numDays, destinationUrl) {
       const results = Array.isArray(json['data'])
         ? json['data'][0]['result']
         : json['data']['result'];
-      log.info(
+      log.debug(
         '[producer] %s %s : %d records',
         target,
         _query.id,
@@ -180,8 +184,16 @@ function producer(xfer, queryType, numDays, destinationUrl) {
 
     return destination
       .fireAndForget('/feed/Metric', item.metric)
-      .then((result) => log.debug(`[${name}] OK`, result))
-      .catch((err) => log.error(`[${name}] ERR ${err}`));
+      .then((result) => {
+        xfer.resultCollector().inc(result['result']);
+
+        log.debug('%s -> %s OK', name, result);
+      })
+      .catch((err) => {
+        xfer.resultCollector().inc('exception');
+        log.error('%s', err);
+        log.error('%j', item);
+      });
   };
 }
 

diff --git a/feeds/utils/portal.js b/feeds/utils/portal.js
@@ -1,5 +1,8 @@
 const fs = require('fs');
 const fetch = require('node-fetch');
+const { Logger } = require('../logger');
+
+const log = Logger('utils.portal');
 
 const checkStatus = require('./checkStatus').checkStatus;
 
@@ -25,7 +28,7 @@ function portal(baseUrl, logFeeds = _logFeeds) {
       let retry = attempts;
       while (retry <= attempts) {
         retry != attempts &&
-          console.log('Retrying [' + (attempts - retry) + '] ' + url);
+          log.error('Retrying [' + (attempts - retry) + '] ' + url);
         try {
           return await fetch(baseUrl + url, {
             method: 'put',
@@ -47,7 +50,7 @@ function portal(baseUrl, logFeeds = _logFeeds) {
       let retry = attempts;
       while (retry <= attempts) {
         retry != attempts &&
-          console.log('Retrying [' + (attempts - retry) + '] ' + url);
+          log.error('Retrying [' + (attempts - retry) + '] ' + url);
         try {
           return await fetch(baseUrl + url, {
             method: 'delete',

diff --git a/feeds/utils/results.js b/feeds/utils/results.js
@@ -0,0 +1,35 @@
+const { Logger } = require('../logger');
+
+const log = Logger('results');
+
+class ResultsService {
+  init() {
+    this.results = {
+      'no-change': 0,
+      exception: 0,
+      created: 0,
+      'created-failed': 0,
+      updated: 0,
+      deleted: 0,
+      'updated-failed': 0,
+      'deleted-failed': 0,
+    };
+  }
+
+  inc(result) {
+    this.results[result]++;
+  }
+
+  output() {
+    const { results } = this;
+    Object.keys(results)
+      .filter((r) => results[r] != 0)
+      .forEach((r) => {
+        log.info('[%s] %d', String(r).padStart(15, ' '), results[r]);
+      });
+  }
+}
+
+module.exports = {
+  ResultsService,
+};
diff --git a/feeds/utils/transfers.js b/feeds/utils/transfers.js
@@ -5,13 +5,19 @@ const PromisePool = require('es6-promise-pool');
 const { checkStatus } = require('./checkStatus');
 const { Logger } = require('../logger');
 const stringify = require('json-stable-stringify');
+const { ResultsService } = require('./results');
 
 const log = Logger('utils.xfer');
 
 function transfers(workingPath, baseUrl, exceptions) {
   fs.mkdirSync(workingPath, { recursive: true });
 
+  const resultCollector = new ResultsService();
+  resultCollector.init();
+
   return {
+    resultCollector: () => resultCollector,
+
     copy: async function (_url, filename, index = 0) {
       log.debug('[copy] %s%s', baseUrl, _url);
       const out = workingPath + '/' + filename + '-' + index + '.json';
@@ -79,7 +85,7 @@ function transfers(workingPath, baseUrl, exceptions) {
       // Wait for the pool to settle.
       return poolPromise.then(
         function () {
-          log.info('All promises fulfilled');
+          log.debug('All promises fulfilled');
         },
         function (error) {
           log.error('Some promise rejected: ' + error.message);