From f22d94a5005fc2bf5ccce955b42970c54a937633 Mon Sep 17 00:00:00 2001 From: Ally Shaban Date: Thu, 4 Oct 2018 15:45:48 +0300 Subject: [PATCH] merging with changes made in facility-recon-datim --- dhis2sync/loadDHIS2Metadata.js | 6 +- .../config/default_multiple_instances.json | 50 - facility-recon-backend/lib/index.js | 907 ++++++++++-------- facility-recon-backend/lib/mcsd.js | 250 ++--- facility-recon-backend/package-lock.json | 16 + facility-recon-backend/package.json | 1 + facility-recon-gui/src/App.vue | 5 +- .../DataSync/FacilityReconDataSync.vue | 3 +- .../DataSync/FacilityReconUpload.vue | 119 ++- .../src/components/FacilityRecoStatus.vue | 8 +- .../src/components/FacilityReconScores.vue | 106 +- .../src/components/FacilityReconUpload.vue | 508 ---------- facility-recon-gui/src/mixins/scoresMixin.js | 17 +- facility-recon-gui/src/store/store.js | 3 +- 14 files changed, 846 insertions(+), 1153 deletions(-) delete mode 100755 facility-recon-backend/config/default_multiple_instances.json delete mode 100644 facility-recon-gui/src/components/FacilityReconUpload.vue diff --git a/dhis2sync/loadDHIS2Metadata.js b/dhis2sync/loadDHIS2Metadata.js index d7365c9ce..24be57482 100644 --- a/dhis2sync/loadDHIS2Metadata.js +++ b/dhis2sync/loadDHIS2Metadata.js @@ -62,7 +62,11 @@ async function processMetaData() { if ( !nconf.get("full") && hasKey ) { lastUpdate = await getLastUpdate() // Convert to yyyy-mm-dd format (dropping time as it is ignored by DHIS2) - lastUpdate = new Date( Date.parse( lastUpdate ) ).toISOString().substr(0,10) + try { + lastUpdate = new Date(Date.parse(lastUpdate)).toISOString().substr(0, 10) + } catch(err) { + console.log(err) + }; } let uflag = 'false' diff --git a/facility-recon-backend/config/default_multiple_instances.json b/facility-recon-backend/config/default_multiple_instances.json deleted file mode 100755 index cceeebf11..000000000 --- a/facility-recon-backend/config/default_multiple_instances.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "server": { - "hostname": "0.0.0.0", - "port": 3000 - }, - "mongodb": { - "host":"localhost", - "port":"27017", - "database":"gofr", - "user":"", - "password":"" - }, - "mCSDDATIM":{ - "url":"http://localhost:3448/fhir", - "username":"", - "password":"" - }, - "mCSDMOH":{ - "url":"http://localhost:3449/fhir", - "username":"", - "password":"" - }, - "CSD":{ - "url":"http://localhost:8984/CSD", - "username":"", - "password":"" - }, - "UUID":{ - "namespace":"de2026e0-62ad-11e8-98ab-fb2f18c0266" - }, - "oauth": { - "accessTokenLifetime": 3600, - "debug":true - }, - "logger": { - "level": "info" - }, - "levels":[ - "level1", - "level2", - "level3", - "level4", - "level5", - "level6", - "level7" - ], - "matchResults":{ - "maxSuggestions":5 - } -} \ No newline at end of file diff --git a/facility-recon-backend/lib/index.js b/facility-recon-backend/lib/index.js index b7db192b0..a1b37fb68 100755 --- a/facility-recon-backend/lib/index.js +++ b/facility-recon-backend/lib/index.js @@ -9,8 +9,10 @@ const formidable = require('formidable'); const winston = require('winston'); const https = require('https'); const http = require('http'); +const cors = require('cors'); const redis = require('redis'); const redisClient = redis.createClient(); +const csv = require('fast-csv'); const URI = require('urijs'); const async = require('async'); const mongoose = require('mongoose'); @@ -23,6 +25,11 @@ const scores = require('./scores')(); const app = express(); const server = require('http').createServer(app); +app.use(cors({ + origin: true, + credentials: true +})); + app.use(bodyParser.urlencoded({ extended: true, })); @@ -73,45 +80,115 @@ if (cluster.isMaster) { cluster.fork(); }); } else { + const levelMaps = { + 'ds0ADyc9UCU': { // Cote D'Ivoire + 4: 5, + } + } + + app.get('/doubleMapping/:db', (req, res) => { + winston.info('Received a request to check MOH Locations that are double mapped') + let mohDB = req.params.db + let mappingDB = "MOHDATIM" + req.params.db + async.parallel({ + mohData: function (callback) { + mcsd.getLocations(mohDB, (data) => { + return callback(false, data) + }) + }, + mappingData: function (callback) { + mcsd.getLocations(mappingDB, (data) => { + return callback(false, data) + }) + } + }, (err, results) => { + let dupplicated = [] + let url = 'http://localhost:3447/' + mohDB + '/fhir/Location/' + async.each(results.mohData.entry, (mohEntry, nxtMoh) => { + mohid = mohEntry.resource.id + let checkDup = [] + async.each(results.mappingData.entry, (mappingEntry, nxtMap) => { + var isMapped = mappingEntry.resource.identifier.find((ident) => { + return ident.system === 'http://geoalign.datim.org/MOH' && ident.value === url + mohid + }) + if (isMapped) { + checkDup.push({ + mohName: mohEntry.resource.name, + mohID: mohEntry.resource.id, + datimName: mappingEntry.resource.name, + datimID: mappingEntry.resource.id + }) + } + return nxtMap() + }, () => { + if (checkDup.length > 1) { + dupplicated.push(checkDup) + } + return nxtMoh() + }) + }, () => { + winston.info('Found ' + dupplicated.length + ' MOH Locations with Double Matching') + res.send(dupplicated) + }) + }) + }) + app.get('/countLevels/:orgid', (req, res) => { if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); } else { const orgid = req.params.orgid; + const namespace = config.getConf('UUID:namespace'); + const mohTopId = uuid5(orgid, `${namespace}000`); winston.info(`Getting total levels for ${orgid}`); - mcsd.countLevels('DATIM', orgid, (err, totalLevels) => { + const db = config.getConf('mCSD:database'); + + async.parallel({ + MOHLevels: function (callback) { + mcsd.countLevels('MOH', orgid, mohTopId, (err, mohTotalLevels) => { + winston.info(`Received total MOH levels of ${mohTotalLevels} for ${orgid}`); + return callback(err, mohTotalLevels) + }) + }, + DATIMLevels: function (callback) { + mcsd.countLevels('DATIM', db, orgid, (err, datimTotalLevels) => { + winston.info(`Received total DATIM levels of ${datimTotalLevels} for ${orgid}`); + return callback(err, datimTotalLevels) + }) + } + }, (err, results) => { res.set('Access-Control-Allow-Origin', '*'); if (err) { winston.error(err); res.status(401).json({ - error: 'Missing Orgid', + error: err }); } else { const recoLevel = 2; - winston.info(`Received total levels of ${totalLevels} for ${orgid}`); res.status(200).json({ - totalLevels, - recoLevel, + totalMOHLevels: results.MOHLevels, + totalDATIMLevels: results.DATIMLevels, + recoLevel }); } - }); + }) } }); app.get('/uploadAvailable/:orgid', (req, res) => { if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); } else { const orgid = req.params.orgid; @@ -120,26 +197,26 @@ if (cluster.isMaster) { if (mohData.hasOwnProperty('entry') && mohData.entry.length > 0) { res.set('Access-Control-Allow-Origin', '*'); res.status(200).json({ - dataUploaded: true, + dataUploaded: true }); } else { res.set('Access-Control-Allow-Origin', '*'); res.status(200).json({ - dataUploaded: false, + dataUploaded: false }); } - }); + }) } }); app.get('/getArchives/:orgid', (req, res) => { if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); } else { const orgid = req.params.orgid; @@ -148,26 +225,26 @@ if (cluster.isMaster) { res.set('Access-Control-Allow-Origin', '*'); if (err) { winston.error({ - error: 'Unexpected error has occured', + error: 'Unexpected error has occured' }); res.status(400).json({ - error: 'Unexpected error', + error: 'Unexpected error' }); - return; + return } - res.status(200).json(archives); - }); + res.status(200).json(archives) + }) } }); app.post('/restoreArchive/:orgid', (req, res) => { if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); } else { const orgid = req.params.orgid; @@ -177,14 +254,14 @@ if (cluster.isMaster) { mcsd.restoreDB(fields.archive, orgid, (err) => { res.set('Access-Control-Allow-Origin', '*'); if (err) { - winston.error(err); + winston.error(err) res.status(401).json({ - error: 'Unexpected error occured while restoring the database,please retry', + error: 'Unexpected error occured while restoring the database,please retry' }); } res.status(200).send(); - }); - }); + }) + }) } }); @@ -211,11 +288,11 @@ if (cluster.isMaster) { app.get('/hierarchy/:source/:id/:start/:count', (req, res) => { if (!req.query.OrgId || !req.query.OrgName || !req.params.source) { winston.error({ - error: 'Missing Orgid or source', + error: 'Missing Orgid or source' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid or source', + error: 'Missing Orgid or source' }); } else { const topOrgId = req.query.OrgId; @@ -240,8 +317,8 @@ if (cluster.isMaster) { }) winston.info(`Done Fetching ${source} Locations`); }); - }); - }); + }) + }) } else if (source == 'DATIM') { var database = config.getConf('mCSD:database'); var locationReceived = new Promise((resolve, reject) => { @@ -253,8 +330,8 @@ if (cluster.isMaster) { }) winston.info(`Done Fetching ${source} Locations`); }); - }); - }); + }) + }) } locationReceived.then((data) => { @@ -323,36 +400,47 @@ if (cluster.isMaster) { } }); - app.get('/mappingStatus/:orgid/:level/:clientId', (req, res) => { + app.get('/mappingStatus/:orgid/:level/:totalDATIMLevels/:totalMOHLevels/:clientId', (req, res) => { winston.info('Getting mapping status'); const orgid = req.params.orgid; const mohDB = orgid; const datimTopId = orgid; const datimDB = config.getConf('mCSD:database'); const recoLevel = req.params.level; + const totalDATIMLevels = req.params.totalDATIMLevels; + const totalMOHLevels = req.params.totalMOHLevels; const namespace = config.getConf('UUID:namespace'); const mohTopId = uuid5(orgid, `${namespace}000`); const clientId = req.params.clientId; - const statusRequestId = `mappingStatus${datimTopId}${clientId}`; + let statusRequestId = `mappingStatus${datimTopId}${clientId}` statusResData = JSON.stringify({ status: '1/2 - Loading DATIM and MOH Data', error: null, - percent: null, - }); - redisClient.set(statusRequestId, statusResData); + percent: null + }) + redisClient.set(statusRequestId, statusResData) const datimLocationReceived = new Promise((resolve, reject) => { mcsd.getLocationChildren(datimDB, datimTopId, (mcsdDATIM) => { mcsdDatimAll = mcsdDATIM; - mcsd.filterLocations(mcsdDATIM, datimTopId, 0, recoLevel, 0, (mcsdDatimTotalLevels, mcsdDatimLevel, mcsdDatimBuildings) => { + let level + if (recoLevel === totalMOHLevels) { + level = totalDATIMLevels + } else { + level = recoLevel + } + if (levelMaps[orgid] && levelMaps[orgid][recoLevel]) { + level = levelMaps[orgid][recoLevel]; + } + mcsd.filterLocations(mcsdDATIM, datimTopId, level, (mcsdDatimLevel) => { resolve(mcsdDatimLevel); }); }); }); const mohLocationReceived = new Promise((resolve, reject) => { mcsd.getLocations(mohDB, (mcsdMOH) => { - mcsd.filterLocations(mcsdMOH, mohTopId, 0, recoLevel, 0, (mcsdMohTotalLevels, mcsdMohLevel, mcsdMohBuildings) => { + mcsd.filterLocations(mcsdMOH, mohTopId, recoLevel, (mcsdMohLevel) => { resolve(mcsdMohLevel); }); }); @@ -364,30 +452,31 @@ if (cluster.isMaster) { }); }); Promise.all([datimLocationReceived, mohLocationReceived, mappingLocationReceived]).then((locations) => { - const datimLocations = locations[0]; - const mohLocations = locations[1]; - const mappedLocations = locations[2]; + var datimLocations = locations[0] + var mohLocations = locations[1] + var mappedLocations = locations[2] scores.getMappingStatus(mohLocations, datimLocations, mappedLocations, datimTopId, clientId, (mappingStatus) => { res.set('Access-Control-Allow-Origin', '*'); - res.status(200).json(mappingStatus); - }); - }); - }); + res.status(200).json(mappingStatus) + }) + }) + }) - app.get('/reconcile/:orgid/:totalLevels/:recoLevel/:clientId', (req, res) => { + app.get('/reconcile/:orgid/:totalLevels/:totalDATIMLevels/:recoLevel/:clientId', (req, res) => { if (!req.params.orgid || !req.params.recoLevel) { winston.error({ - error: 'Missing Orgid or reconciliation Level', + error: 'Missing Orgid or reconciliation Level' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid or reconciliation Level', + error: 'Missing Orgid or reconciliation Level' }); } else { winston.info('Getting scores'); const orgid = req.params.orgid; const recoLevel = req.params.recoLevel; const totalLevels = req.params.totalLevels; + const totalDATIMLevels = req.params.totalDATIMLevels; const clientId = req.params.clientId; const datimDB = config.getConf('mCSD:database'); const mohDB = orgid; @@ -397,146 +486,167 @@ if (cluster.isMaster) { let mcsdDatimAll = null; let mcsdMohAll = null; - const scoreRequestId = `scoreResults${datimTopId}${clientId}`; + let scoreRequestId = `scoreResults${datimTopId}${clientId}` scoreResData = JSON.stringify({ status: '1/3 - Loading DATIM and MOH Data', error: null, - percent: null, - }); - redisClient.set(scoreRequestId, scoreResData); - const datimLocationReceived = new Promise((resolve, reject) => { - mcsd.getLocationChildren(datimDB, datimTopId, (mcsdDATIM) => { - mcsdDatimAll = mcsdDATIM; - mcsd.filterLocations(mcsdDATIM, datimTopId, 0, recoLevel, 0, (mcsdDatimTotalLevels, mcsdDatimLevel, mcsdDatimBuildings) => { - resolve(mcsdDatimLevel); - }); - }); - }); + percent: null + }) + redisClient.set(scoreRequestId, scoreResData) + async.parallel({ + datimLocations: function (callback) { + mcsd.getLocationChildren(datimDB, datimTopId, (mcsdDATIM) => { + mcsdDatimAll = mcsdDATIM; + let level + if (recoLevel === totalLevels) { + level = totalDATIMLevels + } else { + level = recoLevel + } - const mohLocationReceived = new Promise((resolve, reject) => { - mcsd.getLocations(mohDB, (mcsdMOH) => { - mcsdMohAll = mcsdMOH; - mcsd.filterLocations(mcsdMOH, mohTopId, 0, recoLevel, 0, (mcsdMohTotalLevels, mcsdMohLevel, mcsdMohBuildings) => { - resolve(mcsdMohLevel); + if (levelMaps[orgid] && levelMaps[orgid][recoLevel]) { + level = levelMaps[orgid][recoLevel]; + } + mcsd.filterLocations(mcsdDATIM, datimTopId, level, (mcsdDatimLevel) => { + return callback(false, mcsdDatimLevel) + }); }); - }); - }); - - const mappingDB = config.getConf('mapping:dbPrefix') + orgid; - const mappingLocationReceived = new Promise((resolve, reject) => { - mcsd.getLocationByID(mappingDB, false, false, (mcsdMapped) => { - resolve(mcsdMapped); - }); - }); - Promise.all([datimLocationReceived, mohLocationReceived, mappingLocationReceived]).then((locations) => { + }, + mohLoations: function (callback) { + mcsd.getLocations(mohDB, (mcsdMOH) => { + mcsdMohAll = mcsdMOH; + mcsd.filterLocations(mcsdMOH, mohTopId, recoLevel, (mcsdMohLevel) => { + return callback(false, mcsdMohLevel); + }); + }); + }, + mappingData: function (callback) { + const mappingDB = config.getConf('mapping:dbPrefix') + orgid; + mcsd.getLocationByID(mappingDB, false, false, (mcsdMapped) => { + return callback(false, mcsdMapped); + }); + } + }, (error, results) => { if (recoLevel == totalLevels) { - scores.getBuildingsScores(locations[1], locations[0], locations[2], mcsdDatimAll, mcsdMohAll, mohDB, datimDB, mohTopId, datimTopId, recoLevel, totalLevels, clientId, (scoreResults) => { + scores.getBuildingsScores(results.mohLoations, results.datimLocations, results.mappingData, mcsdDatimAll, mcsdMohAll, mohDB, datimDB, mohTopId, datimTopId, recoLevel, totalLevels, clientId, (scoreResults) => { res.set('Access-Control-Allow-Origin', '*'); recoStatus(orgid, (totalAllMapped, totalAllNoMatch, totalAllFlagged) => { - const mohTotalAllNotMapped = (mcsdMohAll.entry.length - 1) - totalAllMapped; + scoreResData = JSON.stringify({ + status: 'Done', + error: null, + percent: 100 + }) + redisClient.set(scoreRequestId, scoreResData) + var mohTotalAllNotMapped = (mcsdMohAll.entry.length - 1) - totalAllMapped res.status(200).json({ scoreResults, recoLevel, - datimTotalRecords: locations[0].entry.length, + datimTotalRecords: results.datimLocations.entry.length, datimTotalAllRecords: mcsdDatimAll.entry.length, - totalAllMapped, - totalAllFlagged, - totalAllNoMatch, - mohTotalAllNotMapped, - mohTotalAllRecords: mcsdMohAll.entry.length - 1, + totalAllMapped: totalAllMapped, + totalAllFlagged: totalAllFlagged, + totalAllNoMatch: totalAllNoMatch, + mohTotalAllNotMapped: mohTotalAllNotMapped, + mohTotalAllRecords: mcsdMohAll.entry.length - 1 }); winston.info('Score results sent back'); - }); + }) }); } else { - scores.getJurisdictionScore(locations[1], locations[0], locations[2], mcsdDatimAll, mcsdMohAll, mohDB, datimDB, mohTopId, datimTopId, recoLevel, totalLevels, clientId, (scoreResults) => { + scores.getJurisdictionScore(results.mohLoations, results.datimLocations, results.mappingData, mcsdDatimAll, mcsdMohAll, mohDB, datimDB, mohTopId, datimTopId, recoLevel, totalLevels, clientId, (scoreResults) => { res.set('Access-Control-Allow-Origin', '*'); recoStatus(orgid, (totalAllMapped, totalAllNoMatch, totalAllFlagged) => { - const mohTotalAllNotMapped = (mcsdMohAll.entry.length - 1) - totalAllMapped; + var mohTotalAllNotMapped = (mcsdMohAll.entry.length - 1) - totalAllMapped res.status(200).json({ scoreResults, recoLevel, - datimTotalRecords: locations[0].entry.length, + datimTotalRecords: results.datimLocations.entry.length, datimTotalAllRecords: mcsdDatimAll.entry.length, - totalAllMapped, - totalAllFlagged, - totalAllNoMatch, - mohTotalAllNotMapped, - mohTotalAllRecords: mcsdMohAll.entry.length - 1, + totalAllMapped: totalAllMapped, + totalAllFlagged: totalAllFlagged, + totalAllNoMatch: totalAllNoMatch, + mohTotalAllNotMapped: mohTotalAllNotMapped, + mohTotalAllRecords: mcsdMohAll.entry.length - 1 }); winston.info('Score results sent back'); - }); + }) }); } - }).catch((err) => { - winston.error(err); - }); + }) } function recoStatus(orgid, callback) { - // getting total Mapped - const database = config.getConf('mapping:dbPrefix') + orgid; - const url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') + //getting total Mapped + var database = config.getConf('mapping:dbPrefix') + orgid; + var url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') .toString(); const options = { url, }; - let totalAllMapped = 0; - let totalAllNoMatch = 0; - let totalAllFlagged = 0; - const mohTotalAllNotMapped = 0; + var totalAllMapped = 0 + var totalAllNoMatch = 0 + var totalAllFlagged = 0 + var mohTotalAllNotMapped = 0 const noMatchCode = config.getConf('mapping:noMatchCode'); const flagCode = config.getConf('mapping:flagCode'); setTimeout(() => { mcsd.getLocations(database, (body) => { if (!body.hasOwnProperty('entry') || body.length === 0) { - totalAllNoMatch = 0; - totalAllMapped = 0; - return callback(totalAllMapped, mohTotalAllNotMapped, totalAllNoMatch, totalAllFlagged); + totalAllNoMatch = 0 + totalAllMapped = 0 + return callback(totalAllMapped, mohTotalAllNotMapped, totalAllNoMatch, totalAllFlagged) } async.each(body.entry, (entry, nxtEntry) => { if (entry.resource.hasOwnProperty('tag')) { - const nomatch = entry.resource.tag.find(tag => tag.code === noMatchCode); - const flagged = entry.resource.tag.find(tag => tag.code === flagCode); + var nomatch = entry.resource.tag.find((tag) => { + return tag.code === noMatchCode + }) + var flagged = entry.resource.tag.find((tag) => { + return tag.code === flagCode + }) if (nomatch) { - totalAllNoMatch++; + totalAllNoMatch++ } if (flagged) { - totalAllFlagged++; + totalAllFlagged++ } - return nxtEntry(); + return nxtEntry() + } else { + return nxtEntry() } - - return nxtEntry(); }, () => { - totalAllMapped = body.entry.length - totalAllNoMatch - totalAllFlagged; - return callback(totalAllMapped, totalAllNoMatch, totalAllFlagged); - // res.set('Access-Control-Allow-Origin', '*'); - // res.status(200).json({totalAllMapped,totalAllNoMatch,totalAllFlagged}) - }); - }); - }, 1000); + totalAllMapped = body.entry.length - totalAllNoMatch - totalAllFlagged + return callback(totalAllMapped, totalAllNoMatch, totalAllFlagged) + //res.set('Access-Control-Allow-Origin', '*'); + //res.status(200).json({totalAllMapped,totalAllNoMatch,totalAllFlagged}) + }) + }) + }, 1000) } + }); app.get('/getUnmatched/:orgid/:source/:recoLevel', (req, res) => { winston.info(`Getting DATIM Unmatched Orgs for ${req.params.orgid}`); if (!req.params.orgid || !req.params.source) { winston.error({ - error: 'Missing Orgid or Source', + error: 'Missing Orgid or Source' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid or Source', + error: 'Missing Orgid or Source' }); return; } const orgid = req.params.orgid; const source = req.params.source.toUpperCase(); - const recoLevel = req.params.recoLevel; + let recoLevel = req.params.recoLevel; + if (levelMaps[orgid] && levelMaps[orgid][recoLevel]) { + recoLevel = levelMaps[orgid][recoLevel]; + } const datimDB = config.getConf('mCSD:database'); mcsd.getLocationChildren(datimDB, orgid, (locations) => { - mcsd.filterLocations(locations, orgid, 0, recoLevel, 0, (mcsdLevels, mcsdLevel, mcsdBuildings) => { + mcsd.filterLocations(locations, orgid, recoLevel, (mcsdLevel) => { scores.getUnmatched(locations, mcsdLevel, orgid, (unmatched) => { winston.info(`sending back DATIM unmatched Orgs for ${req.params.orgid}`); res.set('Access-Control-Allow-Origin', '*'); @@ -550,11 +660,11 @@ if (cluster.isMaster) { winston.info('Received data for matching'); if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); return; } @@ -568,11 +678,11 @@ if (cluster.isMaster) { const totalLevels = fields.totalLevels; if (!mohId || !datimId) { winston.error({ - error: 'Missing either MOHID or DATIMID or both', + error: 'Missing either MOHID or DATIMID or both' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing either MOHID or DATIMID or both', + error: 'Missing either MOHID or DATIMID or both' }); return; } @@ -585,7 +695,7 @@ if (cluster.isMaster) { res.set('Access-Control-Allow-Origin', '*'); if (err) { res.status(401).send({ - error: err, + error: err }); } else { res.status(200).send(); @@ -598,11 +708,11 @@ if (cluster.isMaster) { winston.info('Received data for marking flag as a match'); if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); return; } @@ -614,11 +724,11 @@ if (cluster.isMaster) { const totalLevels = fields.totalLevels; if (!datimId) { winston.error({ - error: 'Missing DATIMID', + error: 'Missing DATIMID' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing DATIMID', + error: 'Missing DATIMID' }); return; } @@ -629,11 +739,10 @@ if (cluster.isMaster) { mcsd.acceptFlag(datimId, orgid, (err) => { winston.info('Done marking flag as a match'); res.set('Access-Control-Allow-Origin', '*'); - if (err) { - res.status(401).send({ - error: err, - }); - } else res.status(200).send(); + if (err) res.status(401).send({ + error: err + }); + else res.status(200).send(); }); }); }); @@ -642,11 +751,11 @@ if (cluster.isMaster) { winston.info('Received data for matching'); if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); return; } @@ -658,11 +767,11 @@ if (cluster.isMaster) { const totalLevels = fields.totalLevels; if (!mohId) { winston.error({ - error: 'Missing either MOHID', + error: 'Missing either MOHID' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing either MOHID', + error: 'Missing either MOHID' }); return; } @@ -673,11 +782,10 @@ if (cluster.isMaster) { mcsd.saveNoMatch(mohId, orgid, recoLevel, totalLevels, (err) => { winston.info('Done matching'); res.set('Access-Control-Allow-Origin', '*'); - if (err) { - res.status(401).send({ - error: err, - }); - } else res.status(200).send(); + if (err) res.status(401).send({ + error: err + }); + else res.status(200).send(); }); }); }); @@ -685,11 +793,11 @@ if (cluster.isMaster) { app.post('/breakMatch/:orgid', (req, res) => { if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); return; } @@ -709,27 +817,27 @@ if (cluster.isMaster) { app.post('/breakNoMatch/:orgid', (req, res) => { if (!req.params.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); return; } const form = new formidable.IncomingForm(); form.parse(req, (err, fields, files) => { winston.info(`Received break no match request for ${fields.mohId}`); - let mohId = fields.mohId; + var mohId = fields.mohId; if (!mohId) { winston.error({ - error: 'Missing MOH ID', - }); + 'error': 'Missing MOH ID' + }) res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing MOH ID', + error: 'Missing MOH ID' }); - return; + return } const recoLevel = fields.recoLevel; const totalLevels = fields.totalLevels; @@ -747,84 +855,84 @@ if (cluster.isMaster) { }); app.get('/markRecoUnDone/:orgid', (req, res) => { - winston.info(`received a request to mark reconciliation for ${req.params.orgid} as undone`); - const mongoUser = config.getConf('mCSD:databaseUser'); - const mongoPasswd = config.getConf('mCSD:databasePassword'); - const mongoHost = config.getConf('mCSD:databaseHost'); - const mongoPort = config.getConf('mCSD:databasePort'); - const orgid = req.params.orgid; - const database = config.getConf('mapping:dbPrefix') + orgid; + winston.info(`received a request to mark reconciliation for ${req.params.orgid} as undone`) + const mongoUser = config.getConf('mCSD:databaseUser') + const mongoPasswd = config.getConf('mCSD:databasePassword') + const mongoHost = config.getConf('mCSD:databaseHost') + const mongoPort = config.getConf('mCSD:databasePort') + const orgid = req.params.orgid + const database = config.getConf('mapping:dbPrefix') + orgid if (mongoUser && mongoPasswd) { - var uri = `mongodb://${mongoUser}:${mongoPasswd}@${mongoHost}:${mongoPort}/${database}`; + var uri = `mongodb://${mongoUser}:${mongoPasswd}@${mongoHost}:${mongoPort}/${database}` } else { - var uri = `mongodb://${mongoHost}:${mongoPort}/${database}`; + var uri = `mongodb://${mongoHost}:${mongoPort}/${database}` } - mongoose.connect(uri); - const Schema = mongoose.Schema; - let ReconciliationStatusModel; + mongoose.connect(uri) + const Schema = mongoose.Schema + let ReconciliationStatusModel try { - ReconciliationStatusModel = mongoose.model('ReconciliationStatus'); + ReconciliationStatusModel = mongoose.model('ReconciliationStatus') } catch (e) { mongoose.model('ReconciliationStatus', new Schema({ status: { - type: Object, - }, - })); - ReconciliationStatusModel = mongoose.model('ReconciliationStatus'); + type: Object + } + })) + ReconciliationStatusModel = mongoose.model('ReconciliationStatus') } const recoStatusCode = config.getConf('mapping:recoStatusCode'); const query = { status: { code: recoStatusCode, - text: 'Done', - }, - }; + text: 'Done' + } + } const update = { status: { code: recoStatusCode, - text: 'on-progress', - }, - }; + text: 'on-progress' + } + } ReconciliationStatusModel.findOneAndUpdate(query, update, (err, data) => { res.set('Access-Control-Allow-Origin', '*'); if (err) { res.status(500).json({ - error: 'An error occured while processing request', + error: 'An error occured while processing request' }); } else { res.status(200).json({ - status: 'on-progresss', + status: 'on-progresss' }); } - }); - }); + }) + }) app.get('/markRecoDone/:orgid', (req, res) => { - winston.info(`received a request to mark reconciliation for ${req.params.orgid} as done`); - const mongoUser = config.getConf('mCSD:databaseUser'); - const mongoPasswd = config.getConf('mCSD:databasePassword'); - const mongoHost = config.getConf('mCSD:databaseHost'); - const mongoPort = config.getConf('mCSD:databasePort'); - const orgid = req.params.orgid; - const database = config.getConf('mapping:dbPrefix') + orgid; + winston.info(`received a request to mark reconciliation for ${req.params.orgid} as done`) + const mongoUser = config.getConf('mCSD:databaseUser') + const mongoPasswd = config.getConf('mCSD:databasePassword') + const mongoHost = config.getConf('mCSD:databaseHost') + const mongoPort = config.getConf('mCSD:databasePort') + const orgid = req.params.orgid + const database = config.getConf('mapping:dbPrefix') + orgid if (mongoUser && mongoPasswd) { - var uri = `mongodb://${mongoUser}:${mongoPasswd}@${mongoHost}:${mongoPort}/${database}`; + var uri = `mongodb://${mongoUser}:${mongoPasswd}@${mongoHost}:${mongoPort}/${database}` } else { - var uri = `mongodb://${mongoHost}:${mongoPort}/${database}`; + var uri = `mongodb://${mongoHost}:${mongoPort}/${database}` } - mongoose.connect(uri); - const Schema = mongoose.Schema; - let ReconciliationStatusModel; + mongoose.connect(uri) + const Schema = mongoose.Schema + let ReconciliationStatusModel try { - ReconciliationStatusModel = mongoose.model('ReconciliationStatus'); + ReconciliationStatusModel = mongoose.model('ReconciliationStatus') } catch (e) { mongoose.model('ReconciliationStatus', new Schema({ status: { - type: Object, - }, - })); - ReconciliationStatusModel = mongoose.model('ReconciliationStatus'); + type: Object + } + })) + ReconciliationStatusModel = mongoose.model('ReconciliationStatus') } const recoStatusCode = config.getConf('mapping:recoStatusCode'); @@ -832,114 +940,114 @@ if (cluster.isMaster) { ReconciliationStatusModel.findOne({ status: { code: recoStatusCode, - text: 'on-progress', - }, + text: 'on-progress' + } }, (err, data) => { if (err) { - winston.error('Unexpected error occured,please retry'); + winston.error('Unexpected error occured,please retry') res.status(500).json({ - error: 'Unexpected error occured,please retry', + error: 'Unexpected error occured,please retry' }); - return; + return } if (!data) { - const recoStatus = new ReconciliationStatusModel({ + var recoStatus = new ReconciliationStatusModel({ status: { code: recoStatusCode, - text: 'Done', - }, - }); - recoStatus.save((err, data) => { + text: 'Done' + } + }) + recoStatus.save(function (err, data) { if (err) { - winston.error('Unexpected error occured,please retry'); + winston.error('Unexpected error occured,please retry') res.set('Access-Control-Allow-Origin', '*'); res.status(500).json({ - error: 'Unexpected error occured,please retry', + error: 'Unexpected error occured,please retry' }); } - winston.info(`${orgid} marked as done with reconciliation`); + winston.info(`${orgid} marked as done with reconciliation`) res.set('Access-Control-Allow-Origin', '*'); res.status(200).json({ - status: 'done', + status: 'done' }); - }); + }) } else { ReconciliationStatusModel.findByIdAndUpdate(data.id, { status: { code: recoStatusCode, - text: 'Done', - }, + text: 'Done' + } }, (err, data) => { if (err) { - winston.error('Unexpected error occured,please retry'); + winston.error('Unexpected error occured,please retry') res.set('Access-Control-Allow-Origin', '*'); res.status(500).json({ - error: 'Unexpected error occured,please retry', + error: 'Unexpected error occured,please retry' }); - return; + return } - winston.info(`${orgid} already marked as done with reconciliation`); + winston.info(`${orgid} already marked as done with reconciliation`) res.set('Access-Control-Allow-Origin', '*'); res.status(200).json({ - status: 'done', + status: 'done' }); - }); + }) } - }); - }); + }) + }) app.get('/recoStatus/:orgid', (req, res) => { - const mongoUser = config.getConf('mCSD:databaseUser'); - const mongoPasswd = config.getConf('mCSD:databasePassword'); - const mongoHost = config.getConf('mCSD:databaseHost'); - const mongoPort = config.getConf('mCSD:databasePort'); - const orgid = req.params.orgid; - const database = config.getConf('mapping:dbPrefix') + orgid; + const mongoUser = config.getConf('mCSD:databaseUser') + const mongoPasswd = config.getConf('mCSD:databasePassword') + const mongoHost = config.getConf('mCSD:databaseHost') + const mongoPort = config.getConf('mCSD:databasePort') + const orgid = req.params.orgid + const database = config.getConf('mapping:dbPrefix') + orgid if (mongoUser && mongoPasswd) { - var uri = `mongodb://${mongoUser}:${mongoPasswd}@${mongoHost}:${mongoPort}/${database}`; + var uri = `mongodb://${mongoUser}:${mongoPasswd}@${mongoHost}:${mongoPort}/${database}` } else { - var uri = `mongodb://${mongoHost}:${mongoPort}/${database}`; + var uri = `mongodb://${mongoHost}:${mongoPort}/${database}` } - mongoose.connect(uri); - const Schema = mongoose.Schema; + mongoose.connect(uri) + const Schema = mongoose.Schema const recoStatusCode = config.getConf('mapping:recoStatusCode'); - let ReconciliationStatusModel; + let ReconciliationStatusModel try { - ReconciliationStatusModel = mongoose.model('ReconciliationStatus'); + ReconciliationStatusModel = mongoose.model('ReconciliationStatus') } catch (e) { mongoose.model('ReconciliationStatus', new Schema({ status: { - type: Object, - }, - })); - ReconciliationStatusModel = mongoose.model('ReconciliationStatus'); + type: Object + } + })) + ReconciliationStatusModel = mongoose.model('ReconciliationStatus') } res.set('Access-Control-Allow-Origin', '*'); ReconciliationStatusModel.findOne({ status: { code: recoStatusCode, - text: 'Done', - }, + text: 'Done' + } }, (err, data) => { if (err) { res.status(500).json({ - error: 'Unexpected error occured,please retry', + error: 'Unexpected error occured,please retry' }); - return; + return } if (data) { res.status(200).json({ - status: 'done', + status: 'done' }); } else { res.status(200).json({ - status: 'on-progress', + status: 'on-progress' }); } - }); - }); + }) + }) app.get('/progress/:type/:clientId', (req, res) => { const clientId = req.params.clientId; @@ -960,64 +1068,65 @@ if (cluster.isMaster) { res.status(200).json(results); }); }); + app.get('/uploadProgress/:orgid/:clientId', (req, res) => { - const orgid = req.params.orgid; - const clientId = req.params.clientId; + const orgid = req.params.orgid + const clientId = req.params.clientId redisClient.get(`uploadProgress${orgid}${clientId}`, (error, results) => { - results = JSON.parse(results); + results = JSON.parse(results) res.set('Access-Control-Allow-Origin', '*'); - res.status(200).json(results); - // reset progress + res.status(200).json(results) + //reset progress if (results && (results.error !== null || results.status === 'Done')) { - const uploadRequestId = `uploadProgress${orgid}${clientId}`; - const uploadReqPro = JSON.stringify({ + var uploadRequestId = `uploadProgress${orgid}${clientId}` + let uploadReqPro = JSON.stringify({ status: null, error: null, - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) } - }); + }) }); app.get('/mappingStatusProgress/:orgid/:clientId', (req, res) => { - const orgid = req.params.orgid; - const clientId = req.params.clientId; + const orgid = req.params.orgid + const clientId = req.params.clientId redisClient.get(`mappingStatus${orgid}${clientId}`, (error, results) => { - results = JSON.parse(results); + results = JSON.parse(results) res.set('Access-Control-Allow-Origin', '*'); - res.status(200).json(results); - // reset progress + res.status(200).json(results) + //reset progress if (results && (results.error !== null || results.status === 'Done')) { - const statusRequestId = `mappingStatus${orgid}${clientId}`; - const statusResData = JSON.stringify({ + var statusRequestId = `mappingStatus${orgid}${clientId}` + let statusResData = JSON.stringify({ status: null, error: null, - percent: null, - }); - redisClient.set(statusRequestId, statusResData); + percent: null + }) + redisClient.set(statusRequestId, statusResData) } - }); + }) }); app.get('/scoreProgress/:orgid/:clientId', (req, res) => { - const orgid = req.params.orgid; - const clientId = req.params.clientId; + const orgid = req.params.orgid + const clientId = req.params.clientId redisClient.get(`scoreResults${orgid}${clientId}`, (error, results) => { - results = JSON.parse(results); + results = JSON.parse(results) res.set('Access-Control-Allow-Origin', '*'); - res.status(200).json(results); - // reset progress + res.status(200).json(results) + //reset progress if (results && (results.error !== null || results.status === 'Done')) { - const scoreRequestId = `scoreResults${orgid}${clientId}`; - const uploadReqPro = JSON.stringify({ + const scoreRequestId = `scoreResults${orgid}${clientId}` + let uploadReqPro = JSON.stringify({ status: null, error: null, - percent: null, - }); - redisClient.set(scoreRequestId, uploadReqPro); + percent: null + }) + redisClient.set(scoreRequestId, uploadReqPro) } - }); + }) }); app.post('/addServer', (req, res) => { @@ -1102,11 +1211,11 @@ if (cluster.isMaster) { winston.info(`Received MOH Data with fields Mapping ${JSON.stringify(fields)}`); if (!fields.orgid) { winston.error({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Missing Orgid', + error: 'Missing Orgid' }); return; } @@ -1114,27 +1223,19 @@ if (cluster.isMaster) { const orgname = fields.orgname; const database = config.getConf('mCSD:database'); const expectedLevels = config.getConf('levels'); - const clientId = fields.clientId; - const uploadRequestId = `uploadProgress${orgid}${clientId}`; - const uploadReqPro = JSON.stringify({ + const clientId = fields.clientId + var uploadRequestId = `uploadProgress${orgid}${clientId}` + let uploadReqPro = JSON.stringify({ status: 'Request received by server', error: null, - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) if (!Array.isArray(expectedLevels)) { winston.error('Invalid config data for key Levels '); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Un expected error occured while processing this request', - }); - res.end(); - return; - } - if (Object.keys(files).length == 0) { - winston.error('No file submitted for reconciliation'); - res.status(401).json({ - error: 'Please submit CSV file for facility reconciliation', + error: 'Un expected error occured while processing this request' }); res.end(); return; @@ -1149,14 +1250,18 @@ if (cluster.isMaster) { } const fileName = Object.keys(files)[0]; winston.info('validating CSV File'); - validateCSV(fields, (valid, missing) => { - if (!valid) { - winston.error({ - MissingHeaders: missing, - }); + uploadReqPro = JSON.stringify({ + status: '2/5 Validating CSV Data', + error: null, + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) + validateCSV(files[fileName].path, fields, (valid, invalid) => { + if (invalid.length > 0) { + winston.error("Uploaded CSV is invalid (has either duplicated IDs or empty levels/facility),execution stopped"); res.set('Access-Control-Allow-Origin', '*'); res.status(401).json({ - error: 'Some Headers are Missing', + error: invalid }); res.end(); return; @@ -1165,96 +1270,150 @@ if (cluster.isMaster) { res.status(200).end(); } winston.info('CSV File Passed Validation'); - // archive existing DB first - const uploadReqPro = JSON.stringify({ - status: '2/4 Archiving Old DB', + //archive existing DB first + let uploadReqPro = JSON.stringify({ + status: '3/5 Archiving Old DB', error: null, - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) mcsd.archiveDB(orgid, (err) => { if (err) { - const uploadReqPro = JSON.stringify({ - status: '1/3 Archiving Old DB', + let uploadReqPro = JSON.stringify({ + status: '3/5 Archiving Old DB', error: 'An error occured while archiving Database,retry', - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); - winston.error('An error occured while Archiving existing DB,Upload of new dataset was stopped'); - return; + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) + winston.error('An error occured while Archiving existing DB,Upload of new dataset was stopped') + return } - // ensure old archives are deleted - const uploadReqPro = JSON.stringify({ - status: '3/4 Deleting Old DB', + //ensure old archives are deleted + let uploadReqPro = JSON.stringify({ + status: '4/5 Deleting Old DB', error: null, - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); - mcsd.cleanArchives(orgid, () => {}); - // delete existing db + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) + mcsd.cleanArchives(orgid, () => {}) + //delete existing db winston.info('Deleting DB ' + orgid) mcsd.deleteDB(orgid, (err) => { if (!err) { - winston.info(`Uploading data for ${orgid} now`); - const uploadReqPro = JSON.stringify({ - status: '4/4 Uploading of DB started', + winston.info(`Uploading data for ${orgid} now`) + let uploadReqPro = JSON.stringify({ + status: '5/5 Uploading of DB started', error: null, - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) mcsd.CSVTomCSD(files[fileName].path, fields, orgid, clientId, () => { - winston.info(`Data upload for ${orgid} is done`); - const uploadReqPro = JSON.stringify({ + winston.info(`Data upload for ${orgid} is done`) + let uploadReqPro = JSON.stringify({ status: 'Done', error: null, - percent: 100, - }); - redisClient.set(uploadRequestId, uploadReqPro); + percent: 100 + }) + redisClient.set(uploadRequestId, uploadReqPro) }); } else { - winston.error('An error occured while dropping existing DB,Upload of new dataset was stopped'); - const uploadReqPro = JSON.stringify({ + winston.error('An error occured while dropping existing DB,Upload of new dataset was stopped') + let uploadReqPro = JSON.stringify({ status: '3/4 Deleting Old DB', error: 'An error occured while dropping existing Database,retry', - percent: null, - }); - redisClient.set(uploadRequestId, uploadReqPro); + percent: null + }) + redisClient.set(uploadRequestId, uploadReqPro) } - }); - }); + }) + }) }); }); + function validateCSV(filePath, headerMapping, callback) { + let invalid = [] + let ids = [] + const levels = config.getConf('levels'); + levels.sort(); + levels.reverse(); + csv + .fromPath(filePath, { + headers: true, + }) + .on('data', (data) => { + let rowMarkedInvalid = false + let index = 0 + async.eachSeries(levels, (level, nxtLevel) => { + if (headerMapping[level] === null || + headerMapping[level] === 'null' || + headerMapping[level] === undefined || + !headerMapping[level]) { + return nxtLevel() + } + if (index === 0) { + index++ + if (ids.length == 0) { + ids.push(data[headerMapping.code]) + } else { + let idExist = ids.find((id) => { + return id === data[headerMapping.code] + }) + if (idExist) { + rowMarkedInvalid = true + let reason = 'Duplicate ID' + populateData(headerMapping, data, reason, invalid) + } else { + ids.push(data[headerMapping.code]) + } + } + } + if (!rowMarkedInvalid) { + if (data[headerMapping[level]] === null || + data[headerMapping[level]] === undefined || + data[headerMapping[level]] === false || + !data[headerMapping[level]] || + data[headerMapping[level]] === '' || + !isNaN(headerMapping[level]) || + data[headerMapping[level]] == 0) { + let reason = headerMapping[level] + ' is blank' + populateData(headerMapping, data, reason, invalid) + } else { + return nxtLevel() + } + } + }, () => { + if (data[headerMapping.facility] === null || + data[headerMapping.facility] === undefined || + data[headerMapping.facility] === false || + data[headerMapping.facility] === '' || + data[headerMapping.facility] == 0) { + let reason = headerMapping.facility + ' is blank' + populateData(headerMapping, data, reason, invalid) - function validateCSV(cols, callback) { - const missing = []; - if (!cols.hasOwnProperty('facility') || cols.facility === null || cols.facility === undefined || cols.facility === false) { - missing.push('facility'); - } - if (!cols.hasOwnProperty('code') || cols.code === null || cols.code === undefined || cols.code === false) { - missing.push('code'); - } - if (!cols.hasOwnProperty('lat') || cols.lat === null || cols.lat === undefined || cols.lat === false) { - missing.push('lat'); - } - if (!cols.hasOwnProperty('long') || cols.long === null || cols.long === undefined || cols.long === false) { - missing.push('long'); - } - if (!cols.hasOwnProperty('level1') || cols.level1 === null || cols.level1 === undefined || cols.facility === false) { - missing.push('level1'); - } - if (!cols.hasOwnProperty('level2') || cols.level2 === null || cols.level2 === undefined || cols.level2 === false) { - missing.push('level2'); - } - if (!cols.hasOwnProperty('level3') || cols.level3 === null || cols.level3 === undefined || cols.level3 === false) { - missing.push('level3'); - } - if (!cols.hasOwnProperty('level4') || cols.level4 === null || cols.level4 === undefined || cols.level4 === false) { - missing.push('level4'); - } - if (missing.length > 0) { - return callback(false, missing); + } + }) + }) + .on('end', () => { + return callback(true, invalid); + }) + + function populateData(headerMapping, data, reason, invalid) { + let row = {} + async.each(headerMapping, (header, nxtHeader) => { + if (header == 'null') { + return nxtHeader() + } + if (!data.hasOwnProperty(header)) { + return nxtHeader() + } + row[header] = data[header] + return nxtHeader() + }, () => { + invalid.push({ + data: row, + reason + }) + }) } - return callback(true, missing); } }); diff --git a/facility-recon-backend/lib/mcsd.js b/facility-recon-backend/lib/mcsd.js index b97d4bcdf..b886f8563 100755 --- a/facility-recon-backend/lib/mcsd.js +++ b/facility-recon-backend/lib/mcsd.js @@ -23,10 +23,26 @@ const config = require('./config'); module.exports = function () { return { getLocations(database, callback) { - let url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') + '?_count=37000' - .toString(); - const locations = {}; - locations.entry = []; + let baseUrl = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location').toString(); + let url = baseUrl + '?_count=37000' + let locations + locations = cache.get('url_' + baseUrl); + if (locations) { + winston.info("Getting " + baseUrl + " from cache"); + return callback(locations) + } else { + locations = { + entry: [] + } + } + const started = cache.get('started_' + baseUrl); + if (started) { + winston.info('getLocations is in progress will try again in 10 seconds.') + setTimeout(this.getLocations, 10000, database, callback) + return + } + cache.put('started_' + baseUrl, true); + winston.info("Getting " + baseUrl + " from server"); async.doWhilst( (callback) => { const options = { @@ -48,6 +64,13 @@ module.exports = function () { }, () => url != false, () => { + if (locations.entry.length > 1) { + winston.info("Saving " + baseUrl + " to cache"); + cache.put('url_' + baseUrl, locations, config.getConf("mCSD:cacheTime")); + } else { + winston.info("Not more than 1 entry for " + baseUrl + " so not caching."); + } + cache.del('started_' + baseUrl); callback(locations); }, ); @@ -68,21 +91,16 @@ module.exports = function () { url, }; url = false; - const cachedData = cache.get(`getLocationByID${url}`); - if (cachedData && getCached) { - return callback(cachedData); - } request.get(options, (err, res, body) => { if (!isJSON(body)) { return callback(false, false); } - const cacheData = JSON.parse(body); - const next = cacheData.link.find(link => link.relation == 'next'); + const mcsd = JSON.parse(body); + const next = mcsd.link.find(link => link.relation == 'next'); if (next) { url = next.url; } - cache.put(`getLocationByID${url}`, cacheData, 120 * 2000); - locations.entry = locations.entry.concat(cacheData.entry); + locations.entry = locations.entry.concat(mcsd.entry); return callback(false, url); }); }, @@ -131,6 +149,22 @@ module.exports = function () { .segment(topOrgId) .segment('$hierarchy') .toString(); + + let data = cache.get('url_' + url); + if (data) { + winston.info("Getting " + url + " from cache"); + return callback(data); + } + + const started = cache.get('started_' + url); + if (started) { + winston.info('getLocationChildren is in progress will try again in 10 seconds.') + setTimeout(this.getLocationChildren, 10000, database, topOrgId, callback) + return + } + cache.put('started_' + url, true); + winston.info("Getting " + url + " from server"); + const options = { url, }; @@ -138,9 +172,17 @@ module.exports = function () { if (!isJSON(body)) { const mcsd = {}; mcsd.entry = []; + cache.del('started_' + url); return callback(mcsd); } body = JSON.parse(body); + if (body.entry.length > 1) { + winston.info("Saving " + url + " to cache"); + cache.put('url_' + url, body, config.getConf("mCSD:cacheTime")); + } else { + winston.info("Not more than 1 entry for " + url + " so not caching."); + } + cache.del('started_' + url); callback(body); }); }, @@ -286,7 +328,7 @@ module.exports = function () { /* This function finds parents of an entity from passed mCSD data */ - getLocationParentsFromData (entityParent, mcsd, details, callback) { + getLocationParentsFromData(entityParent, mcsd, details, callback) { if (mcsd.hasOwnProperty('parentCache') && mcsd.parentCache.id === entityParent && mcsd.parentCache.details === details) { // return a copy return callback(mcsd.parentCache.parents.slice()) @@ -299,7 +341,7 @@ module.exports = function () { return callback(parents); } - function filter (entityParent, callback) { + function filter(entityParent, callback) { const splParent = entityParent.split('/'); entityParent = splParent[(splParent.length - 1)]; @@ -365,39 +407,20 @@ module.exports = function () { }) return callback(buildings) }, - /* - if totalLevels is set then this functions returns all locations from level 1 to level totalLevels - if levelNumber is set then it returns locations at level levelNumber only - if buildings is set then it returns all buildings - buildings argument accepts a building level - */ - filterLocations(mcsd, topOrgId, totalLevels, levelNumber, buildings, callback) { - // holds all entities for a maximum of x Levels defined by the variable totalLevels i.e all entities at level 1,2 and 3 - const mcsdTotalLevels = {}; - // holds all entities for just one level,specified by variable levelNumber i.e all entities at level 1 or at level 2 - const mcsdlevelNumber = {}; - // holds buildings only - const mcsdBuildings = {}; - mcsdTotalLevels.entry = []; - mcsdlevelNumber.entry = []; - mcsdBuildings.entry = []; + + filterLocations(mcsd, topOrgId, levelNumber, callback) { + const mcsdLevelNumber = {}; + mcsdLevelNumber.entry = []; if (!mcsd.hasOwnProperty('entry') || mcsd.entry.length == 0 || !topOrgId) { - return callback(mcsdTotalLevels, mcsdlevelNumber, mcsdBuildings); + return callback(mcsdLevelNumber); } const entry = mcsd.entry.find(entry => entry.resource.id == topOrgId); if (!entry) { - return callback(mcsdTotalLevels, mcsdlevelNumber, mcsdBuildings); + return callback(mcsdLevelNumber); } if (levelNumber == 1) { - mcsdlevelNumber.entry = mcsdlevelNumber.entry.concat(entry); - } - if (totalLevels) { - mcsdTotalLevels.entry = mcsdTotalLevels.entry.concat(entry); - } - const building = entry.resource.physicalType.coding.find(coding => coding.code == 'building'); - - if (building) { - mcsdBuildings.entry = mcsdBuildings.entry.concat(entry); + mcsdLevelNumber.entry = mcsdLevelNumber.entry.concat(entry); + return callback(mcsdLevelNumber); } function filter(id, callback) { @@ -410,13 +433,7 @@ module.exports = function () { } let totalLoops = 0; - if (totalLevels >= levelNumber && totalLevels >= buildings) { - totalLoops = totalLevels; - } else if (levelNumber >= totalLevels && levelNumber >= buildings) { - totalLoops = levelNumber; - } else if (buildings >= totalLevels && buildings >= levelNumber) { - totalLoops = buildings; - } + totalLoops = levelNumber; let tmpArr = []; tmpArr.push(entry); @@ -428,29 +445,11 @@ module.exports = function () { promises.push(new Promise((resolve, reject) => { filter(arr.resource.id, (res) => { tmpArr = tmpArr.concat(res); - if (totalLevels) { - mcsdTotalLevels.entry = mcsdTotalLevels.entry.concat(res); - } if (levelNumber == loop + 1) { - mcsdlevelNumber.entry = mcsdlevelNumber.entry.concat(res); + mcsdLevelNumber.entry = mcsdLevelNumber.entry.concat(res); } - const promises1 = []; - for (var k in res) { - promises1.push(new Promise((resolve1, reject1) => { - var building = res[k].resource.physicalType.coding.find((coding) => { - if (building) { - mcsdBuildings.entry = mcsdBuildings.entry.concat(entry); - } - }); - resolve1(); - })); - } - Promise.all(promises1).then(() => { - totalElements++; - resolve(); - }).catch((err) => { - winston.error(err); - }); + totalElements++; + resolve(); }); })); }); @@ -461,19 +460,28 @@ module.exports = function () { winston.error(err); }); }, () => { - callback(mcsdTotalLevels, mcsdlevelNumber, mcsdBuildings); + callback(mcsdLevelNumber); }); }, - countLevels(source, topOrgId, callback) { - const database = config.getConf('mCSD:database'); - if (source == 'MOH') { - var url = `${URI(config.getConf('mCSD:url')).segment(topOrgId).segment('fhir').segment('Location')}?partof=Location/${topOrgId.toString()}`; - } else if (source == 'DATIM') { - var url = `${URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location')}?partof=Location/${topOrgId.toString()}`; + countLevels(source, db, topOrgId, callback) { + function constructURL(id, callback) { + if (source == 'MOH') { + var url = `${URI(config.getConf('mCSD:url')) + .segment(db) + .segment('fhir') + .segment('Location')}?partof=Location/${id.toString()}`; + } else if (source == 'DATIM') { + var url = `${URI(config.getConf('mCSD:url')) + .segment(db) + .segment('fhir') + .segment('Location')}?partof=Location/${id.toString()}`; + } + return callback(url) } let totalLevels = 1; + let prev_entry = {} function cntLvls(url, callback) { const options = { @@ -484,40 +492,32 @@ module.exports = function () { return callback(0); } body = JSON.parse(body); - if (body.total == 0) { + let entry + if (body.entry.length === 0 && prev_entry.length > 0) { + entry = prev_entry.shift() + } else if (body.entry.length === 0 && Object.keys(prev_entry).length === 0) { return callback(totalLevels); - } - let counter = 0; - async.eachSeries(body.entry, (entry, nxtEntry) => { - if (entry.resource.name.startsWith('_') || counter > 0) { - return nxtEntry(); - } + } else { + prev_entry = [] + prev_entry = body.entry.slice() + entry = prev_entry.shift() totalLevels++; - counter++; - if (entry.resource.hasOwnProperty('id') && - entry.resource.id != false && - entry.resource.id != null && - entry.resource.id != undefined) { - const reference = entry.resource.id; + } + const reference = entry.resource.id; + constructURL(reference, (url) => { + cntLvls(url, totalLevels => callback(totalLevels)); + }) - if (source == 'MOH') { - var url = `${URI(config.getConf('mCSD:url')).segment(topOrgId).segment('fhir').segment('Location')}?partof=Location/${reference.toString()}`; - } else if (source == 'DATIM') { - var url = `${URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location')}?partof=Location/${reference.toString()}`; - } - cntLvls(url, totalLevels => callback(totalLevels)); - } else { - return callback(totalLevels); - } - }, () => callback(totalLevels)); }); } - cntLvls(url, totalLevels => callback(false, totalLevels)); + constructURL(topOrgId, (url) => { + cntLvls(url, totalLevels => callback(false, totalLevels)); + }) }, saveLocations(mCSD, database, callback) { const url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').toString(); const options = { - url: url.toString(), + url: url, headers: { 'Content-Type': 'application/json', }, @@ -528,9 +528,18 @@ module.exports = function () { winston.error(err); return callback(err); } + this.cleanCache(url + '/Location') callback(err, body); }); }, + cleanCache(url) { + for (const key of cache.keys()) { + if (key.substring(0, url.length + 4) === 'url_' + url) { + winston.info("DELETING " + key + " from cache because something was modified.") + cache.del(key) + } + } + }, saveMatch(mohId, datimId, topOrgId, recoLevel, totalLevels, type, callback) { const database = config.getConf('mCSD:database'); const flagCode = config.getConf('mapping:flagCode'); @@ -667,13 +676,13 @@ module.exports = function () { flagged.type = 'document'; // deleting existing location - const url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') - .segment(datimId) - .toString(); + const url_prefix = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') + const url = URI(url_prefix).segment(datimId).toString(); const options = { url, }; request.delete(options, (err, res, body) => { + this.cleanCache(url_prefix.toString()); if (err) { winston.error(err); return callback(err); @@ -776,9 +785,11 @@ module.exports = function () { }); }, breakMatch(id, database, topOrgId, callback) { - const url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') - .segment(id) - .toString(); + const url_prefix = URI(config.getConf('mCSD:url')) + .segment(database) + .segment('fhir') + .segment('Location') + const url = URI(url_prefix).segment(id).toString() const options = { url, }; @@ -787,6 +798,7 @@ module.exports = function () { return callback(true, null); } request.delete(options, (err, res, body) => { + this.cleanCache(url_prefix.toString()); if (err) { winston.error(err); } @@ -829,17 +841,18 @@ module.exports = function () { }); }, breakNoMatch(id, database, callback) { - const url = URI(config.getConf('mCSD:url')).segment(database).segment('fhir').segment('Location') + const url_prefix = URI(config.getConf('mCSD:url')) + .segment(database) + .segment('fhir') + .segment('Location') + const url = URI(url_prefix) .segment(id) .toString(); const options = { url, }; - const cachedData = cache.get(`getLocationByID${url}`); - if (cachedData) { - return callback(cachedData); - } request.delete(options, (err, res, body) => { + this.cleanCache(url_prefix.toString()); if (err) { winston.error(err); } @@ -878,7 +891,7 @@ module.exports = function () { countRow++ const percent = parseFloat((countRow * 100 / totalRows).toFixed(2)); const uploadReqPro = JSON.stringify({ - status: '4/4 Writing Uploaded data into server', + status: '5/5 Writing Uploaded data into server', error: null, percent, }); @@ -896,7 +909,7 @@ module.exports = function () { data[headerMapping[level]] != false && data[headerMapping[level]] != '' ) { - const name = data[headerMapping[level]]; + const name = data[headerMapping[level]].trim(); const levelNumber = level.replace('level', ''); if (levelNumber.toString().length < 2) { var namespaceMod = `${namespace}00${levelNumber}`; @@ -925,8 +938,8 @@ module.exports = function () { } async.eachSeries(topLevels, (topLevel, nxtTopLevel) => { const topLevelName = `level${topLevel}`; - if (data[headerMapping[topLevelName]] != '' && parentFound == false) { - parent = data[headerMapping[topLevelName]]; + if (data[headerMapping[topLevelName]] && parentFound == false) { + parent = data[headerMapping[topLevelName]].trim(); if (topLevel.toString().length < 2) { var namespaceMod = `${namespace}00${topLevel}`; } else { @@ -963,7 +976,7 @@ module.exports = function () { } recordCount += jurisdictions.length this.buildJurisdiction(jurisdictions, saveBundle) - const facilityName = data[headerMapping.facility]; + const facilityName = data[headerMapping.facility].trim(); const UUID = uuid5(data[headerMapping.code], `${namespace}100`); const building = { uuid: UUID, @@ -992,7 +1005,7 @@ module.exports = function () { countRow += tmpBundle.entry.length const percent = parseFloat((countRow * 100 / totalRows).toFixed(2)); const uploadReqPro = JSON.stringify({ - status: '4/4 Writing Uploaded data into server', + status: '5/5 Writing Uploaded data into server', error: null, percent, }); @@ -1007,7 +1020,6 @@ module.exports = function () { this.saveLocations(saveBundle, orgid, () => { Promise.all(promises).then(() => { var uploadRequestId = `uploadProgress${orgid}${clientId}`; - winston.info('done') const uploadReqPro = JSON.stringify({ status: 'Done', error: null, diff --git a/facility-recon-backend/package-lock.json b/facility-recon-backend/package-lock.json index 8472f760d..e02cd9033 100755 --- a/facility-recon-backend/package-lock.json +++ b/facility-recon-backend/package-lock.json @@ -1023,6 +1023,22 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cors": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.4.tgz", + "integrity": "sha1-K9OB8usgECAQXNUOpZ2mMJBpRoY=", + "requires": { + "object-assign": "^4", + "vary": "^1" + }, + "dependencies": { + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + } + } + }, "create-error-class": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", diff --git a/facility-recon-backend/package.json b/facility-recon-backend/package.json index caec91524..1680b6750 100755 --- a/facility-recon-backend/package.json +++ b/facility-recon-backend/package.json @@ -24,6 +24,7 @@ "dependencies": { "async": "^2.6.1", "body-parser": "^1.18.3", + "cors": "^2.8.4", "express": "^4.16.3", "express-formidable": "^1.0.0", "fast-csv": "^2.4.1", diff --git a/facility-recon-gui/src/App.vue b/facility-recon-gui/src/App.vue index c9c8a2997..6099aee6d 100755 --- a/facility-recon-gui/src/App.vue +++ b/facility-recon-gui/src/App.vue @@ -118,8 +118,8 @@ export default { var orgUnit = this.$store.state.orgUnit axios.get(backendServer + '/countLevels/' + orgUnit.OrgId).then((levels) => { this.initializingApp = false - this.$store.state.totalLevels = levels.data.totalLevels - // this.getOrgHierarchy() + this.$store.state.totalMOHLevels = levels.data.totalMOHLevels + this.$store.state.totalDATIMLevels = levels.data.totalDATIMLevels this.getScores() }) }, @@ -165,7 +165,6 @@ export default { this.$root.$on('reloadTree', () => { this.$store.state.mohHierarchy = '' this.$store.state.datimHierarchy = '' - // this.getOrgHierarchy() }) this.$root.$on('refreshApp', () => { this.getTotalLevels() diff --git a/facility-recon-gui/src/components/DataSync/FacilityReconDataSync.vue b/facility-recon-gui/src/components/DataSync/FacilityReconDataSync.vue index bbedbd921..4a47e2f98 100644 --- a/facility-recon-gui/src/components/DataSync/FacilityReconDataSync.vue +++ b/facility-recon-gui/src/components/DataSync/FacilityReconDataSync.vue @@ -74,7 +74,7 @@ - + syncSync (Update) @@ -82,6 +82,7 @@ syncForce Full Sync + editEdit diff --git a/facility-recon-gui/src/components/DataSync/FacilityReconUpload.vue b/facility-recon-gui/src/components/DataSync/FacilityReconUpload.vue index eba706a6a..d97649300 100755 --- a/facility-recon-gui/src/components/DataSync/FacilityReconUpload.vue +++ b/facility-recon-gui/src/components/DataSync/FacilityReconUpload.vue @@ -1,6 +1,6 @@