From cf539e3ec145d0713d77300f0dea3498ab1ee21f Mon Sep 17 00:00:00 2001 From: Ally Shaban Date: Mon, 5 Oct 2020 08:20:10 +0300 Subject: [PATCH] adding an option on either to automatch or not patients that has human adjudication tag. add PUT end point for submitting patients --- .../config/config_development_template.json | 26 +- server/config/config_production_template.json | 26 +- server/config/config_test_template.json | 7 +- server/lib/esMatching.js | 1 - server/lib/mixins/matchMixin.js | 232 ++++++++++++------ server/lib/routes/fhir.js | 14 +- server/lib/routes/match.js | 20 ++ tests/sampleSinglePatient.json | 34 +-- ui/src/views/Resolve.vue | 2 +- 9 files changed, 244 insertions(+), 118 deletions(-) diff --git a/server/config/config_development_template.json b/server/config/config_development_template.json index 95bcb740..de5e9ace 100644 --- a/server/config/config_development_template.json +++ b/server/config/config_development_template.json @@ -18,9 +18,9 @@ "register": false }, "fhirServer": { - "baseURL": "http://localhost:8081/clientregistry/fhir", "username": "hapi", - "password": "hapi" + "password": "hapi", + "baseURL": "http://localhost:8080/hapi/fhir" }, "elastic": { "server": "http://localhost:9200", @@ -30,7 +30,8 @@ "index": "patients" }, "matching": { - "tool": "elasticsearch" + "tool": "elasticsearch", + "autoMatchPatientWithHumanAdjudTag": false }, "codes": { "goldenRecord": "5c827da5-4858-4f3d-a50c-62ece001efea" @@ -44,9 +45,17 @@ "id": "openmrs", "displayName": "OpenMRS" }, + { + "id": "dhis2", + "displayName": "DHIS2" + }, { "id": "lims", "displayName": "Lab Info Management System" + }, + { + "id": "cr", + "displayName": "Client Registry" } ], "systems": { @@ -54,16 +63,19 @@ "internalid": { "uri": [ "http://health.go.ug/cr/internalid", - "http://clientregistry.org/openmrs" + "http://openmrs.org/openmrs2", + "http://clientregistry.org/openmrs", + "http://clientregistry.org/dhis2", + "http://clientregistry.org/lims" ], "displayName": "Internal ID" }, "nationalid": { - "uri": "http://health.go.ug/cr/natioanlid", + "uri": "http://clientregistry.org/cr/natioanlid", "displayName": "National ID" }, "artnumber": { - "uri": "http://health.go.ug/cr/artnumber", + "uri": "http://clientregistry.org/cr/artnumber", "displayName": "ART Number" }, "brokenMatch": { @@ -71,7 +83,7 @@ } }, "sync": { - "lastFHIR2ESSync": "2020-02-21T14:51:41" + "lastFHIR2ESSync": "1970-10-05T08:15:12" }, "__comments": { "matching.tool": "this tells if the app should use mediator algorithms or elasticsearch algorithms for matching, two options mediator and elasticsearch" diff --git a/server/config/config_production_template.json b/server/config/config_production_template.json index 95bcb740..de5e9ace 100644 --- a/server/config/config_production_template.json +++ b/server/config/config_production_template.json @@ -18,9 +18,9 @@ "register": false }, "fhirServer": { - "baseURL": "http://localhost:8081/clientregistry/fhir", "username": "hapi", - "password": "hapi" + "password": "hapi", + "baseURL": "http://localhost:8080/hapi/fhir" }, "elastic": { "server": "http://localhost:9200", @@ -30,7 +30,8 @@ "index": "patients" }, "matching": { - "tool": "elasticsearch" + "tool": "elasticsearch", + "autoMatchPatientWithHumanAdjudTag": false }, "codes": { "goldenRecord": "5c827da5-4858-4f3d-a50c-62ece001efea" @@ -44,9 +45,17 @@ "id": "openmrs", "displayName": "OpenMRS" }, + { + "id": "dhis2", + "displayName": "DHIS2" + }, { "id": "lims", "displayName": "Lab Info Management System" + }, + { + "id": "cr", + "displayName": "Client Registry" } ], "systems": { @@ -54,16 +63,19 @@ "internalid": { "uri": [ "http://health.go.ug/cr/internalid", - "http://clientregistry.org/openmrs" + "http://openmrs.org/openmrs2", + "http://clientregistry.org/openmrs", + "http://clientregistry.org/dhis2", + "http://clientregistry.org/lims" ], "displayName": "Internal ID" }, "nationalid": { - "uri": "http://health.go.ug/cr/natioanlid", + "uri": "http://clientregistry.org/cr/natioanlid", "displayName": "National ID" }, "artnumber": { - "uri": "http://health.go.ug/cr/artnumber", + "uri": "http://clientregistry.org/cr/artnumber", "displayName": "ART Number" }, "brokenMatch": { @@ -71,7 +83,7 @@ } }, "sync": { - "lastFHIR2ESSync": "2020-02-21T14:51:41" + "lastFHIR2ESSync": "1970-10-05T08:15:12" }, "__comments": { "matching.tool": "this tells if the app should use mediator algorithms or elasticsearch algorithms for matching, two options mediator and elasticsearch" diff --git a/server/config/config_test_template.json b/server/config/config_test_template.json index 8c73fcbd..de5e9ace 100644 --- a/server/config/config_test_template.json +++ b/server/config/config_test_template.json @@ -20,7 +20,7 @@ "fhirServer": { "username": "hapi", "password": "hapi", - "baseURL": "http://localhost:8081/clientregistry/fhir" + "baseURL": "http://localhost:8080/hapi/fhir" }, "elastic": { "server": "http://localhost:9200", @@ -30,7 +30,8 @@ "index": "patients" }, "matching": { - "tool": "elasticsearch" + "tool": "elasticsearch", + "autoMatchPatientWithHumanAdjudTag": false }, "codes": { "goldenRecord": "5c827da5-4858-4f3d-a50c-62ece001efea" @@ -82,7 +83,7 @@ } }, "sync": { - "lastFHIR2ESSync": "2020-09-22T18:02:56" + "lastFHIR2ESSync": "1970-10-05T08:15:12" }, "__comments": { "matching.tool": "this tells if the app should use mediator algorithms or elasticsearch algorithms for matching, two options mediator and elasticsearch" diff --git a/server/lib/esMatching.js b/server/lib/esMatching.js index 1aff7edb..b770270e 100644 --- a/server/lib/esMatching.js +++ b/server/lib/esMatching.js @@ -417,7 +417,6 @@ const performMatch = ({ goldenRecords.entry = goldenRecords.entry.filter((entry) => { return entry.resource.resourceType + '/' + entry.resource.id === goldenID; }); - logger.info('Done matching'); return callback({ error, diff --git a/server/lib/mixins/matchMixin.js b/server/lib/mixins/matchMixin.js index 3a171727..e23dcd83 100644 --- a/server/lib/mixins/matchMixin.js +++ b/server/lib/mixins/matchMixin.js @@ -10,6 +10,7 @@ const cacheFHIR = require('../tools/cacheFHIR'); const logger = require('../winston'); const config = require('../config'); const matchIssuesURI = URI(config.get("systems:CRBaseURI")).segment('matchIssues').toString(); +const humanAdjudURI = URI(config.get("systems:CRBaseURI")).segment('humanAdjudication').toString(); function createAddPatientAudEvent(operationSummary, req) { const auditBundle = {}; @@ -121,6 +122,7 @@ function createAddPatientAudEvent(operationSummary, req) { } const addPatient = (clientID, patientsBundle, callback) => { + let autoMatchPatientWithHumanAdjudTag = config.get("matching:autoMatchPatientWithHumanAdjudTag"); const responseBundle = { resourceType: 'Bundle', entry: [] @@ -176,28 +178,12 @@ const addPatient = (clientID, patientsBundle, callback) => { } }; - const getLinksFromResources = (resourceBundle) => { - const links = []; - for (const entry of resourceBundle.entry) { - if (entry.resource.link && entry.resource.link.length > 0) { - for (const link of entry.resource.link) { - const exist = links.find((pushedLink) => { - return pushedLink === link.other.reference; - }); - if (!exist) { - links.push(link.other.reference); - } - } - } - } - return links; - }; - const findMatches = ({ patient, currentLinks = [], newPatient = true, bundle, + hasHumanAdjudTag = false, operSummary }, callback) => { const patientEntry = {}; @@ -234,7 +220,21 @@ const addPatient = (clientID, patientsBundle, callback) => { operSummary.ESMatches = ESMatches; // if there is potential matches or conflict matches then add a tag - if(FHIRPotentialMatches.entry.length > 0) { + let existsPotentialMatches = false; + for(const potential of FHIRPotentialMatches.entry) { + let isCurrentLink = currentLinks.find((currentLink) => { + return potential.resource.link.find((link) => { + return link.other.reference === currentLink.resource.resourceType + "/" + currentLink.resource.id; + }); + }); + // if this potential match is currently linked to this patient and patient has human adjudication tag then this will be kept as a link, dont count as potential + if(isCurrentLink && hasHumanAdjudTag && !autoMatchPatientWithHumanAdjudTag) { + continue; + } else { + existsPotentialMatches = true; + } + } + if(existsPotentialMatches) { if(!patient.meta) { patient.meta = {}; } @@ -279,7 +279,21 @@ const addPatient = (clientID, patientsBundle, callback) => { resourceID: patient.id }); } - if(FHIRConflictsMatches.entry.length > 0) { + let existsConflictMatches = false; + for(const conflict of FHIRConflictsMatches.entry) { + let isCurrentLink = currentLinks.find((currentLink) => { + return conflict.resource.link.find((link) => { + return link.other.reference === currentLink.resource.resourceType + "/" + currentLink.resource.id; + }); + }); + // if this conflict match is currently linked to this patient and patient has human adjudication tag then this will be kept as a link, dont count as conflict + if(isCurrentLink && hasHumanAdjudTag && !autoMatchPatientWithHumanAdjudTag) { + continue; + } else { + existsConflictMatches = true; + } + } + if(existsConflictMatches) { if(!patient.meta) { patient.meta = {}; } @@ -392,7 +406,10 @@ const addPatient = (clientID, patientsBundle, callback) => { logger.error('An error occured while saving patient and golden record'); return callback(err); } - addLinks(patient, goldenRecord); + // if has human adjudication tag then already has golden record + if(!hasHumanAdjudTag || (hasHumanAdjudTag && autoMatchPatientWithHumanAdjudTag)) { + addLinks(patient, goldenRecord); + } const patientResource = { resource: patient, request: { @@ -420,74 +437,116 @@ const addPatient = (clientID, patientsBundle, callback) => { }); }); } else if (matchedGoldenRecords.entry && matchedGoldenRecords.entry.length > 0) { - if (currentLinks.length > 0) { - /** - * The purpose for this piece of code is to remove this patient from existing golden links - * if the existing golden link has just one link which is this patient, then link this golden link to new golden link of a patient - * otherwise just remove the patient from this exisitng golden link - * It also remove the exisitng golden link from the patient - */ + if(hasHumanAdjudTag && !autoMatchPatientWithHumanAdjudTag) { for (const currentLink of currentLinks) { - const exist = currentLink.resource.link && currentLink.resource.link.find((link) => { - return link.other.reference === 'Patient/' + patient.id; + operSummary.cruid.push(currentLink.resource.resourceType + '/' + currentLink.resource.id); + responseBundle.entry.push({ + response: { + location: currentLink.resource.resourceType + '/' + currentLink.resource.id + } }); - let replacedByNewGolden = false; - if (currentLink.resource.link && currentLink.resource.link.length === 1 && exist) { - const inNewMatches = matchedGoldenRecords.entry.find((entry) => { - return entry.resource.id === currentLink.resource.id; - }); - if (!inNewMatches) { - replacedByNewGolden = true; + } + for (const goldenRecord of matchedGoldenRecords.entry) { + let isSame = currentLinks.find((currLink) => { + return currLink.resource.id === goldenRecord.resource.id; + }); + // if matched golden record is different from the existing golden record then this is a conflict + if(!isSame) { + if(!patient.meta) { + patient.meta = {}; } - } - for (const index in currentLink.resource.link) { - if (currentLink.resource.link[index].other.reference === 'Patient/' + patient.id) { - // remove patient from golden link - if (replacedByNewGolden) { - currentLink.resource.link[index].other.reference = 'Patient/' + matchedGoldenRecords.entry[0].resource.id; - currentLink.resource.link[index].type = 'replaced-by'; - } else { - currentLink.resource.link.splice(index, 1); - } - // remove golden link from patient - for (const index in patient.link) { - if (patient.link[index].other.reference === 'Patient/' + currentLink.resource.id) { - patient.link.splice(index, 1); - } - } + if(!patient.meta.tag) { + patient.meta.tag = []; + } + let tagExist = patient.meta.tag.find((tag) => { + return tag.system === matchIssuesURI && tag.code === 'conflictMatches'; + }); + if(!tagExist) { + patient.meta.tag.push({ + system: matchIssuesURI, + code: 'conflictMatches', + display: 'Conflict On Match' + }); bundle.entry.push({ - resource: currentLink.resource, + resource: patient, request: { method: 'PUT', - url: `Patient/${currentLink.resource.id}`, + url: `Patient/${patient.id}`, }, }); } } } - } - // adding new links now to the patient - for (const goldenRecord of matchedGoldenRecords.entry) { - operSummary.cruid.push(goldenRecord.resource.resourceType + '/' + goldenRecord.resource.id); - responseBundle.entry.push({ - response: { - location: goldenRecord.resource.resourceType + '/' + goldenRecord.resource.id + } else { + if (currentLinks.length > 0) { + /** + * The purpose for this piece of code is to remove this patient from existing golden links + * if the existing golden link has just one link which is this patient, then link this golden link to new golden link of a patient + * otherwise just remove the patient from this exisitng golden link + * It also remove the exisitng golden link from the patient + */ + for (const currentLink of currentLinks) { + const exist = currentLink.resource.link && currentLink.resource.link.find((link) => { + return link.other.reference === 'Patient/' + patient.id; + }); + let replacedByNewGolden = false; + if (currentLink.resource.link && currentLink.resource.link.length === 1 && exist) { + const inNewMatches = matchedGoldenRecords.entry.find((entry) => { + return entry.resource.id === currentLink.resource.id; + }); + if (!inNewMatches) { + replacedByNewGolden = true; + } + } + for (const index in currentLink.resource.link) { + if (currentLink.resource.link[index].other.reference === 'Patient/' + patient.id) { + // remove patient from golden link + if (replacedByNewGolden) { + currentLink.resource.link[index].other.reference = 'Patient/' + matchedGoldenRecords.entry[0].resource.id; + currentLink.resource.link[index].type = 'replaced-by'; + } else { + currentLink.resource.link.splice(index, 1); + } + // remove golden link from patient + for (const index in patient.link) { + if (patient.link[index].other.reference === 'Patient/' + currentLink.resource.id) { + patient.link.splice(index, 1); + } + } + bundle.entry.push({ + resource: currentLink.resource, + request: { + method: 'PUT', + url: `Patient/${currentLink.resource.id}`, + }, + }); + } + } } - }); - addLinks(patient, goldenRecord.resource); - bundle.entry.push({ - resource: patient, - request: { - method: 'PUT', - url: `Patient/${patient.id}`, - }, - }, { - resource: goldenRecord.resource, - request: { - method: 'PUT', - url: `Patient/${goldenRecord.resource.id}`, - }, - }); + } + // adding new links now to the patient + for (const goldenRecord of matchedGoldenRecords.entry) { + operSummary.cruid.push(goldenRecord.resource.resourceType + '/' + goldenRecord.resource.id); + responseBundle.entry.push({ + response: { + location: goldenRecord.resource.resourceType + '/' + goldenRecord.resource.id + } + }); + addLinks(patient, goldenRecord.resource); + bundle.entry.push({ + resource: patient, + request: { + method: 'PUT', + url: `Patient/${patient.id}`, + }, + }, { + resource: goldenRecord.resource, + request: { + method: 'PUT', + url: `Patient/${goldenRecord.resource.id}`, + }, + }); + } } operSummary.FHIRMatches = FHIRAutoMatched.entry; operSummary.ESMatches = ESMatches; @@ -564,7 +623,9 @@ const addPatient = (clientID, patientsBundle, callback) => { if (existingPatients.length === 0) { operSummary.action = 'create'; delete newPatient.resource.link; - newPatient.resource.id = uuid4(); + if(!newPatient.resource.id) { + newPatient.resource.id = uuid4(); + } operSummary.submittedResource = newPatient.resource; findMatches({ patient: newPatient.resource, @@ -610,7 +671,10 @@ const addPatient = (clientID, patientsBundle, callback) => { operSummary.submittedResource = existingPatient.resource; operSummary.what = existingPatient.resource.resourceType + '/' + existingPatient.resource.id; logger.info(`Patient ${JSON.stringify(newPatient.resource.identifier)} exists, updating database records`); - let adjudTag; + + let adjudTag = existingPatient.resource.meta && existingPatient.resource.meta.tag && existingPatient.resource.meta.tag.find((tag) => { + return tag.system === humanAdjudURI && tag.code === 'humanAdjudication'; + }); async.series([ /** * overwrite with this new Patient to existing CR patient who has same identifier as the new patient @@ -622,7 +686,10 @@ const addPatient = (clientID, patientsBundle, callback) => { existingLinks = _.cloneDeep(goldenRecords); } delete newPatient.resource.link; - delete existingPatient.resource.link; + // if patient has human adjudication tag then dont delete existing link as no modification will be done even if there will be some new auto matches + if(!adjudTag || (adjudTag && autoMatchPatientWithHumanAdjudTag)) { + delete existingPatient.resource.link; + } existingPatient.resource = _.merge(existingPatient.resource, newPatient.resource); existingPatient.resource.id = id; bundle.entry.push({ @@ -635,9 +702,12 @@ const addPatient = (clientID, patientsBundle, callback) => { return callback(null); }, /** - * Drop links to every CR patient who is linked to this new patient + * Remove link of this submitted patient from the golden record */ callback => { + if(adjudTag) { + return callback(null); + } const link = `Patient/${existingPatient.resource.id}`; for (const goldenRecord of goldenRecords) { for (const index in goldenRecord.resource.link) { @@ -659,9 +729,9 @@ const addPatient = (clientID, patientsBundle, callback) => { findMatches({ patient: existingPatient.resource, currentLinks: existingLinks, - adjudicationTag: adjudTag, newPatient: false, bundle, + hasHumanAdjudTag: adjudTag, operSummary }, (err) => { if (err) { diff --git a/server/lib/routes/fhir.js b/server/lib/routes/fhir.js index 500eb24c..8cc50413 100644 --- a/server/lib/routes/fhir.js +++ b/server/lib/routes/fhir.js @@ -240,8 +240,20 @@ router.post('/', (req, res) => { }); router.post('/:resourceType', (req, res) => { + saveResource(req, res); +}); + +router.put('/:resourceType/:id', (req, res) => { + saveResource(req, res); +}); + +function saveResource(req, res) { let resource = req.body; let resourceType = req.params.resourceType; + let id = req.params.id; + if(id && !resource.id) { + resource.id = id; + } logger.info('Received a request to add resource type ' + resourceType); if(resourceType !== "Patient") { fhirWrapper.create(resource, (code, err, response, body) => { @@ -309,6 +321,6 @@ router.post('/:resourceType', (req, res) => { }); }); } -}); +} module.exports = router; \ No newline at end of file diff --git a/server/lib/routes/match.js b/server/lib/routes/match.js index 2aff0a2d..e91a0176 100644 --- a/server/lib/routes/match.js +++ b/server/lib/routes/match.js @@ -196,6 +196,7 @@ router.post('/resolve-match-issue', async(req, res) => { sourceResource: resolvePatientResource.resource, ignoreList: [resolvePatientResource.resource.id], }, ({ + FHIRAutoMatched, FHIRPotentialMatches, FHIRConflictsMatches }) => { @@ -215,6 +216,24 @@ router.post('/resolve-match-issue', async(req, res) => { } return needsResolving; }); + + // this checks those that has higher scores but are not linked to this patient + FHIRConflictsMatches.entry = FHIRAutoMatched.entry.filter((entry) => { + let needsResolving = true; + let link; + if(entry.resource.link) { + link = entry.resource.link[0].other.reference.split('/')[1]; + } + if(resolvePatient.uid === link) { + needsResolving = false; + } + //if a potential match comes from patient selected for resolving and user decided to remove the flag then dont add this to potential matches + if(entry.resource.id === resolvingFrom && removeFlag) { + needsResolving = false; + } + return needsResolving; + }); + // end of removing any resolved potential matches // end of removing resolved conflicts // remove any resolved potential matches @@ -234,6 +253,7 @@ router.post('/resolve-match-issue', async(req, res) => { return needsResolving; }); // end of removing any resolved potential matches + async.parallel({ potentialMatches: async (callback) => { if(FHIRPotentialMatches.entry.length === 0 || (flagType === 'potentialMatches' && removeFlag)) { diff --git a/tests/sampleSinglePatient.json b/tests/sampleSinglePatient.json index f5f7687d..fcb979cf 100644 --- a/tests/sampleSinglePatient.json +++ b/tests/sampleSinglePatient.json @@ -1,32 +1,32 @@ { "resourceType": "Patient", "identifier": [{ - "system": "http://clientregistry.org/openmrs", - "value": "patient3" - }, { - "system": "http://health.go.ug/cr/nationalid", - "value": "228374844" - }, { - "system": "http://system1.org", - "value": "12347", - "period": { - "start": "2001-05-07" - }, - "assigner": { - "display": "test Org" - } + "system": "http://clientregistry.org/openmrs", + "value": "patient3" + }, { + "system": "http://health.go.ug/cr/nationalid", + "value": "228374844" + }, { + "system": "http://system1.org", + "value": "12347", + "period": { + "start": "2001-05-07" + }, + "assigner": { + "display": "test Org" + } }], "active": true, "name": [{ "use": "official", - "family": "", + "family": "Joshua", "given": [ - "Emmmanuel" + "Emanuel" ] }], "telecom": [{ "system": "phone", - "value": "0678 56160" + "value": "0678 5616088" }], "gender": "male", "birthDate": "1972-01-08" diff --git a/ui/src/views/Resolve.vue b/ui/src/views/Resolve.vue index 5f6deed8..c055cba1 100644 --- a/ui/src/views/Resolve.vue +++ b/ui/src/views/Resolve.vue @@ -24,7 +24,7 @@

Options

- +