Skip to content

Commit

Permalink
Updated the standard ad hoc methods. It was also created a feature to…
Browse files Browse the repository at this point in the history 
… define the date format in the settings file.
  • Loading branch information
@joaorafaelalmeida
joaorafaelalmeida committed Oct 11, 2019
1 parent f3d14dd commit 7e54091
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 26 deletions.
3 changes: 2 additions & 1 deletion material/Berlin/6_Scripts/BerlinMigrator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ def addMissingRows(self):
missingRows += self.__processCeradWLRounds()
missingRows += self.__processCeralWLRecognition()
missingRows += self.__processDiagnosisAndEtiology()
missingRows += self.__processApoE()
#missingRows += self.__processApoE()
#missingRows += self.__process...
#....
return missingRows
Expand Down Expand Up @@ -330,6 +330,7 @@ def __processDiagnosisAndEtiology(self):
return results

def __processApoE(self):
print("Check and remove this shit, because now we have a standard ad hoc method for this")
results = []
if len(self.apoE) > 0:
for row in self.apoE:
Expand Down
3 changes: 3 additions & 0 deletions material/Berlin/6_Scripts/settings.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ patientcsv=../5_Content_Harmonized/TH_20190510 EMIF Patient Data.csv
obsdir=../5_Content_Harmonized/
results=../7_Results/
vocabulariesdir=../../../UsagiConceptMapping/Vocabularies/
log=execution.log
cohortname=Berlin
formatdate=%%d-%%M-%%Y


[cohort_mappings]
Expand Down
6 changes: 5 additions & 1 deletion src/Migrator/CutOffCalculator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@
"2000000073":{"cutOff" :"2000000463", "cutOffName" :"Phosphorylated Tau Cut-off",
"abnormal":"2000000074", "abnormalName" :"Phosphorylated Tau Abnormal"},
"2000000075":{"cutOff" :"2000000298", "cutOffName" :"Total Tau Cut-off",
"abnormal":"2000000076", "abnormalName" :"Total Tau Abnormal"}
"abnormal":"2000000076", "abnormalName" :"Total Tau Abnormal"},
"2000000168":{"cutOff" :"2000000310", "cutOffName" :"MTA Bilateral Abnormal Cut-off",
"abnormal":"2000000169", "abnormalName" :"MTA Bilateral Abnormal"},
"2000000121":{"cutOff" :"", "cutOffName" :"",
"abnormal":"2000000122", "abnormalName" :"HDS Abnormal"}
}

class CutOffCalculator():
Expand Down
1 change: 1 addition & 0 deletions src/Migrator/MigratorArgs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def __init__(self, args):
self.results = super().argAsDir(self.__defineArg(args, "results"))
self.log = self.__defineArg(args, "log")
self.cohortname = self.__defineArg(args, "cohortname")
self.formatdate = self.__defineArg(args, "formatdate")
sep = self.__defineArg(args, "cohortsep")
self.cohortsep = '\t' if sep == "\\t" else sep
sep = self.__defineArg(args, "usagisep")
Expand Down
151 changes: 127 additions & 24 deletions src/Migrator/StandardAdHoc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from ZcoreCalculator import ZcoreCalculator
from CutOffCalculator import CutOffCalculator
from SAHGlobalVariables import SAHGlobalVariables
from MigratorArgs import MigratorArgs
from Logger import *
from datetime import date
import datetime
Expand All @@ -12,6 +13,7 @@
class StandardAdHoc(object, metaclass=Singleton):
def __init__(self, cutOffs):
self.logger = Logger()
self.formatDate = MigratorArgs().formatdate
self.patientIDLabel = None
self.zcoreCalculator = ZcoreCalculator()
self.cutOffsCalculator = None
Expand All @@ -25,8 +27,9 @@ def __init__(self, cutOffs):
#Temporary variables calculated based on other variables
self.ageMeasurement = []
self.bodyMass = []
self.cardiovascularDisordersYes = []
self.comorbidities = {}
self.comorbidityYes = []
self.apoE = []

def definePatientIDLabel(self, patientIDLabel):
self.patientIDLabel = patientIDLabel
Expand Down Expand Up @@ -77,14 +80,48 @@ def __loadingStage(self, row, patientID, variableConcept):
self.__loadBodyLength(row, patientID)
if "2000000462" in variableConcept:
self.__loadWeight(row, patientID)
if "2000000013" in variableConcept:
self.__loadExtraApoE(row, patientID)

#Cardiovascular Disorders (Yes)
listOfCardiovascularDisorders = ["2000000326", "2000000341", "2000000357", "2000000360",
"2000000367", "2000000390", "2000000400"]
for variable in listOfCardiovascularDisorders:
if variable in variableConcept:
return self.__addCardiovascularDisorders(row, patientID)

#Comorbidities to calculate yes
allComorbidities = {
"2000000637":{
"name":"Cardiovascular Disorders",
"concepts":["2000000326", "2000000341", "2000000357", "2000000360", "2000000367", "2000000390", "2000000400"]
},
"2000000638":{
"name":"Cardiovascular Risk Factors",
"concepts":["2000000381", "2000000382", "2000000383", "2000000396", "2000000402"]
},
"2000000639":{
"name":"Cerebrovascular Disorders",
"concepts":["2000000337", "2000000403", "2000000438", "2000000441"]
},
"2000000640":{
"name":"Endocrine Disorders",
"concepts":["2000000363", "2000000384", "2000000385", "2000000405"]
},
"2000000641":{
"name":"Neurological Disorders",
"concepts":["2000000378", "2000000408", "2000000416", "2000000434"]
},
"2000000473":{
"name":"Other Cardiac Diseases",
"concepts":["2000000334", "2000000379", "2000000415"]
},
"2000000642":{
"name":"Psychiatric Disorders",
"concepts":["2000000331", "2000000469", "2000000470", "2000000410"]
},
"2000000643":{
"name":"Somatic Disorders",
"concepts":["2000000343", "2000000432", "2000000433", "2000000412"]
}
}
for comorbidity in allComorbidities:
for variable in allComorbidities[comorbidity]["concepts"]:
if variable in variableConcept:
return self.__addComorbiditiesSubClass(row, patientID, allComorbidities[comorbidity]["name"], comorbidity)

def __loadDateOfDiagnosis(self, row, patientID):
if row['Measure'] != "":
Expand Down Expand Up @@ -118,16 +155,55 @@ def __loadWeight(self, row, patientID):
if len(self.bodyLength) > 0:
self.__calculateBodyMassIndex(row, patientID)

def __addCardiovascularDisorders(self, row, patientID):
def __loadExtraApoE(self, row, patientID):
measures = row['MeasureString'].split("/")
if len(measures) == 2:
self.apoE += [
self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
'Variable': row['Variable'],
'Measure': row['Measure'],
'VariableConcept': '2000000320', #ApoE Allele 1
'MeasureConcept': None,
'MeasureNumber': None,
'MeasureString': measures[0]
}),
self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
'Variable': row['Variable'],
'Measure': row['Measure'],
'VariableConcept': '2000000321', #ApoE Allele 2
'MeasureConcept': None,
'MeasureNumber': None,
'MeasureString': measures[1]
}),
self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
'Variable': row['Variable'],
'Measure': row['Measure'],
'VariableConcept': '2000000014', #ApoE4 Carrier
'MeasureConcept': YES if measures[0] == "4" or measures[1] == "4" else NO,
'MeasureNumber': None,
'MeasureString': None
})]
else:
self.logger.warn(warnType = WRONG_VALUE,
patientID = patientID,
variable = row['Variable'],
measure = row['Measure'],
msg = "This method was not able to split the measure by /")

def __addComorbiditiesSubClass(self, row, patientID, variable, conceptID):
if row["MeasureConcept"] == YES:
if len(list(filter(lambda line: line[self.patientIDLabel] == patientID, self.cardiovascularDisordersYes))) == 0:
self.cardiovascularDisordersYes += [self.__mergeDictionaries(row, {
if variable not in self.comorbidities:
self.comorbidities[variable] = []
if len(list(filter(lambda line: line[self.patientIDLabel] == patientID, self.comorbidities[variable]))) == 0:
self.comorbidities[variable] += [self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
#add observation date to do
'Variable': 'Ontology rule (Cardiovascular Disorders - Yes)',
'Variable': "Ontology rule " + variable,
'Measure': "",
'MeasureNumber': None,
'VariableConcept': "2000000637",
'VariableConcept': conceptID,
'MeasureConcept': YES
})]
self.__addComorbidity(row, patientID)
Expand All @@ -136,23 +212,21 @@ def __addComorbidity(self, row, patientID):
if len(list(filter(lambda line: line[self.patientIDLabel] == patientID, self.comorbidityYes))) == 0:
self.comorbidityYes += [self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
#add observation date to do
'Variable': 'Ontology rule (Comorbidity - Yes)',
'Measure': "",
'MeasureNumber': None,
'VariableConcept': "2000000526",
'VariableConcept': "2000000525",
'MeasureConcept': YES
})]

def __calculateAge(self, row, patientID):
try:
delta = self.__compareDates(SAHGlobalVariables.dateOfDiagnosis[patientID], SAHGlobalVariables.birthdayDate[patientID], '%d-%M-%Y')
delta = self.__compareDates(SAHGlobalVariables.dateOfDiagnosis[patientID], SAHGlobalVariables.birthdayDate[patientID])
if delta:
age = int(delta.days/365)
SAHGlobalVariables.age[patientID] = age
self.ageMeasurement += [self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
#add observation date to do
'Age':age,
'Variable': 'Calculated age',
'Measure': "",
Expand All @@ -161,11 +235,26 @@ def __calculateAge(self, row, patientID):
'MeasureConcept': None
})]
except:
pass
var = "Calculated age"
msg = "Age not calculated due to missing variable."
if patientID not in SAHGlobalVariables.dateOfDiagnosis:
var = "Date of Diagnosis"
elif patientID not in SAHGlobalVariables.birthdayDate:
var = "Birthday Date"
else:
msg = "Age calculation fail maybe due to the date format."
self.logger.warn(warnType = MISSING_VALUE,
patientID = patientID,
variable = var,
msg = msg)

def __calculateBodyMassIndex(self, row, patientID):
try:
bmi = self.weight[patientID]/((self.bodyLength[patientID]/100)*(self.bodyLength[patientID]/100))
if bmi > 30:
value = YES
else:
value = NO
self.bodyMass += [self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
'Body Mass Index':bmi,
Expand All @@ -174,7 +263,15 @@ def __calculateBodyMassIndex(self, row, patientID):
'MeasureNumber': bmi,
'VariableConcept': '2000000339',
'MeasureConcept': None
}), self.__mergeDictionaries(row, {
self.patientIDLabel:patientID,
'Variable': 'Calculated obesity',
'Measure': "",
'MeasureNumber': None,
'VariableConcept': '2000000396',
'MeasureConcept': value
})]

except Exception as ex:
var = "Body Mass Index"
if patientID not in self.weight:
Expand Down Expand Up @@ -204,6 +301,10 @@ def __validateMeasureContent(self, row, patientID, variableConcept):

listOfRanges = {
"2000000173":{"min":0, "max":144},
"2000000170":{"min":0, "max":4},
"2000000171":{"min":0, "max":4},
"2000000168":{"min":0, "max":8},
"2000000121":{"min":0, "max":52},
}
for variable in listOfRanges:
if variable in variableConcept:
Expand Down Expand Up @@ -245,7 +346,7 @@ def __processRow(self, row, patientID, variableConcept):

def __dealWithDatesDifferencesInDays(self, row, patientID):
try:
delta = self.__compareDates(SAHGlobalVariables.dateOfDiagnosis[patientID], row["Measure"], '%d-%M-%Y')
delta = self.__compareDates(SAHGlobalVariables.dateOfDiagnosis[patientID], row["Measure"])
if delta:
if round(delta.days/365, 5) > -15 and round(delta.days/365, 5) < 15:
row["MeasureNumber"] = round(delta.days/365, 5)
Expand All @@ -267,10 +368,10 @@ def __dealWithDatesDifferencesInDays(self, row, patientID):
msg = "Missing date to calculate the difference of dates")
return []

def __compareDates(self, initalDate, finalDate, formatDate):
def __compareDates(self, initalDate, finalDate):
try:
d0 = datetime.datetime.strptime(initalDate, formatDate)
d1 = datetime.datetime.strptime(finalDate, formatDate)
d0 = datetime.datetime.strptime(initalDate, self.formatDate)
d1 = datetime.datetime.strptime(finalDate, self.formatDate)
return (d0 - d1)
except:
return None
Expand All @@ -291,8 +392,10 @@ def __addNewMeasurements(self):
newMeasurements = []
newMeasurements += self.__addMeasurement(self.ageMeasurement)
newMeasurements += self.__addMeasurement(self.bodyMass)
newMeasurements += self.__addMeasurement(self.cardiovascularDisordersYes)
for comorbidity in self.comorbidities:
newMeasurements += self.__addMeasurement(self.comorbidities[comorbidity])
newMeasurements += self.__addMeasurement(self.comorbidityYes)
newMeasurements += self.__addMeasurement(self.apoE)
#newMeasurements += self.__addMeasurement...
#....
return newMeasurements
Expand Down

0 comments on commit 7e54091

Please sign in to comment.