IJCAI2022 Experiments BCBR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SETUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests import get\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import datetime\n",
    "from IPython.display import HTML\n",
    "import pyodbc\n",
    "import time\n",
    "\n",
    "\n",
    "from pandas.io.json import json_normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mycbr_py_api import MyCBRRestApi as mycbr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "server = 'localhost'\n",
    "\n",
    "port = '8080'\n",
    "base_url = 'http://' + server + ':' + port + '/'\n",
    "headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0' }\n",
    "\n",
    "obj = mycbr(base_url)\n",
    "res=obj.getAllCases()\n",
    "\n",
    "def retrieve_c_att22(case,k):\n",
    "    raw = pd.DataFrame(requests.post(url=base_url + 'concepts/KP/casebases/CB_csvImport/amalgamationFunctions/default function/retrievalByMultipleAttributes?k='+str(k), json={'IndustryAreaCode':case['IndustryAreaCode'],'IndustryGroupCode':case['IndustryGroupCode'],'IndustryCode':case['IndustryCode'],'MunicipalityNumber':case['MunicipalityNumber'],'IndustrySubgroupCode':case['IndustrySubgroupCode'],'IndustryMainAreaCode':case['IndustryMainAreaCode'], 'ProbabilityForNonComplianceGivenControlPointandMunicipality':100,\n",
    "    'ProbabilityForNonComplianceGivenControlPointandIndustrySubgroup':100,'NumberOfNonComplianceGivenControlPointandMunicipality':70,'NumberOfNonComplianceGivenControlPointandIndustrySubgroup':70, 'Fylke':case['Fylke']},headers=headers).json())\n",
    "    #print(case)\n",
    "    results = raw.apply(pd.to_numeric, errors='coerce').fillna(raw).sort_values(by='similarity', ascending=False)\n",
    "    results['similarity'] = results['similarity'].astype(float)\n",
    "    results['NonCompliance'] = results['NonCompliance'].astype(int)\n",
    "    returned_df = results.apply(pd.to_numeric, errors='coerce').fillna(results)\n",
    "    #results=results.head(k)\n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "initCBCopy=obj.getAllCases();\n",
    "currentCBCopy=obj.getAllCases();\n",
    "#Add multiple contorl points to the case base\n",
    "def addManyCases(cases):\n",
    "    cases=cases.to_dict('records')\n",
    "    rec={'cases':cases}\n",
    "    #print(rec)\n",
    "    #structure of case dictionary must be {'cases':[{'IndustrySubgroupCode':45.2},{\"Municipality\":\"Oslo\"}]}\n",
    "    #NB! int values must be formatted as strings\n",
    "    results=requests.post(url=base_url + 'concepts/KP/casebases/CB_csvImport/cases', json=rec,headers={'Accept': 'application/json'})\n",
    "    return results\n",
    "\n",
    "def fillCaseBase(splitindex):\n",
    "    # Specifying the ODBC driver, server name, database, etc. directly\n",
    "    cnxn = pyodbc.connect('DRIVER={SQL Server Native Client 11.0};SERVER=localhost;DATABASE=IJCAI2022;Trusted_Connection=yes;')\n",
    "    cursor = cnxn.cursor()\n",
    "\n",
    "    traindateid=\"\"\n",
    "    testdateid=\"\"\n",
    "    cases=pd.read_sql(\"SELECT  [MuncipalityIndex] as MunicipalityIndex ,[IndustrySubgroupCodeIndex],[IndustryGroupCode],[IndustryAreaCode],[IndustryCode],[Fylke],[MunicipalityNumber],  [InspectionId] as 'InspectionId' ,[IndustryMainAreaCode],ControlPointText,[IndustrySubgroupCode],[Municipality],[NonCompliance]\"\n",
    "+\",round(isNull(SannsynlighetForBruddGittKPogKommune,(1.0/(6.0+isNull(tttg2,0))))*100,0) as ProbabilityForNonComplianceGivenControlPointandMunicipality\"\n",
    "+\",round(isNull(SannsynlighetForBruddGittKPogNaring,(1.0/(6.0+isNull(tttg,0))))*100,0) as ProbabilityForNonComplianceGivenControlPointandIndustrySubgroup\"\n",
    "+\",round(isNull(AntBruddGittKPogNaring,0),0) as NumberOfNonComplianceGivenControlPointandIndustrySubgroup\"\n",
    "+\",round(isNull(AntBruddGittKPogKommune,0),0) as NumberOfNonComplianceGivenControlPointandMunicipality\"\n",
    "                      +\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTest\"+str(splitindex)+\"]\"\n",
    "+\" Left join(Select ( (Convert(real,isNull(Count(IndustrySubgroupCode),0))+1.0)/(AVG(Convert(real, Kommune3))+6.0))  as SannsynlighetForBruddGittKPogNaring,Count(IndustrySubgroupCode) as AntBruddGittKPogNaring,ControlPointText as KPT2, IndustrySubgroupCode as Nugk\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \"  \n",
    "+\" join (Select Count(IndustrySubgroupCode) as Kommune3 , ControlPointText as KT2, IndustrySubgroupCode as Kom\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \"+traindateid+\"  group by ControlPointText, IndustrySubgroupCode) as a on a.Kom=IndustrySubgroupCode and ControlPointText=KT2\" \n",
    "+\" join (Select Count(Municipality) as Kommune5 ,  max(NonCompliance) as AntB2\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \" +traindateid+\") as c on c.AntB2=NonCompliance\"\n",
    "                    +traindateid\n",
    "+\" group by ControlPointText, NonCompliance, IndustrySubgroupCode) as b on b.KPT2=ControlPointText and b.Nugk=IndustrySubgroupCode\"\n",
    "+\" Left join(Select ( (Convert(real,isNull(Count(Municipality),0))+1.0)/(AVG(Convert(real, Kommune3))+6.0))  as SannsynlighetForBruddGittKPogKommune,Count(Municipality) as AntBruddGittKPogKommune, ControlPointText as KPT, Municipality as KOMM\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\"  join (Select Count(Municipality) as Kommune3 , ControlPointText as KT2, Municipality as Kom\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \"+traindateid+\" group by ControlPointText, Municipality) as a on a.Kom=Municipality and ControlPointText=KT2\" \n",
    "+\" join (Select Count(Municipality) as Kommune5 ,  max(NonCompliance) as AntB2\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \" +traindateid+\") as c on c.AntB2=NonCompliance\"\n",
    "                    +traindateid+\"  group by ControlPointText, NonCompliance, Municipality) as a on a.KOMM=Municipality and a.KPT=ControlPointText\"\n",
    "+\" left join(Select Count(IndustrySubgroupCode) as tttg , ControlPointText as KT10, IndustrySubgroupCode as Kom10\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \"+traindateid+\" group by ControlPointText, IndustrySubgroupCode) as z on z.Kom10=IndustrySubgroupCode and ControlPointText=z.KT10\" \n",
    "+\" left join(Select Count(Municipality) as tttg2 , ControlPointText as KT10, Municipality as NUG10\"\n",
    "+\" FROM [IJCAI2022].[dbo].[BayesianDynamicChecklistLocalDbRandomizedTrain\"+str(splitindex)+\"]\"+\" \"+traindateid+\" group by ControlPointText, Municipality) as zz on ControlPointText=zz.KT10 and zz.NUG10=Municipality\" \n",
    "+\" where InspectionDateId<20190601\"\n",
    "                      +\" Order by InspectionDateId \" + testdateid,cnxn)\n",
    "    #print(cases)\n",
    "    cases['NonCompliance']= cases['NonCompliance'].map(str)\n",
    "    cases['NumberOfNonComplianceGivenControlPointandIndustrySubgroup']= cases['NumberOfNonComplianceGivenControlPointandIndustrySubgroup'].map(str)\n",
    "    cases['NumberOfNonComplianceGivenControlPointandMunicipality']= cases['NumberOfNonComplianceGivenControlPointandMunicipality'].map(str)\n",
    "    cases['ProbabilityForNonComplianceGivenControlPointandIndustrySubgroup']= cases['ProbabilityForNonComplianceGivenControlPointandIndustrySubgroup'].astype(int)\n",
    "    cases['ProbabilityForNonComplianceGivenControlPointandMunicipality']= cases['ProbabilityForNonComplianceGivenControlPointandMunicipality'].astype(int)\n",
    "    cases['ProbabilityForNonComplianceGivenControlPointandIndustrySubgroup']= cases['ProbabilityForNonComplianceGivenControlPointandIndustrySubgroup'].map(str)\n",
    "    cases['ProbabilityForNonComplianceGivenControlPointandMunicipality']= cases['ProbabilityForNonComplianceGivenControlPointandMunicipality'].map(str)\n",
    "    cases['IndustrySubgroupCodeIndex']= cases['IndustrySubgroupCodeIndex'].map(str)\n",
    "    cases['MunicipalityIndex']= cases['MunicipalityIndex'].map(str)\n",
    "    cases['InspectionId']=cases['InspectionId'].apply(int)+0\n",
    "    cases['InspectionId']= cases['InspectionId'].map(str)\n",
    "    cases['MunicipalityNumber']= cases['MunicipalityNumber'].map(str)\n",
    "    cases['IndustryAreaCode']= cases['IndustryAreaCode'].map(str)\n",
    "    cases['IndustryCode']= cases['IndustryCode'].map(str)\n",
    "    cases['IndustryCode']=cases['IndustryCode'].replace(to_replace=r',', value='.', regex=True)\n",
    "    cases=cases.dropna()\n",
    "\n",
    "    return addManyCases(cases)\n",
    "#Delete all items (control points) from the case base\n",
    "def deleteAllCases():\n",
    "    final_urli = base_url + '/concepts/KP/casebases/CB_csvImport/cases/'\n",
    "    response = requests.delete( url= final_urli)\n",
    "    return response\n",
    "\n",
    "\n",
    "\n",
    "#Restores the casebase back to it's original form\n",
    "def restoreOriginalCaseBase(splitindex):\n",
    "    deleteAllCases()\n",
    "    fillCaseBase(splitindex)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXPERIMENT, STATISTICS FOR GENERATED CHECKLISTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: Use the actual ground truth labels also, not only calculate matches with the validation base.\n",
    "#TODO: Need to make precision (gt), recall (gt)... precision, recall by average pr item instead of counting. Number of observations in validation base varies and we get bias.\n",
    "#Prec(gt)=average precision (precision calculated by counting number of retrieved items that both exists and are positive in the original checklist)\n",
    "#divided by number of retrieved items that also exists in the original checklist\n",
    "\n",
    "\n",
    "timetotal=0\n",
    "traintime=0\n",
    "precctot=0\n",
    "accctot=0\n",
    "preccgttot=0\n",
    "recctot=0\n",
    "traintimetot=0\n",
    "fh2=open(\"Log_\" + \"CBRNBIChecklistsTest\" + '.txt', 'w+')\n",
    "\n",
    "for ik in range(0,8):\n",
    "    start_time = time.time()\n",
    "\n",
    "    if len(initCBCopy)==0:\n",
    "        fillCaseBase(ik)\n",
    "        initCBCopy=obj.getAllCases()\n",
    "        currentCBCopy=initCBCopy\n",
    "    else:\n",
    "        restoreOriginalCaseBase(ik)\n",
    "        initCBCopy=obj.getAllCases()\n",
    "        currentCBCopy=initCBCopy\n",
    "    traintime=traintime+(time.time()-start_time)\n",
    "    traintimetot=traintimetot+(time.time()-start_time)\n",
    "    print(\"Current avg. training time:\"+ str(traintime/(ik+1)))\n",
    "    noncompliance=0 #true positive(gt)\n",
    "    controlpoints=0 #predicted positive(gt)\n",
    "    noncompliancengt=0 #true positive (ngt)\n",
    "    controlpointsngt=0#predicted positive (ngt)\n",
    "    falsepositivegt=0\n",
    "    precisionval=0\n",
    "    recallval=0\n",
    "    accuracyval=0\n",
    "    precision=0\n",
    "    precisiongt=0\n",
    "    lengthprecgt=0\n",
    "    validationpercentage=0.000001\n",
    "    recall=0\n",
    "    accuracy=0\n",
    "    lengthprec=0.00001\n",
    "    lengthvalprec=0\n",
    "    lengthrec=0.00001\n",
    "    lengthvalrec=0\n",
    "    lengthacc=0.00001\n",
    "    lengthvalacc=0\n",
    "    truenegative=0\n",
    "    falsenegative=0\n",
    "    similarity=0\n",
    "    counter=0\n",
    "    Kcp=15\n",
    "    pd.set_option('display.max_rows', None)\n",
    "    pd.set_option('display.max_columns', None)\n",
    "    pd.set_option('display.width', None)\n",
    "    pd.set_option('display.max_colwidth', -1)\n",
    "    noncompliantInspection=0\n",
    "    numberofOverlappingControlpoints=0\n",
    "    inspections=obj.getAllCases().drop_duplicates(subset = [\"InspectionId\"])\n",
    "    cases=obj.getAllCases()\n",
    "    cases[\"NonCompliance\"]=cases[\"NonCompliance\"].astype(int)\n",
    "    casestest=len(cases.drop_duplicates(subset = [\"ControlPointText\"]))\n",
    "    casesWithoutgt=list()\n",
    "    negatives=list()\n",
    "    cltotallen=0\n",
    "    inspectionstot=0\n",
    "    precchecklists=0\n",
    "    recchecklists=0\n",
    "    accchecklists=0\n",
    "    precchecklistsval=0\n",
    "    recchecklistsval=0\n",
    "    accchecklistsval=0\n",
    "\n",
    "    for ind, case in inspections.iterrows():\n",
    "\n",
    "        inspectionstot+=1\n",
    "        uniquechecklistlengthval=0\n",
    "        casesWithoutgt=list()\n",
    "            \n",
    "        r=0#Stop variable to ensure at least 10 cases are selected for each inspection (case)\n",
    "        k=500#Finding the Top K cases\n",
    "        existingChecklist=cases[cases['InspectionId']==case['InspectionId']]#Retrieve the existing checklist\n",
    "        clck=cases[cases[\"IndustrySubgroupCode\"].astype(float)==float(case[\"IndustrySubgroupCode\"])]\n",
    "        clck=clck[clck[\"MunicipalityNumber\"].astype(int)==int(case[\"MunicipalityNumber\"])]\n",
    "        clck[\"NonCompliance\"]=clck[\"NonCompliance\"].astype(float)\n",
    "        negatives=clck\n",
    "        if len(clck)>0:\n",
    "            caseAvgNoncompliancePrControlpoint=clck.groupby([\"ControlPointText\"],as_index=False).mean()\n",
    "\n",
    "            #print(caseAvgNoncompliancePrControlpoint)\n",
    "        while r==0:\n",
    "            checklist=retrieve_c_att22(case,k)#Retrieve items that matches each inspection (input case). checklist=CL from paper\n",
    "\n",
    "            #caseAvgNoncompliancePrControlpoint=checklist.groupby([\"ControlPointText\"],as_index=False).mean()#FIX THIS\n",
    "            #Find the mean value of each column in the returned data set of items. \n",
    "            #The mean values are grouped by each unique item, such that each solution s(non-compliance) is reassigned\n",
    "            uniqueChecklist=checklist.drop_duplicates(subset = [\"ControlPointText\"])#Find all unique items by removing duplicates\n",
    "            uniqueChecklist=uniqueChecklist.head(Kcp)\n",
    "            if len(uniqueChecklist)>(Kcp-1):#Check if there are at least 10 unique items\n",
    "                truepositive=0\n",
    "                truepositiveval=0\n",
    "                r=1#Stops the while loop since 10 unique items are found\n",
    "\n",
    "                #Record statistics. Add to the total, the number of unique items that were found in current generated checklist. \n",
    "                similarity+=uniqueChecklist[\"similarity\"].sum() \n",
    "                \n",
    "                #Record statistics. Add the ratio/fraction of non-compliance for each of the (unique) item in the checklist.\n",
    "                precpercl=0\n",
    "                lengthorgcl=0\n",
    "                for ind2, generatedChecklistControlpoint in uniqueChecklist.iterrows():#Find overlap between the existing and new generated checklist\n",
    "                    cltotallen+=1\n",
    "                    negatives=negatives[negatives[\"ControlPointText\"]!=generatedChecklistControlpoint[\"ControlPointText\"]]#This is wrong. Negatives need to be outside the whole checklist\n",
    "                    excp=existingChecklist[existingChecklist[\"ControlPointText\"]==generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                    if len(excp)>0:\n",
    "                        numberofOverlappingControlpoints+=1 \n",
    "                       # print(excp[\"ControlPointText\"])\n",
    "                        precpercl+=((excp[\"NonCompliance\"].sum())/len(excp[\"NonCompliance\"]))\n",
    "                        lengthorgcl+=1\n",
    "                        controlpoints+=1\n",
    "                        \n",
    "                    \n",
    "                    caseAvgNoncompliancePrControlpoint2=caseAvgNoncompliancePrControlpoint[caseAvgNoncompliancePrControlpoint[\"ControlPointText\"]==generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                    \n",
    "                    sumgt=caseAvgNoncompliancePrControlpoint2[\"NonCompliance\"].sum() \n",
    "                    if len(caseAvgNoncompliancePrControlpoint2[\"NonCompliance\"])>0:\n",
    "                        noncompliance+=(sumgt/len(caseAvgNoncompliancePrControlpoint2[\"NonCompliance\"]))\n",
    "                        truepositiveval+=(sumgt/len(caseAvgNoncompliancePrControlpoint2[\"NonCompliance\"]))\n",
    "                    \n",
    "                    matchlen=len(caseAvgNoncompliancePrControlpoint2[\"NonCompliance\"])\n",
    "                    if matchlen>1:\n",
    "                        print(caseAvgNoncompliancePrControlpoint2[\"NonCompliance\"])\n",
    "                    controlpointsngt+=1\n",
    "                    falsepositivegt+=(matchlen-sumgt)\n",
    "                    if matchlen>0:\n",
    "                        lengthvalprec+=1\n",
    "                        uniquechecklistlengthval+=1\n",
    "                    sumNoGt=0\n",
    "                    if matchlen==0:#if validation record of checklist item does not exist\n",
    "                        casesWithoutgt=cases[cases[\"IndustrySubgroupCode\"].astype(float)==float(generatedChecklistControlpoint[\"IndustrySubgroupCode\"])]\n",
    "                        casesWithoutgt=casesWithoutgt[casesWithoutgt[\"MunicipalityNumber\"].astype(int)==int(generatedChecklistControlpoint[\"MunicipalityNumber\"])]\n",
    "                        casesWithoutgt=casesWithoutgt[casesWithoutgt[\"ControlPointText\"]==generatedChecklistControlpoint[\"ControlPointText\"]]\n",
    "                        casesWithoutgt[\"NonCompliance\"]=casesWithoutgt[\"NonCompliance\"].astype(float)\n",
    "                        sumNoGt=casesWithoutgt[\"NonCompliance\"].sum()\n",
    "                        \n",
    "                        \n",
    "\n",
    "                        if len(casesWithoutgt[\"NonCompliance\"])>0:\n",
    "                            noncompliancengt+=(sumNoGt/len(casesWithoutgt[\"NonCompliance\"]))\n",
    "                            truepositive+=(sumNoGt/len(casesWithoutgt[\"NonCompliance\"]))\n",
    "\n",
    "                if lengthorgcl>0:\n",
    "                    precisiongt+=(precpercl/lengthorgcl)#Average precision per item with available ground truth.\n",
    "                    lengthprecgt+=1\n",
    "                #Calculate the number of non-compliance among items outside the checklist (false negatives)\n",
    "                uniquenegatives=len(negatives.drop_duplicates(subset = [\"ControlPointText\"]))\n",
    "                negativecpy=negatives.copy()\n",
    "                negativecpy[\"NonCompliance\"]=negativecpy[\"NonCompliance\"].astype(float)\n",
    "                groupedby=negativecpy.groupby([\"ControlPointText\"],as_index=False).count()\n",
    "                groupedbys=negativecpy.groupby([\"ControlPointText\"],as_index=False).sum()\n",
    "                groupedby[\"NonCompliance\"]=groupedbys[\"NonCompliance\"]/groupedby[\"NonCompliance\"]\n",
    "                uniquecalcnegatives=groupedby.drop_duplicates(subset = [\"ControlPointText\"])\n",
    "                noncompliancenegatives=uniquecalcnegatives[\"NonCompliance\"].sum()\n",
    "\n",
    "                falsenegativeelement=0\n",
    "                if uniquenegatives>0: #sanity check\n",
    "                    falsenegativeelement=noncompliancenegatives/uniquenegatives #number of unique non-compliant items in the current checklist divided by the number of unique items in the current checklist being evaluated\n",
    "                truenegativeelement=1-falsenegativeelement\n",
    "                \n",
    "                truepositivesprchecklistval=(truepositiveval)\n",
    "                falsepositivesprchecklistval=(uniquechecklistlengthval-truepositivesprchecklistval)\n",
    "                #Final calculations for finding the numbers of true/false positives and negatives for the current checklist.\n",
    "                truepositivesprchecklist=(truepositive+truepositiveval)\n",
    "                falsepositivesprchecklist=(len(uniqueChecklist)-(truepositive+truepositiveval)) #number of false positive = predicted positives - true positives\n",
    "                truenegativesprchecklist=truenegativeelement*uniquenegatives #Truenegativeelement (rate of true negatives per item in the checklist) is a number between 0 and 1 and unique negatives is the number of unique items outside the checklist (predicted negatives)\n",
    "                falsenegativesprchecklist=falsenegativeelement*uniquenegatives #Falsenegativeelement=1-truenegativeelemnt and vice versa\n",
    "\n",
    "                #Calculating statistics\n",
    "                precprchecklist=truepositivesprchecklist/len(uniqueChecklist)\n",
    "                precchecklists+=precprchecklist\n",
    "                #print(\"prec: \"+str(precprchecklist))\n",
    "                precprchecklistval=0\n",
    "                if uniquechecklistlengthval>0:\n",
    "                    precprchecklistval=truepositivesprchecklistval/uniquechecklistlengthval\n",
    "                precchecklistsval+=precprchecklistval\n",
    "                #print(\"prec(val): \"+str(precprchecklistval))\n",
    "                recprchecklist=0\n",
    "                if (truepositivesprchecklist+falsenegativesprchecklist)>0:\n",
    "                    recprchecklist=truepositivesprchecklist/(truepositivesprchecklist+falsenegativesprchecklist)\n",
    "                recchecklists+=recprchecklist\n",
    "               # print(\"rec: \"+str(recprchecklist))\n",
    "                recprchecklistval=0\n",
    "                if (truepositivesprchecklistval+falsenegativesprchecklist)>0:\n",
    "                    recprchecklistval=truepositivesprchecklistval/(truepositivesprchecklistval+falsenegativesprchecklist)\n",
    "                recchecklistsval+=recprchecklistval\n",
    "                #print(\"rec(val): \"+str(recprchecklistval))\n",
    "                accprchecklist=0\n",
    "                if (len(uniqueChecklist)+uniquenegatives)>0:\n",
    "                    accprchecklist=(truepositivesprchecklist+truenegativesprchecklist)/(len(uniqueChecklist)+uniquenegatives)\n",
    "                accchecklists+=accprchecklist\n",
    "                #print(\"acc: \"+str(accprchecklist))\n",
    "                accprchecklistval=0\n",
    "                if (uniquechecklistlengthval+uniquenegatives)>0:\n",
    "                    accprchecklistval=(truepositivesprchecklistval+truenegativesprchecklist)/(uniquechecklistlengthval+uniquenegatives)\n",
    "                accchecklistsval+=accprchecklistval\n",
    "\n",
    "                \n",
    "                #|true positives for each checklist|=sum(true positive elements in checklist)\n",
    "                #|false positives for each checklist|=sum(false positive elements in checklist)\n",
    "                #|true negatives for each checklist|=|true negative element|*|unique items(2)|\n",
    "                #|false negatives for each checklist|=|false negative element|*|unique items(2)|\n",
    "                #prec pr checklist\n",
    "                #rec pr checklist\n",
    "                #acc pr checklist\n",
    "           \n",
    "            k+=500 #Increment k while there are less then 10 unique items\n",
    "\n",
    "    if inspectionstot>0:\n",
    "        precctot+=precchecklists/inspectionstot\n",
    "        accctot+=accchecklists/inspectionstot\n",
    "        preccgttot+=precisiongt/lengthprecgt\n",
    "        recctot+=recchecklists/inspectionstot\n",
    "    timetotal=timetotal+(time.time()-start_time)\n",
    "    print(\"Current avg time:\"+ str(timetotal/(ik+1)))\n",
    "    print(\"Precision (gt): \"+str(precisiongt/lengthprecgt))\n",
    "    print(\"Precision: \"+str((precchecklists)/(inspectionstot)))\n",
    "    print(\"Recall: \"+str((recchecklists)/(inspectionstot)))\n",
    "    print(\"Accuracy: \"+str((accchecklists)/(inspectionstot)))\n",
    "    print(\"Average number of items per checklist(gt): \"+str(controlpoints/inspectionstot))\n",
    "    print(\"Average number of items per checklist: \"+str((controlpointsngt)/inspectionstot))\n",
    "    print(\"Average number of non-compliant items per checklist(gt): \"+str(precisiongt/inspectionstot))\n",
    "    print(\"Average number of non-compliant items per checklist: \"+str((noncompliance+noncompliancengt)/inspectionstot))\n",
    "\n",
    "print(\"\\nAverage Traintime: \"+str(traintimetot/8))                                \n",
    "print(\"\\nAverage Accuracy: \"+str(accctot/8))\n",
    "print(\"\\nAverage Prec: \"+str(precctot/8))\n",
    "print(\"\\nAverage Rec: \"+str(recctot/8))\n",
    "print(\"\\nAverage Precgt: \"+str(preccgttot/8))\n",
    "fh2.write(\"\\nAverage Traintime: \"+str(traintimetot/8))                                \n",
    "fh2.write(\"\\nAverage Accuracy: \"+str(accctot/8))\n",
    "fh2.write(\"\\nAverage Prec: \"+str(precctot/8))\n",
    "fh2.write(\"\\nAverage Rec: \"+str(recctot/8))\n",
    "fh2.write(\"\\nAverage Precgt: \"+str(preccgttot/8))\n",
    "fh2.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}