From 1d851bbd22a8c47568dfc9af1c6f38b8dea9e36a Mon Sep 17 00:00:00 2001
From: sroy9 <sroy9@illinois.edu>
Date: Thu, 1 Jun 2017 21:10:45 -0500
Subject: [PATCH] better script and relevance

---
 data/handcrafted/questions.json    | 63 ++++++++++--------------------
 runAllExpts2.sh                    |  5 ++-
 src/main/java/logic/Relevance.java | 15 +++----
 src/main/java/reader/Dataset.java  | 47 +++++++++++++++++++---
 4 files changed, 75 insertions(+), 55 deletions(-)

diff --git a/data/handcrafted/questions.json b/data/handcrafted/questions.json
index ef6a4e3..8028e2f 100644
--- a/data/handcrafted/questions.json
+++ b/data/handcrafted/questions.json
@@ -12,8 +12,7 @@
     "lSolutions": [
       43.0
     ],
-    "infType": 0,
-    "key": "0_0"
+    "rates": []
   },
   {
     "iIndex": 100002,
@@ -28,8 +27,7 @@
     "lSolutions": [
       97.0
     ],
-    "infType": 0,
-    "key": "0_1"
+    "rates": []
   },
   {
     "iIndex": 100003,
@@ -44,8 +42,7 @@
     "lSolutions": [
       43.0
     ],
-    "infType": 0,
-    "key": "0_1"
+    "rates": []
   },
   {
     "iIndex": 100004,
@@ -60,8 +57,7 @@
     "lSolutions": [
       43.0
     ],
-    "infType": 0,
-    "key": "1_0"
+    "rates": []
   },
   {
     "iIndex": 100005,
@@ -76,8 +72,7 @@
     "lSolutions": [
       97.0
     ],
-    "infType": 0,
-    "key": "1_0"
+    "rates": []
   },
   {
     "iIndex": 100006,
@@ -92,8 +87,7 @@
     "lSolutions": [
       43.0
     ],
-    "infType": 0,
-    "key": "0_1"
+    "rates": []
   },
   {
     "iIndex": 100007,
@@ -108,8 +102,7 @@
     "lSolutions": [
       97.0
     ],
-    "infType": 0,
-    "key": "0_1"
+    "rates": []
   },
   {
     "iIndex": 100008,
@@ -124,8 +117,7 @@
     "lSolutions": [
       97.0
     ],
-    "infType": 0,
-    "key": "1_0"
+    "rates": []
   },
   {
     "iIndex": 100009,
@@ -140,8 +132,7 @@
     "lSolutions": [
       43.0
     ],
-    "infType": 0,
-    "key": "1_0"
+    "rates": []
   },
   {
     "iIndex": 100010,
@@ -156,8 +147,7 @@
     "lSolutions": [
       5
     ],
-    "infType": 2,
-    "key": "0_0"
+    "rates": []
   },
   {
     "iIndex": 100011,
@@ -172,8 +162,7 @@
     "lSolutions": [
       11
     ],
-    "infType": 2,
-    "key": "0_1"
+    "rates": []
   },
   {
     "iIndex": 100012,
@@ -188,8 +177,7 @@
     "lSolutions": [
       5
     ],
-    "infType": 2,
-    "key": "0_0"
+    "rates": []
   },
   {
     "iIndex": 100013,
@@ -204,8 +192,7 @@
     "lSolutions": [
       11
     ],
-    "infType": 2,
-    "key": "0_1"
+    "rates": []
   },
   {
     "iIndex": 100014,
@@ -220,8 +207,7 @@
     "lSolutions": [
       1
     ],
-    "infType": 2,
-    "key": "1"
+    "rates": []
   },
   {
     "iIndex": 100015,
@@ -236,8 +222,7 @@
     "lSolutions": [
       20
     ],
-    "infType": 3,
-    "key": "1_DEN"
+    "rates": [1]
   },
   {
     "iIndex": 100016,
@@ -252,8 +237,7 @@
     "lSolutions": [
       4
     ],
-    "infType": 3,
-    "key": "0_NUM"
+    "rates": [1]
   },
   {
     "iIndex": 100017,
@@ -268,8 +252,7 @@
     "lSolutions": [
       4
     ],
-    "infType": 3,
-    "key": "QUES"
+    "rates": [-1]
   },
   {
     "iIndex": 100018,
@@ -284,8 +267,7 @@
     "lSolutions": [
       4
     ],
-    "infType": 3,
-    "key": "QUES"
+    "rates": [-1]
   },
   {
     "iIndex": 100019,
@@ -300,8 +282,7 @@
     "lSolutions": [
       11
     ],
-    "infType": 2,
-    "key": "SIBLING"
+    "rates": []
   },
   {
     "iIndex": 100020,
@@ -316,8 +297,7 @@
     "lSolutions": [
       5
     ],
-    "infType": 2,
-    "key": "HYPER"
+    "rates": []
   },
   {
     "iIndex": 100021,
@@ -332,8 +312,7 @@
     "lSolutions": [
       5
     ],
-    "infType": 2,
-    "key": "HYPO"
+    "rates": []
   }
 ]
   
diff --git a/runAllExpts2.sh b/runAllExpts2.sh
index a8e7e13..6754dd5 100644
--- a/runAllExpts2.sh
+++ b/runAllExpts2.sh
@@ -6,6 +6,8 @@ sh run.sh --mode UnitDep --cv data/allArithLex/fold0.txt data/allArithLex/fold1.
 
 sh run.sh --mode UnitDep --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/UnitDepAllTmpl.out 
 
+sh run.sh --mode UnitDep --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/UnitDepPerturb.out
+
 
 
 sh run.sh --mode LCA --cv data/allArith/fold0.txt data/allArith/fold1.txt data/allArith/fold2.txt data/allArith/fold3.txt data/allArith/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAll.out
@@ -14,6 +16,7 @@ sh run.sh --mode LCA --cv data/allArithLex/fold0.txt data/allArithLex/fold1.txt
 
 sh run.sh --mode LCA --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAllTmpl.out 
 
+sh run.sh --mode LCA --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/LCAPerturb.out
 
 
 
@@ -23,4 +26,4 @@ sh run.sh --mode E2ELogic --cv data/allArithLex/fold0.txt data/allArithLex/fold1
 
 sh run.sh --mode E2ELogic --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/E2ELogicAllTmpl.out 
 
-
+sh run.sh --mode E2ELogic --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/E2ELogicPerturb.out
diff --git a/src/main/java/logic/Relevance.java b/src/main/java/logic/Relevance.java
index 1f6edd6..e08eebe 100644
--- a/src/main/java/logic/Relevance.java
+++ b/src/main/java/logic/Relevance.java
@@ -165,8 +165,9 @@ public static boolean irrelevance(LogicX x, int quantIndex) {
         double maxSim = 0.0, minSim = 1.0, maxSimForIndex = 0.0;
         for(int i=0; i<n; ++i) {
             for(int j=i+1; j<n; ++j) {
-                double sim = Tools.jaccardSim(unitRates.get(new IntPair(i, 0)),
-                                unitRates.get(new IntPair(j, 0)));
+                double sim = Math.max(
+                        Tools.jaccardSim(unitRates.get(new IntPair(i, 0)), unitRates.get(new IntPair(j, 0))),
+                        Tools.jaccardSim(unitRates.get(new IntPair(i, 0)), unitRates.get(new IntPair(j, 1))));
                 if (i == quantIndex || j == quantIndex) {
                     if(sim > maxSimForIndex) maxSimForIndex = sim;
                 } else {
@@ -179,11 +180,11 @@ public static boolean irrelevance(LogicX x, int quantIndex) {
                 }
             }
         }
-        System.out.println("ProblemId: "+x.problemId+"|| QuantIndex: "+quantIndex+
-                "|| MaxSimForIndex: "+maxSimForIndex+"|| MaxSim: "+maxSim+
-                "|| MinSim: "+minSim);
-        System.out.println("Extractions: "+Arrays.asList(unitRates));
-        System.out.println();
+//        System.out.println("ProblemId: "+x.problemId+"|| QuantIndex: "+quantIndex+
+//                "|| MaxSimForIndex: "+maxSimForIndex+"|| MaxSim: "+maxSim+
+//                "|| MinSim: "+minSim);
+//        System.out.println("Extractions: "+Arrays.asList(unitRates));
+//        System.out.println();
         if(minSim - maxSimForIndex > 0.51 && !otherDependentNumber) {
             if(Tools.jaccardSim(quesUnit, unitRates.get(
                         new IntPair(quantIndex, 0))) < minSim-0.01) {
diff --git a/src/main/java/reader/Dataset.java b/src/main/java/reader/Dataset.java
index 6239f71..d11822f 100644
--- a/src/main/java/reader/Dataset.java
+++ b/src/main/java/reader/Dataset.java
@@ -243,6 +243,7 @@ public static void combineTwoSetsToOneDataset() throws Exception {
 	}
 
 	public static void consistencyChecks() throws Exception {
+		System.out.println("Running consistency checks");
 		String json = FileUtils.readFileToString(new File("data/questions.json"));
 		List<DataFormat> kushmanProbs = new Gson().fromJson(json,
 				new TypeToken<List<DataFormat>>(){}.getType());
@@ -403,7 +404,11 @@ public static void analyzeErrors(String fileName1, String fileName2, int total)
 		Set<Integer> bothWrong = new HashSet<>();
 		Set<Integer> RightWrong = new HashSet<>();
 		Set<Integer> WrongRight = new HashSet<>();
-		for(String line : lines1) {
+		Map<Integer, String> probIdToString1 = new HashMap<>();
+		Map<Integer, String> probIdToString2 = new HashMap<>();
+		String bW = "", RW = "", WR = "";
+		for(int j=0; j<lines1.size(); ++j) {
+			String line = lines1.get(j);
 			Integer i;
 			try {
 				i = Integer.parseInt(line.split(" : ")[0].trim());
@@ -411,8 +416,16 @@ public static void analyzeErrors(String fileName1, String fileName2, int total)
 				continue;
 			}
 			err1.add(i);
+			String s = lines1.get(j) + "\n";
+			do {
+				j++;
+				s += lines1.get(j) + "\n";
+			} while(!lines1.get(j).startsWith("Pred"));
+
+			probIdToString1.put(i, s);
 		}
-		for(String line : lines2) {
+		for(int j=0; j<lines2.size(); ++j) {
+			String line = lines2.get(j);
 			Integer i;
 			try {
 				i = Integer.parseInt(line.split(" : ")[0].trim());
@@ -420,6 +433,13 @@ public static void analyzeErrors(String fileName1, String fileName2, int total)
 				continue;
 			}
 			err2.add(i);
+			String s = lines2.get(j) + "\n";
+			do {
+				j++;
+				s += lines2.get(j) + "\n";
+			} while(!lines2.get(j).startsWith("Pred"));
+
+			probIdToString2.put(i, s);
 		}
 		double b = 0, c = 0, union = err2.size(), intersection = 0;
 		for(Integer i : err1) {
@@ -439,13 +459,30 @@ public static void analyzeErrors(String fileName1, String fileName2, int total)
 				RightWrong.add(i);
 			}
 		}
+		System.out.println("Acc1: "+(1.0 - (err1.size()*1.0/total)));
+		System.out.println("Acc2: "+(1.0 - (err2.size()*1.0/total)));
 		System.out.println("Yes Yes : "+(total - union));
 		System.out.println("Yes No : "+c);
 		System.out.println("No Yes : "+b);
 		System.out.println("No No : "+intersection);
-		System.out.println("RightWrong: "+Arrays.asList(RightWrong));
-		System.out.println("WrongRight: "+Arrays.asList(WrongRight));
-		System.out.println("bothWrong: "+Arrays.asList(bothWrong));
+//		System.out.println("RightWrong: "+Arrays.asList(RightWrong));
+//		System.out.println("WrongRight: "+Arrays.asList(WrongRight));
+//		System.out.println("bothWrong: "+Arrays.asList(bothWrong));
+		String str = "";
+		for(Integer i : RightWrong) {
+			str += probIdToString2.get(i)+"\n";
+		}
+		FileUtils.writeStringToFile(new File("right_wrong.txt"), str);
+		str = "";
+		for(Integer i : WrongRight) {
+			str += probIdToString1.get(i)+"\n";
+		}
+		FileUtils.writeStringToFile(new File("wrong_right.txt"), str);
+		str = "";
+		for(Integer i : bothWrong) {
+			str += probIdToString2.get(i)+"\n";
+		}
+		FileUtils.writeStringToFile(new File("both_wrong.txt"), str);
 	}