Skip to content

Commit

Permalink
better script and relevance
Browse files Browse the repository at this point in the history
  • Loading branch information
sroy9 committed Jun 2, 2017
1 parent 103cc7e commit 1d851bb
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 55 deletions.
63 changes: 21 additions & 42 deletions data/handcrafted/questions.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
"lSolutions": [
43.0
],
"infType": 0,
"key": "0_0"
"rates": []
},
{
"iIndex": 100002,
Expand All @@ -28,8 +27,7 @@
"lSolutions": [
97.0
],
"infType": 0,
"key": "0_1"
"rates": []
},
{
"iIndex": 100003,
Expand All @@ -44,8 +42,7 @@
"lSolutions": [
43.0
],
"infType": 0,
"key": "0_1"
"rates": []
},
{
"iIndex": 100004,
Expand All @@ -60,8 +57,7 @@
"lSolutions": [
43.0
],
"infType": 0,
"key": "1_0"
"rates": []
},
{
"iIndex": 100005,
Expand All @@ -76,8 +72,7 @@
"lSolutions": [
97.0
],
"infType": 0,
"key": "1_0"
"rates": []
},
{
"iIndex": 100006,
Expand All @@ -92,8 +87,7 @@
"lSolutions": [
43.0
],
"infType": 0,
"key": "0_1"
"rates": []
},
{
"iIndex": 100007,
Expand All @@ -108,8 +102,7 @@
"lSolutions": [
97.0
],
"infType": 0,
"key": "0_1"
"rates": []
},
{
"iIndex": 100008,
Expand All @@ -124,8 +117,7 @@
"lSolutions": [
97.0
],
"infType": 0,
"key": "1_0"
"rates": []
},
{
"iIndex": 100009,
Expand All @@ -140,8 +132,7 @@
"lSolutions": [
43.0
],
"infType": 0,
"key": "1_0"
"rates": []
},
{
"iIndex": 100010,
Expand All @@ -156,8 +147,7 @@
"lSolutions": [
5
],
"infType": 2,
"key": "0_0"
"rates": []
},
{
"iIndex": 100011,
Expand All @@ -172,8 +162,7 @@
"lSolutions": [
11
],
"infType": 2,
"key": "0_1"
"rates": []
},
{
"iIndex": 100012,
Expand All @@ -188,8 +177,7 @@
"lSolutions": [
5
],
"infType": 2,
"key": "0_0"
"rates": []
},
{
"iIndex": 100013,
Expand All @@ -204,8 +192,7 @@
"lSolutions": [
11
],
"infType": 2,
"key": "0_1"
"rates": []
},
{
"iIndex": 100014,
Expand All @@ -220,8 +207,7 @@
"lSolutions": [
1
],
"infType": 2,
"key": "1"
"rates": []
},
{
"iIndex": 100015,
Expand All @@ -236,8 +222,7 @@
"lSolutions": [
20
],
"infType": 3,
"key": "1_DEN"
"rates": [1]
},
{
"iIndex": 100016,
Expand All @@ -252,8 +237,7 @@
"lSolutions": [
4
],
"infType": 3,
"key": "0_NUM"
"rates": [1]
},
{
"iIndex": 100017,
Expand All @@ -268,8 +252,7 @@
"lSolutions": [
4
],
"infType": 3,
"key": "QUES"
"rates": [-1]
},
{
"iIndex": 100018,
Expand All @@ -284,8 +267,7 @@
"lSolutions": [
4
],
"infType": 3,
"key": "QUES"
"rates": [-1]
},
{
"iIndex": 100019,
Expand All @@ -300,8 +282,7 @@
"lSolutions": [
11
],
"infType": 2,
"key": "SIBLING"
"rates": []
},
{
"iIndex": 100020,
Expand All @@ -316,8 +297,7 @@
"lSolutions": [
5
],
"infType": 2,
"key": "HYPER"
"rates": []
},
{
"iIndex": 100021,
Expand All @@ -332,8 +312,7 @@
"lSolutions": [
5
],
"infType": 2,
"key": "HYPO"
"rates": []
}
]

5 changes: 4 additions & 1 deletion runAllExpts2.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ sh run.sh --mode UnitDep --cv data/allArithLex/fold0.txt data/allArithLex/fold1.

sh run.sh --mode UnitDep --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/UnitDepAllTmpl.out

sh run.sh --mode UnitDep --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/UnitDepPerturb.out



sh run.sh --mode LCA --cv data/allArith/fold0.txt data/allArith/fold1.txt data/allArith/fold2.txt data/allArith/fold3.txt data/allArith/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAll.out
Expand All @@ -14,6 +16,7 @@ sh run.sh --mode LCA --cv data/allArithLex/fold0.txt data/allArithLex/fold1.txt

sh run.sh --mode LCA --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/LCAAllTmpl.out

sh run.sh --mode LCA --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/LCAPerturb.out



Expand All @@ -23,4 +26,4 @@ sh run.sh --mode E2ELogic --cv data/allArithLex/fold0.txt data/allArithLex/fold1

sh run.sh --mode E2ELogic --cv data/allArithTmpl/fold0.txt data/allArithTmpl/fold1.txt data/allArithTmpl/fold2.txt data/allArithTmpl/fold3.txt data/allArithTmpl/fold4.txt --model_dir models2/ --print_mistakes > log/E2ELogicAllTmpl.out


sh run.sh --mode E2ELogic --train data/perturb/old.txt --test data/perturb/new.txt --model_dir models2/ --print_mistakes > log/E2ELogicPerturb.out
15 changes: 8 additions & 7 deletions src/main/java/logic/Relevance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,9 @@ public static boolean irrelevance(LogicX x, int quantIndex) {
double maxSim = 0.0, minSim = 1.0, maxSimForIndex = 0.0;
for(int i=0; i<n; ++i) {
for(int j=i+1; j<n; ++j) {
double sim = Tools.jaccardSim(unitRates.get(new IntPair(i, 0)),
unitRates.get(new IntPair(j, 0)));
double sim = Math.max(
Tools.jaccardSim(unitRates.get(new IntPair(i, 0)), unitRates.get(new IntPair(j, 0))),
Tools.jaccardSim(unitRates.get(new IntPair(i, 0)), unitRates.get(new IntPair(j, 1))));
if (i == quantIndex || j == quantIndex) {
if(sim > maxSimForIndex) maxSimForIndex = sim;
} else {
Expand All @@ -179,11 +180,11 @@ public static boolean irrelevance(LogicX x, int quantIndex) {
}
}
}
System.out.println("ProblemId: "+x.problemId+"|| QuantIndex: "+quantIndex+
"|| MaxSimForIndex: "+maxSimForIndex+"|| MaxSim: "+maxSim+
"|| MinSim: "+minSim);
System.out.println("Extractions: "+Arrays.asList(unitRates));
System.out.println();
// System.out.println("ProblemId: "+x.problemId+"|| QuantIndex: "+quantIndex+
// "|| MaxSimForIndex: "+maxSimForIndex+"|| MaxSim: "+maxSim+
// "|| MinSim: "+minSim);
// System.out.println("Extractions: "+Arrays.asList(unitRates));
// System.out.println();
if(minSim - maxSimForIndex > 0.51 && !otherDependentNumber) {
if(Tools.jaccardSim(quesUnit, unitRates.get(
new IntPair(quantIndex, 0))) < minSim-0.01) {
Expand Down
47 changes: 42 additions & 5 deletions src/main/java/reader/Dataset.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ public static void combineTwoSetsToOneDataset() throws Exception {
}

public static void consistencyChecks() throws Exception {
System.out.println("Running consistency checks");
String json = FileUtils.readFileToString(new File("data/questions.json"));
List<DataFormat> kushmanProbs = new Gson().fromJson(json,
new TypeToken<List<DataFormat>>(){}.getType());
Expand Down Expand Up @@ -403,23 +404,42 @@ public static void analyzeErrors(String fileName1, String fileName2, int total)
Set<Integer> bothWrong = new HashSet<>();
Set<Integer> RightWrong = new HashSet<>();
Set<Integer> WrongRight = new HashSet<>();
for(String line : lines1) {
Map<Integer, String> probIdToString1 = new HashMap<>();
Map<Integer, String> probIdToString2 = new HashMap<>();
String bW = "", RW = "", WR = "";
for(int j=0; j<lines1.size(); ++j) {
String line = lines1.get(j);
Integer i;
try {
i = Integer.parseInt(line.split(" : ")[0].trim());
} catch (Exception e) {
continue;
}
err1.add(i);
String s = lines1.get(j) + "\n";
do {
j++;
s += lines1.get(j) + "\n";
} while(!lines1.get(j).startsWith("Pred"));

probIdToString1.put(i, s);
}
for(String line : lines2) {
for(int j=0; j<lines2.size(); ++j) {
String line = lines2.get(j);
Integer i;
try {
i = Integer.parseInt(line.split(" : ")[0].trim());
} catch (Exception e) {
continue;
}
err2.add(i);
String s = lines2.get(j) + "\n";
do {
j++;
s += lines2.get(j) + "\n";
} while(!lines2.get(j).startsWith("Pred"));

probIdToString2.put(i, s);
}
double b = 0, c = 0, union = err2.size(), intersection = 0;
for(Integer i : err1) {
Expand All @@ -439,13 +459,30 @@ public static void analyzeErrors(String fileName1, String fileName2, int total)
RightWrong.add(i);
}
}
System.out.println("Acc1: "+(1.0 - (err1.size()*1.0/total)));
System.out.println("Acc2: "+(1.0 - (err2.size()*1.0/total)));
System.out.println("Yes Yes : "+(total - union));
System.out.println("Yes No : "+c);
System.out.println("No Yes : "+b);
System.out.println("No No : "+intersection);
System.out.println("RightWrong: "+Arrays.asList(RightWrong));
System.out.println("WrongRight: "+Arrays.asList(WrongRight));
System.out.println("bothWrong: "+Arrays.asList(bothWrong));
// System.out.println("RightWrong: "+Arrays.asList(RightWrong));
// System.out.println("WrongRight: "+Arrays.asList(WrongRight));
// System.out.println("bothWrong: "+Arrays.asList(bothWrong));
String str = "";
for(Integer i : RightWrong) {
str += probIdToString2.get(i)+"\n";
}
FileUtils.writeStringToFile(new File("right_wrong.txt"), str);
str = "";
for(Integer i : WrongRight) {
str += probIdToString1.get(i)+"\n";
}
FileUtils.writeStringToFile(new File("wrong_right.txt"), str);
str = "";
for(Integer i : bothWrong) {
str += probIdToString2.get(i)+"\n";
}
FileUtils.writeStringToFile(new File("both_wrong.txt"), str);
}


Expand Down

0 comments on commit 1d851bb

Please sign in to comment.