modified stats generator

code/data_preprocess_remove_bogus_images.py

@@ -0,0 +1,14 @@
+import os,sys
+import caffe
+import numpy as np
+import glob
+
+if __name__=="__main__":
+    if len(sys.argv)<3:
+        print "usage python data_preprocess_remove_bogus_images.py <reference-image-file> <removal-folder>"
+    else:
+        ref_image_file = sys.argv[1]
+        ref_image_folder = sys.argv[2]
+
+        ref_image = np.array(caffe.io.load_image(ref_image_file))
+

code/vis_generate_geographic_stats.py

@@ -0,0 +1,49 @@
+import os
+import numpy as np
+import glob
+import matplotlib.pyplot as plt
+
+
+src_folder = "/Users/dubeya/normalized/"
+
+cities = {}
+cities['USA'] = ['WashingtonDC','Chicago','Denver','Valparaiso','Minneapolis','Seattle','NewYork','LosAngeles','Portland','Houston','Boston','SanFrancisco']
+cities['Europe'] = ['Kiev','Paris','Stockholm','Zagreb','Moscow','Helsinki','Munich','Bratislava','Lisbon','Dublin','Barcelona','Rome','Bucharest','Prague','Milan','London','Madrid','Glasgow','Amsterdam','Copenhagen','Warsaw']
+cities['NorthAmerica'] = cities['USA']+['Guadalajara','Montreal','MexicoCity','Toronto']
+cities['SouthAmerica'] = ['BeloHorizonte','Santiago','SaoPaulo']
+cities['Asia'] = ['Seoul','Bangkok','TelAviv','HongKong','Tokyo','Taipei','Singapore']
+cities['Africa'] = ['Gaborone']
+
+csv_files = glob.glob(os.path.join(src_folder,"*.csv"))
+
+global_mean = 0
+citywise_stats = {}
+citywise_scores = {}
+
+
+for csv_file in csv_files:
+    scores = []
+    cityname = os.path.basename(csv_file).replace("_normalized.csv","")
+    with open(csv_file,'r') as f:
+        for line in f:
+            score = float(line.strip().split(',')[1])
+            scores.append(score)
+    citywise_scores[cityname]=scores
+
+bins = np.linspace(0,10,200)
+
+print citywise_scores.keys()
+plt.figure(figsize=(20,20))
+
+for cont,cities in cities.iteritems():
+    score_hist = []
+    for city in cities:
+        score_hist += citywise_scores[city]
+    cont_score_mean = np.asarray(score_hist).mean()
+    cont_score_sigma = np.asarray(score_hist).std()
+    plt.hist(np.asarray(score_hist),bins,alpha=0.2,label=cont+" "+str(cont_score_mean)+" "+str(cont_score_sigma),normed=True)
+
+plt.legend(loc='upper right')
+plt.show()
+
+

code/vis_generate_stats.py

@@ -1,11 +1,12 @@
 import os
+import sys
 import glob
 import numpy as np
 import matplotlib
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 
-src_folder = "/home/dubeya/extrapolated_votes/safety/scores_new/normalized/"
+src_folder = sys.argv[1]
 
 csv_files = glob.glob(os.path.join(src_folder,"*.csv"))
 
@@ -15,7 +16,7 @@
 
 for csv_file in csv_files:
     scores = []
-    cityname = os.path.basename(csv_file).replace("_normalized.csv","")
+    cityname = os.path.basename(csv_file).replace(".csv","")
     with open(csv_file,'r') as f:
         for line in f:
             score = float(line.strip().split(',')[1])
@@ -24,7 +25,7 @@
 
 bins = np.linspace(0,10,200)
 
-stats_file = "/home/dubeya/extrapolated_votes/safety/scores_new/normalized/stats.txt"
+stats_file = sys.argv[2]
 
 plt.figure(figsize=(35,35))
 with open(stats_file,'w') as f: