wip

.gitignore

@@ -16,3 +16,7 @@ analysis/*.dat
 analysis/MULTIDIM_PARAM/*
 analysis/NL2SOL/*
 analysis/EGO/*
+
+# others
+*.png
+analysis/summary.txt

analysis/make_plots.py

@@ -10,29 +10,35 @@
     df = pd.read_csv(file, engine="python", sep="\s+")
     df["method"] = file.split('.')[0].split("_")[-1]
     dfs.append(df)
-#%%
+
 df = pd.concat(dfs, ignore_index=True)
 method_cats = CategoricalDtype(categories=["grid", 
                                            "nl2sol", 
                                            "ego"], 
                                ordered=True)
 df["method"] = df["method"].astype(method_cats)
-df = df.set_index(["method", "T", "duration_years"]).drop(columns=["%eval_id", "interface"])
+df = df.set_index(["method", "T", "duration_years"]).drop(columns=["interface"])
 
 # plot evaluations
-p = (ggplot(df.reset_index(), aes(x="T", y="duration_years", fill="rmse", color="rmse")) + 
+p = (ggplot(df.reset_index(), aes(x="T", y="duration_years", color="%eval_id")) + 
+     geom_point() + 
+     scale_color_cmap(name='jet') +
+     facet_wrap("~method"))
+p.save(dpi=300, filename="plot_eval_id.png")
+
+p = (ggplot(df.reset_index(), aes(x="T", y="duration_years", color="rmse")) + 
      geom_point() + 
      facet_wrap("~method"))
-print(p)
+p.save(dpi=300, filename="plot_rmse.png")
 #%%
 # see how results and number of evaluations are influenced by method
-sum_df = df.groupby("method").agg([np.count_nonzero, np.min])
+sum_df = df.drop(columns=["%eval_id"]).groupby("method").agg([np.count_nonzero, np.min])
 sum_df.columns = sum_df.columns.map('|'.join).str.strip('|')
 
 p = (ggplot(sum_df.reset_index(), (aes(x="rmse|count_nonzero", y="rmse|amin", color="method"))) + geom_point())
-print(p)
+p.save(dpi=300, filename="plot_summary.png")
 
 #%%
 # summarized best Ts and durations
 best_df=df[df.rmse.isin(sum_df["rmse|amin"].values)].reset_index()
-print(best_df)
+best_df.to_csv("summary.txt", sep="\t")

resources/clinic_notes.txt

@@ -1,37 +1,26 @@
-# what is calibration/optimization/parameter estimation.
-
-# gradient based vs not
-# complex model vs statistical surogate
-
-# introduction to the model we will use
-diffusion of heat + fitting Clow paper data.
-
-# model A = thermal diffusion
-# model B = temperature history of the surface
-# OF = RMSE fit of temperatures
-
-
-A black box model: Start with Dakota input params.in and 
-create results.out.  
-* pre run (fill in the {{values}}
-* run
-* post run, make a plot with plotnine/pandas
-* you get to choose how much you keep
-
-Step 1: Brute force grid search
-
-- introduction to the input file. 
-- introduction to the model driver
-- introduction to the file structure.
-
-- Run. Look at plot, look at .dat .out file. 
-(FYI, we don't need to keep the file structure) 
-
-
+Intro to calibration
+- what is calibration/optimization/parameter estimation.
+- gradient based vs global
+- complex model vs statistical surogate
+- black box model (parameters > model > outputs)
+- parameters and outputs must be defined.
+
+Our toy model
+- diffusion of heat + surface temperature history
+- OF based on fitting Clow paper data.
+
+Intro to Dakota 
+- Dakota has more bells and whistles, it is well thought out, and the documentation is quite good. Its just extensive and not an iPhone.  
+- Core activity (assuming you have a black box model set up) is to create and run an input file. 
+- Look at .in file. 
+    * discuss each part
+- Look at template file and driver.py (connect this with black box parts)
+- Run Dakota, create plots, look at output. 
+- Discuss Dakota's file structure
+
+Other methods
+We just did a brute force grid search. 
 Step 2: Gradient based method
-
-- we just change the METHOD!!!!!
-
 Step 3: EGO 
 
 Discussion:
@@ -40,7 +29,8 @@ Discussion:
     - increasing dimension
     - 
 * RST file, .out file and reproducible research
-
+* We haven't yet talked about the uncertainty estimates on  parameters, just which  
+parameter is best. That is for another day.  
 
 Exploration if time:
 * Explore other methods