diff --git a/.idea/runConfigurations/MVBatchTest_handSelectedDates.xml b/.idea/runConfigurations/MVBatchTest_handSelectedDates.xml new file mode 100644 index 00000000..79a5b98a --- /dev/null +++ b/.idea/runConfigurations/MVBatchTest_handSelectedDates.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/R_work/include/agg_stat.R b/R_work/include/agg_stat.R index f76b26b1..406c5f53 100644 --- a/R_work/include/agg_stat.R +++ b/R_work/include/agg_stat.R @@ -10,7 +10,7 @@ listArgs = commandArgs(TRUE) if( 0 < length(listArgs) ) { strInputInfoFile = listArgs[1]; } -cat("agg_stat.R\ninput file: ", strInputInfoFile, "\n", sep=""); +cat("input file: ", strInputInfoFile, "\n", sep=""); source(strInputInfoFile); setwd(strWorkingDir); @@ -30,7 +30,8 @@ listStat = NA; if ( nrow(sampleData) > 0){ classes <- sapply(sampleData, class); numeric_columns <- c('stat_value', 'stat_bcl', 'stat_bcu', 'stat_ncu', 'stat_ncl','fbs', 'fss', -'fbar', 'obar', 'fobar', 'ffbar', 'oobar', 'var_mean','total', 'me', 'rmse', 'crps','crpss', 'ign', 'spread', 'me_oerr','rmse_oerr','spread_oerr', 'spread_plus_oerr' +'fbar', 'obar', 'fobar', 'ffbar', 'oobar', 'var_mean','total', 'me', 'rmse', 'crps', + 'crpss', 'ign', 'spread', 'me_oerr','rmse_oerr','spread_oerr', 'spread_plus_oerr', 'mae' ); integer_columns <- c('fcst_lead','nstats'); character_columns <- c('fcst_var', 'model', 'stat_name') @@ -1062,7 +1063,3 @@ cat( " seed: ", intRandomSeed, "\n", " stats: ", length(listStat), "\n", sep=""); - - -# clean up -cat("agg_stat.R done\n"); \ No newline at end of file diff --git a/R_work/include/agg_stat_bootstrap.R b/R_work/include/agg_stat_bootstrap.R index 53eff670..0d4561f7 100644 --- a/R_work/include/agg_stat_bootstrap.R +++ b/R_work/include/agg_stat_bootstrap.R @@ -6,7 +6,7 @@ listArgs = commandArgs(TRUE) if( 0 < length(listArgs) ) { strInputInfoFile = listArgs[1]; } -cat("agg_stat_bootstrap.R\ninput file: ", strInputInfoFile, "\n", sep=""); +cat("input file: ", strInputInfoFile, "\n", sep=""); source(strInputInfoFile); setwd(strWorkingDir); @@ -1213,13 +1213,11 @@ for(strIndyVal in listIndyVal){ } # END: for intY - cat(" PROC TIME: ", formatTimeSpan(as.numeric(Sys.time() - stBoot, units="secs")), "\n") + #cat(" PROC TIME: ", formatTimeSpan(as.numeric(Sys.time() - stBoot, units="secs")), "\n") } # END: for strIndyVal write.table( dfOut, file=strOutputFile, row.names=FALSE, quote=FALSE, sep="\t" ); -# clean up -cat("agg_stat_bootstrap.R done\n"); diff --git a/R_work/include/sum_stat.R b/R_work/include/sum_stat.R index 8d647428..5d70e672 100644 --- a/R_work/include/sum_stat.R +++ b/R_work/include/sum_stat.R @@ -10,7 +10,7 @@ listArgs = commandArgs(TRUE) if( 0 < length(listArgs) ) { strInputInfoFile = listArgs[1]; } -cat("sum_stat.R\ninput file: ", strInputInfoFile, "\n", sep=""); +cat("file: ", strInputInfoFile, "\n", sep=""); source(strInputInfoFile); setwd(strWorkingDir); @@ -214,5 +214,3 @@ cat( " total time: ", formatTimeSpan(as.numeric(Sys.time() - stStart, units="secs")), "\n", sep=""); -# clean up -cat("sum_stat.R done\n"); \ No newline at end of file diff --git a/R_work/include/util_plot.R b/R_work/include/util_plot.R index e3d3bf11..02f94842 100644 --- a/R_work/include/util_plot.R +++ b/R_work/include/util_plot.R @@ -396,7 +396,7 @@ formatTimeSpan = function(s){ if( 10 > intMins ){ strFormat = paste(strFormat, "0", sep=""); } strFormat = paste(strFormat, intMins, ":", sep=""); if( 10 > dblSec ){ strFormat = paste(strFormat, "0", sep=""); } - strFormat = paste(strFormat, format(dblSec, digits=6), sep=""); + strFormat = paste(strFormat, format(dblSec, digits=3), sep=""); return( strFormat ); } diff --git a/java/edu/ucar/metviewer/MVBatch.java b/java/edu/ucar/metviewer/MVBatch.java index 44a2f2c7..29682846 100644 --- a/java/edu/ucar/metviewer/MVBatch.java +++ b/java/edu/ucar/metviewer/MVBatch.java @@ -2,8 +2,6 @@ import java.io.File; import java.io.PrintStream; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -328,12 +326,12 @@ public static void main(String[] argv) throws Exception { bat.setNumPlots(bat.getNumPlots() + intNumJobPlots); } - LocalDateTime dateStart = LocalDateTime.now(); - bat.print("Running " + bat.numPlots + " plots\n" - + "Begin time: " + MVUtil.APP_DATE_FORMATTER.format(dateStart) + "\n"); + StopWatch jobsStopWatch = new StopWatch(); + bat.print("Running " + bat.numPlots + " plots"); for (int intJob = 0; intJob < jobs.length; intJob++) { + jobsStopWatch.start(); if (0 < intJob) { bat.print( "\n# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #\n"); @@ -377,24 +375,27 @@ public static void main(String[] argv) throws Exception { jobManager.runJob(jobs[intJob]); bat.numPlotsRun++; + jobsStopWatch.stop(); + bat.print("\n" + "Job " + (intJob+1) + " execution time " + jobsStopWatch.getFormattedDuration()); } + stopWatch.stop(); + long plotAvg= (jobsStopWatch.getTotalDuration()/ 1000000) / (long) bat.numPlots; - LocalDateTime dateEnd = LocalDateTime.now(); - long plotTime = dateStart.until(dateEnd, ChronoUnit.MILLIS); - long plotAvg = 0 < bat.numPlots ? plotTime / (long) bat.numPlots : 0; bat.print("\n" - + MVUtil.padBegin("End time: ") + MVUtil.APP_DATE_FORMATTER.format(dateEnd) + "\n" + MVUtil.padBegin("Plots run: ") + bat.getNumPlotsRun() + " of " + bat.getNumPlots() + "\n" - + MVUtil.padBegin("Total time: ") + MVUtil.formatTimeSpan(plotTime) + "\n" + + MVUtil.padBegin("Total time: ") + jobsStopWatch.getFormattedTotalDuration() + "\n" + MVUtil.padBegin("Avg plot time: ") + MVUtil.formatTimeSpan(plotAvg) + "\n"); + + } catch (Exception e) { bat.print(" ** ERROR: " + e.getMessage()); } bat.print("---- MVBatch Done ----"); + bat.print("\nTotal execution time " + stopWatch.getFormattedTotalDuration()); } diff --git a/java/edu/ucar/metviewer/scorecard/Scorecard.java b/java/edu/ucar/metviewer/scorecard/Scorecard.java index fd05e303..dc4cc501 100644 --- a/java/edu/ucar/metviewer/scorecard/Scorecard.java +++ b/java/edu/ucar/metviewer/scorecard/Scorecard.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Map; +import edu.ucar.metviewer.StopWatch; import edu.ucar.metviewer.scorecard.db.AggDatabaseManagerMySQL; import edu.ucar.metviewer.scorecard.db.DatabaseManager; import edu.ucar.metviewer.scorecard.db.SumDatabaseManagerMySQL; @@ -35,6 +36,8 @@ public class Scorecard { private static final Logger logger = LogManager.getLogger("Console"); + private static final String USAGE = "USAGE: mv_scorecard.sh db_type \n" + + " where db_type - mysql \n specifies the XML scorecard specification document\n"; private String databaseName; private String user; private String pwd; @@ -60,8 +63,117 @@ public class Scorecard { private String stat = "DIFF_SIG"; private String thresholdFile = null; - private static final String USAGE = "USAGE: mv_scorecard.sh db_type \n" + - " where db_type - mysql \n specifies the XML scorecard specification document\n"; + public static void main(String[] args) throws Exception { + + String filename; + String dbType = "mysql"; + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + if (0 == args.length) { + logger.error(" Error: no arguments!!!"); + logger.info(USAGE); + + } else { + + int intArg = 0; + for (; intArg < args.length && !args[intArg].matches(".*\\.xml$"); intArg++) { + if (args[intArg].equals("mysql")) { + dbType = "mysql"; + } + } + + filename = args[intArg]; + XmlParser xmlParser = new XmlParser(); + // parce XML and init parameters + Scorecard scorecard = xmlParser.parseParameters(filename); + + //add a second model ( the same as the first one) if only one is selected + scorecard.fillValues(); + + //remove previous output with similar names + scorecard.cleanOldResults(); + + //TODO implement validation + boolean isValid = scorecard.validate(); + if (isValid) { + + DatabaseManager scorecardDbManager = null; + RscriptManager rscriptManager = null; + //create a list of each row with statistic as a key and columns + List> listRows = scorecard.getListOfEachRowWithDesc(); + + //depending on stat type init mangers + if (scorecard.getAggStat()) { + if (dbType.equals("mysql")) { + scorecardDbManager = new AggDatabaseManagerMySQL(scorecard); + } + rscriptManager = new AggRscriptManager(scorecard); + } else { + if (dbType.equals("mysql")) { + scorecardDbManager = new SumDatabaseManagerMySQL(scorecard); + } + rscriptManager = new SumRscriptManager(scorecard); + } + int rowCounter = 1; + stopWatch.stop(); + logger.info("Scorecard init time " + stopWatch.getFormattedDuration()); + + //for each row calculate statistics in the individual cell + for (Map mapRow : listRows) { + + stopWatch.start(); + StringBuilder logMessage = new StringBuilder(); + for (Map.Entry column : mapRow.entrySet()) { + logMessage.append(column.getKey()).append(": ").append(column.getValue().getName()) + .append(", "); + } + logger.info( + "---------------------------------------------------------------------------------------"); + logger.info("Row #" + rowCounter + ": " + logMessage); + logger.info( + "---------------------------------------------------------------------------------------"); + + try { + //get data from db and save it into file + scorecardDbManager.createDataFile(mapRow, ""); + + //use rscript and data from the db file to calculate stats and append them into the resulting file + rscriptManager.calculateStatsForRow(mapRow, ""); + + } catch (Exception e) { + logger.error(e.getMessage()); + } + stopWatch.stop(); + logger.info("\nRow execution time " + stopWatch.getFormattedDuration()); + logger.info( + "---------------------------------------------------------------------------------------"); + rowCounter++; + + } + stopWatch.start(); + File dataFile = new File(scorecard.getWorkingFolders().getDataDir() + + scorecard.getDataFile()); + //if the resulting file exists - create an image and html file + if (dataFile.exists()) { + stopWatch.stop(); + stopWatch.start(); + GraphicalOutputManager graphicalOutputManager = new GraphicalOutputManager(scorecard); + graphicalOutputManager.createGraphics(); + stopWatch.stop(); + logger.info("\nHTML and image creation time " + stopWatch.getFormattedDuration()); + stopWatch.start(); + + } else { + throw new MissingFileException(dataFile.getAbsolutePath()); + } + + } + + } + stopWatch.stop(); + logger.info("\nTotal execution time " + stopWatch.getFormattedTotalDuration()); + + } public Boolean getPrintSQL() { return printSQL; @@ -135,14 +247,14 @@ public void setrScriptCommand(String rScriptCommand) { this.rScriptCommand = rScriptCommand; } - public void setPwd(String pwd) { - this.pwd = pwd; - } - public String getPwd() { return pwd; } + public void setPwd(String pwd) { + this.pwd = pwd; + } + public String getUser() { return user; } @@ -159,14 +271,14 @@ public void setHost(String host) { this.host = host; } - public void setDatabaseName(String databaseName) { - this.databaseName = databaseName; - } - public String getDatabaseName() { return databaseName; } + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; + } + public WorkingFolders getWorkingFolders() { return workingFolders; } @@ -215,6 +327,10 @@ public String getDataFile() { return dataFile; } + public void setDataFile(String dataFile) { + this.dataFile = dataFile; + } + public String getAggStatDataFile() { return dataFile + ".agg_stat"; } @@ -223,11 +339,6 @@ public String getSumStatDataFile() { return dataFile + ".sum_stat"; } - public void setDataFile(String dataFile) { - this.dataFile = dataFile; - } - - public String getTitle() { return title; } @@ -240,6 +351,10 @@ public String getStat() { return stat; } + public void setStat(String stat) { + this.stat = stat; + } + public String getThresholdFile() { return thresholdFile; } @@ -248,108 +363,6 @@ public void setThresholdFile(String thresholdFile) { this.thresholdFile = thresholdFile; } - public void setStat(String stat) { - this.stat = stat; - } - - - public static void main(String[] args) throws Exception { - long nanos = System.nanoTime(); - String filename; - String dbType = "mysql"; - if (0 == args.length) { - logger.error(" Error: no arguments!!!"); - logger.info(USAGE); - - } else { - - int intArg = 0; - for (; intArg < args.length && !args[intArg].matches(".*\\.xml$"); intArg++) { - if (args[intArg].equals("mysql")) { - dbType = "mysql"; - } - } - - filename = args[intArg]; - XmlParser xmlParser = new XmlParser(); - // parce XML and init parameters - Scorecard scorecard = xmlParser.parseParameters(filename); - - //add a second model ( the same as the first one) if only one is selected - scorecard.fillValues(); - - //remove previous output with similar names - scorecard.cleanOldResults(); - - //TODO implement validation - boolean isValid = scorecard.validate(); - if (isValid) { - - DatabaseManager scorecardDbManager = null; - RscriptManager rscriptManager = null; - //create a list of each row with statistic as a key and columns - List> listRows = scorecard.getListOfEachRowWithDesc(); - - //depending on stat type init mangers - if (scorecard.getAggStat()) { - if (dbType.equals("mysql")) { - scorecardDbManager = new AggDatabaseManagerMySQL(scorecard); - } - rscriptManager = new AggRscriptManager(scorecard); - } else { - if (dbType.equals("mysql")) { - scorecardDbManager = new SumDatabaseManagerMySQL(scorecard); - } - rscriptManager = new SumRscriptManager(scorecard); - } - int rowCounter = 1; - //for each row calculate statistics in the individual cell - for (Map mapRow : listRows) { - StringBuilder logMessage = new StringBuilder(); - for (Map.Entry column : mapRow.entrySet()) { - logMessage.append(column.getKey()).append(": ").append(column.getValue().getName()) - .append(", "); - } - logger.info( - "---------------------------------------------------------------------------------------"); - logger.info("Row #" + rowCounter + ": " + logMessage); - logger.info( - "---------------------------------------------------------------------------------------"); - - try { - //get data from db and save it into file - scorecardDbManager.createDataFile(mapRow, ""); - - //use rscript and data from the db file to calculate stats and append them into the resulting file - rscriptManager.calculateStatsForRow(mapRow, ""); - logger.info( - "---------------------------------------------------------------------------------------"); - } catch (Exception e) { - logger.error(e.getMessage()); - } - rowCounter++; - } - - File dataFile = new File(scorecard.getWorkingFolders().getDataDir() - + scorecard.getDataFile()); - //if the resulting file exists - create an image and html file - if (dataFile.exists()) { - GraphicalOutputManager graphicalOutputManager = new GraphicalOutputManager(scorecard); - graphicalOutputManager.createGraphics(); - - } else { - throw new MissingFileException(dataFile.getAbsolutePath()); - } - - } - - } - long duration = System.nanoTime() - nanos; - int seconds = (int) (duration / 1000000000); - logger.info("---- Scorecard Done ---- " + seconds + " seconds"); - - } - /** * Checks if XML included only one model. If it does - add the second model that is the same as * the first one diff --git a/java/edu/ucar/metviewer/scorecard/db/DatabaseManagerMySQL.java b/java/edu/ucar/metviewer/scorecard/db/DatabaseManagerMySQL.java index a57ee557..a5a51277 100644 --- a/java/edu/ucar/metviewer/scorecard/db/DatabaseManagerMySQL.java +++ b/java/edu/ucar/metviewer/scorecard/db/DatabaseManagerMySQL.java @@ -1,6 +1,7 @@ /** - * DatabaseManagerMySQL.java Copyright UCAR (c) 2017. University Corporation for Atmospheric Research (UCAR), National Center for Atmospheric Research (NCAR), - * Research Applications Laboratory (RAL), P.O. Box 3000, Boulder, Colorado, 80307-3000, USA.Copyright UCAR (c) 2017. + * DatabaseManagerMySQL.java Copyright UCAR (c) 2017. University Corporation for Atmospheric + * Research (UCAR), National Center for Atmospheric Research (NCAR), Research Applications + * Laboratory (RAL), P.O. Box 3000, Boulder, Colorado, 80307-3000, USA.Copyright UCAR (c) 2017. */ package edu.ucar.metviewer.scorecard.db; @@ -23,6 +24,7 @@ import edu.ucar.metviewer.EmptyResultSetException; import edu.ucar.metviewer.MVUtil; +import edu.ucar.metviewer.StopWatch; import edu.ucar.metviewer.db.DatabaseInfo; import edu.ucar.metviewer.db.MysqlDatabaseManager; import edu.ucar.metviewer.scorecard.Scorecard; @@ -43,15 +45,16 @@ public abstract class DatabaseManagerMySQL extends MysqlDatabaseManager implemen private final Map> columnsDescription; private final String databaseName; - String aggStatDataFilePath; private final List fixedVars; private final Boolean printSQL; + String aggStatDataFilePath; DatabaseManagerMySQL(final Scorecard scorecard) throws SQLException { super(new DatabaseInfo(scorecard.getHost(), scorecard.getUser(), scorecard.getPwd()), - IoBuilder.forLogger(DatabaseManagerMySQL.class).setLevel(org.apache.logging.log4j.Level.INFO) - .buildPrintStream()); + IoBuilder.forLogger(DatabaseManagerMySQL.class) + .setLevel(org.apache.logging.log4j.Level.INFO) + .buildPrintStream()); fixedVars = scorecard.getFixedVars(); columnsDescription = scorecard.columnsStructure(); databaseName = scorecard.getDatabaseName(); @@ -71,12 +74,22 @@ public void createDataFile(Map map, String threadName) throws Exc logger.info(mysql); } int lastDot = aggStatDataFilePath.lastIndexOf('.'); - String thredFileName = aggStatDataFilePath.substring(0, lastDot) + threadName + aggStatDataFilePath.substring(lastDot); + String thredFileName = aggStatDataFilePath + .substring(0, lastDot) + threadName + aggStatDataFilePath + .substring(lastDot); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); try (Connection con = getConnection(databaseName); - PreparedStatement pstmt = con.prepareStatement(mysql); ResultSet res = pstmt.executeQuery(); + PreparedStatement pstmt = con.prepareStatement(mysql); + ResultSet res = pstmt.executeQuery(); FileWriter fstream = new FileWriter(new File(thredFileName), false); BufferedWriter out = new BufferedWriter(fstream)) { + stopWatch.stop(); + logger.info("Database query time " + stopWatch.getFormattedDuration()); + stopWatch.start(); printFormattedTable(res, out, "\t", false, true);// isCalc=false, isHeader=true + stopWatch.stop(); + logger.info("Save to file time " + stopWatch.getFormattedDuration()); out.flush(); out.close(); con.close(); @@ -92,8 +105,8 @@ private String getQueryForRow(Map map) throws Exception { StringBuilder whereFields = new StringBuilder(); String aggType = Util.getAggTypeForStat(Util.getStatForRow(map)); - if(aggType.isEmpty()){ - throw new Exception ("Can't find a line type for stat " + Util.getStatForRow(map)); + if (aggType.isEmpty()) { + throw new Exception("Can't find a line type for stat " + Util.getStatForRow(map)); } if (aggType.contains("nbr")) { aggType = aggType.replace("_", ""); @@ -103,18 +116,23 @@ private String getQueryForRow(Map map) throws Exception { for (Map.Entry entry : map.entrySet()) { if ("stat".equals(entry.getKey())) { - selectFields.append("'").append(entry.getValue().getName()).append("' stat_name,").append(getStatValue(table, entry.getValue().getName())).append(" 'NA' stat_value,"); + selectFields.append("'").append(entry.getValue().getName()).append("' stat_name,") + .append(getStatValue(table, entry.getValue().getName())).append(" 'NA' stat_value,"); } else { if (selectFields.indexOf(entry.getKey()) == -1) { selectFields.append(entry.getKey()).append(","); } - whereFields.append(entry.getKey()).append(" IN ('").append(entry.getValue().getName().replaceAll(",", "','")).append("') AND "); + whereFields.append(entry.getKey()).append(" IN ('") + .append(entry.getValue().getName().replaceAll(",", "','")).append("') AND "); } } for (Field fixedField : fixedVars) { StringBuilder values = new StringBuilder(); - if ("fcst_valid_beg".equals(fixedField.getName()) || "fcst_init_beg".equals(fixedField.getName())) { - whereFields.append(fixedField.getName()).append(" BETWEEN ").append("'").append(fixedField.getValues().get(0).getName()).append("' AND '").append(fixedField.getValues().get(1).getName()).append("' AND "); + if ("fcst_valid_beg".equals(fixedField.getName()) || "fcst_init_beg" + .equals(fixedField.getName())) { + whereFields.append(fixedField.getName()).append(" BETWEEN ").append("'") + .append(fixedField.getValues().get(0).getName()).append("' AND '") + .append(fixedField.getValues().get(1).getName()).append("' AND "); } else if ("init_hour".equals(fixedField.getName())) { for (Entry val : fixedField.getValues()) { values.append(Integer.valueOf(val.getName())).append(","); @@ -138,9 +156,11 @@ private String getQueryForRow(Map map) throws Exception { if (values.length() > 0) { values.deleteCharAt(values.length() - 1); } - whereFields.append(fixedField.getName()).append(" IN ('").append(values.toString().replaceAll(",", "','")).append("') AND "); + whereFields.append(fixedField.getName()).append(" IN ('") + .append(values.toString().replaceAll(",", "','")).append("') AND "); } - if (selectFields.indexOf(fixedField.getName()) == -1 && !fixedField.getName().equals("init_hour") && !fixedField.getName().equals("valid_hour")) { + if (selectFields.indexOf(fixedField.getName()) == -1 && !fixedField.getName().equals( + "init_hour") && !fixedField.getName().equals("valid_hour")) { selectFields.append(fixedField.getName()).append(","); } } @@ -158,7 +178,8 @@ private String getQueryForRow(Map map) throws Exception { if (values.length() > 0) { values.deleteCharAt(values.length() - 1); } - whereFields.append(columnEntry.getKey()).append(" IN ('").append(values.toString().replaceAll(",", "','")).append("') AND "); + whereFields.append(columnEntry.getKey()).append(" IN ('") + .append(values.toString().replaceAll(",", "','")).append("') AND "); } @@ -177,7 +198,8 @@ private String getQueryForRow(Map map) throws Exception { String mysql = "SELECT DISTINCT ld.n_thresh FROM stat_header h,line_data_pct ld WHERE " + whereFields + "ld.stat_header_id = h.stat_header_id"; try (Connection con = getConnection(databaseName); - Statement stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + Statement stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY); ResultSet resultSet = stmt.executeQuery(mysql); ) { int numPctThresh = 0; @@ -214,11 +236,13 @@ private String getQueryForRow(Map map) throws Exception { if (i != 1) { whereFields.append(" AND"); } - whereFields.append(" line_data_pct.line_data_id = ldt").append(i).append(".line_data_id AND ldt").append(i).append(".i_value = ").append(i); + whereFields.append(" line_data_pct.line_data_id = ldt").append(i) + .append(".line_data_id AND ldt").append(i).append(".i_value = ").append(i); } whereFields.append(" AND stat_header.stat_header_id = line_data_pct.stat_header_id"); } else { - whereFields.append("stat_header.stat_header_id = ").append(table).append(".stat_header_id;"); + whereFields.append("stat_header.stat_header_id = ").append(table) + .append(".stat_header_id;"); } return "SELECT " + selectFields + " FROM stat_header," + table + " WHERE " + whereFields; } else { @@ -227,7 +251,9 @@ private String getQueryForRow(Map map) throws Exception { } } - private void printFormattedTable(ResultSet res, BufferedWriter bufferedWriter, String delim, boolean isCalc, boolean isHeader) { + private void printFormattedTable( + ResultSet res, BufferedWriter bufferedWriter, String delim, boolean isCalc, + boolean isHeader) { try { ResultSetMetaData met = res.getMetaData(); @@ -287,9 +313,9 @@ private void printFormattedTable(ResultSet res, BufferedWriter bufferedWriter, S } } } - bufferedWriter.write(line); - bufferedWriter.write(System.getProperty("line.separator")); - intLine++; + bufferedWriter.write(line); + bufferedWriter.write(System.getProperty("line.separator")); + intLine++; } @@ -299,7 +325,8 @@ private void printFormattedTable(ResultSet res, BufferedWriter bufferedWriter, S } } catch (Exception e) { - logger.error(" ** ERROR: Caught " + e.getClass() + " in printFormattedTable(ResultSet res): " + e.getMessage()); + logger.error( + " ** ERROR: Caught " + e.getClass() + " in printFormattedTable(ResultSet res): " + e.getMessage()); } } diff --git a/java/edu/ucar/metviewer/scorecard/rscript/AggRscriptManager.java b/java/edu/ucar/metviewer/scorecard/rscript/AggRscriptManager.java index b2b556dd..3e989b70 100644 --- a/java/edu/ucar/metviewer/scorecard/rscript/AggRscriptManager.java +++ b/java/edu/ucar/metviewer/scorecard/rscript/AggRscriptManager.java @@ -136,19 +136,19 @@ public void calculateStatsForRow(Map mapRow, String threadName) { MVUtil.populateTemplateFile(aggStatTemplFile, thredInfoFileName, tableAggStatInfo); StopWatch stopWatch = new StopWatch(); stopWatch.start(); - printStream.print("Running '" + rScriptCommand + " " + aggStatTemplScript + "'"); + printStream.println("Running " + rScriptCommand + " " + aggStatTemplScript ); RscriptResponse rscriptResponse = MVUtil.runRscript(rScriptCommand, aggStatTemplScript, new String[]{thredInfoFileName}); stopWatch.stop(); if (rscriptResponse.getInfoMessage() != null) { - printStream.print(rscriptResponse.getInfoMessage()); + printStream.println(rscriptResponse.getInfoMessage()); } if (rscriptResponse.getErrorMessage() != null) { - printStream.print(rscriptResponse.getErrorMessage()); + printStream.println(rscriptResponse.getErrorMessage()); } - printStream.print("Rscript time " + stopWatch.getFormattedTotalDuration()); + printStream.println("Rscript time " + stopWatch.getFormattedTotalDuration()); } catch (Exception e) { logger.error(e); } diff --git a/java/edu/ucar/metviewer/scorecard/rscript/CalcRscriptManager.java b/java/edu/ucar/metviewer/scorecard/rscript/CalcRscriptManager.java index 6c3ca4a6..025f5fd2 100644 --- a/java/edu/ucar/metviewer/scorecard/rscript/CalcRscriptManager.java +++ b/java/edu/ucar/metviewer/scorecard/rscript/CalcRscriptManager.java @@ -173,17 +173,17 @@ public void calculateStatsForRow(Map mapRow, String threadName) { // run agg_stat/ StopWatch stopWatch = new StopWatch(); stopWatch.start(); - printStream.print("Running '" + rScriptCommand + " " + strRFile + "'"); + printStream.println("Running " + rScriptCommand + " " + strRFile ); RscriptResponse rscriptResponse = MVUtil.runRscript(rScriptCommand, strRFile); stopWatch.stop(); if (rscriptResponse.getInfoMessage() != null) { - printStream.print(rscriptResponse.getInfoMessage()); + printStream.println(rscriptResponse.getInfoMessage()); } if (rscriptResponse.getErrorMessage() != null) { - printStream.print(rscriptResponse.getErrorMessage()); + printStream.println(rscriptResponse.getErrorMessage()); } - printStream.print("Rscript time " + stopWatch.getFormattedTotalDuration()); + printStream.println("Rscript time " + stopWatch.getFormattedTotalDuration()); } catch (Exception e) { logger.error(e); } diff --git a/java/edu/ucar/metviewer/scorecard/rscript/SumRscriptManager.java b/java/edu/ucar/metviewer/scorecard/rscript/SumRscriptManager.java index a9ce516a..27a89e42 100644 --- a/java/edu/ucar/metviewer/scorecard/rscript/SumRscriptManager.java +++ b/java/edu/ucar/metviewer/scorecard/rscript/SumRscriptManager.java @@ -160,7 +160,7 @@ public void calculateStatsForRow(Map mapRow, String threadName) { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - printStream.print("Running '" + rScriptCommand + " " + sumStatTemplScript + "'"); + printStream.println("Running " + rScriptCommand + " " + sumStatTemplScript ); RscriptResponse rscriptResponse = MVUtil.runRscript(rScriptCommand, sumStatTemplScript, @@ -168,12 +168,12 @@ public void calculateStatsForRow(Map mapRow, String threadName) { stopWatch.stop(); if (rscriptResponse.getInfoMessage() != null) { - printStream.print(rscriptResponse.getInfoMessage()); + printStream.println(rscriptResponse.getInfoMessage()); } if (rscriptResponse.getErrorMessage() != null) { - printStream.print(rscriptResponse.getErrorMessage()); + printStream.println(rscriptResponse.getErrorMessage()); } - printStream.print("Rscript time " + stopWatch.getFormattedTotalDuration()); + printStream.println("Rscript time " + stopWatch.getFormattedTotalDuration()); } catch (Exception e) { logger.error(e); logger.error(e); @@ -199,18 +199,18 @@ public void calculateStatsForRow(Map mapRow, String threadName) { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - printStream.print("Running '" + rScriptCommand + " " + strRFile + "'"); + printStream.println("Running " + rScriptCommand + " " + strRFile); RscriptResponse rscriptResponse = MVUtil.runRscript(rScriptCommand, strRFile); stopWatch.stop(); if (rscriptResponse.getInfoMessage() != null) { - printStream.print(rscriptResponse.getInfoMessage()); + printStream.println(rscriptResponse.getInfoMessage()); } if (rscriptResponse.getErrorMessage() != null) { - printStream.print(rscriptResponse.getErrorMessage()); + printStream.println(rscriptResponse.getErrorMessage()); } - printStream.print("Rscript time " + stopWatch.getFormattedTotalDuration()); + printStream.println("Rscript time " + stopWatch.getFormattedTotalDuration()); } catch (Exception e) { logger.error(e); logger.error(e); diff --git a/java/edu/ucar/metviewer/test/CreatePlotBatchTest.java b/java/edu/ucar/metviewer/test/CreatePlotBatchTest.java index 07d1f3be..3a1e26df 100644 --- a/java/edu/ucar/metviewer/test/CreatePlotBatchTest.java +++ b/java/edu/ucar/metviewer/test/CreatePlotBatchTest.java @@ -70,7 +70,7 @@ public static Collection data() throws Exception { } @Before - public void runScript() { + public void runScript() throws Exception { if (System.getProperty("captureCreatedImages") != null) { // if we are capturing images we have to clean the working dir each run // because there are potentially multiple images created for each plotType diff --git a/java/edu/ucar/metviewer/test/MVBatchTest.java b/java/edu/ucar/metviewer/test/MVBatchTest.java index 3c01399b..8e611c75 100644 --- a/java/edu/ucar/metviewer/test/MVBatchTest.java +++ b/java/edu/ucar/metviewer/test/MVBatchTest.java @@ -228,7 +228,7 @@ public void isSetVerbose() { **/ @Test - public void seriesDiffGrouping() { + public void seriesDiffGrouping() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -241,7 +241,7 @@ public void seriesDiffGrouping() { } @Test - public void handSelectedDates() { + public void handSelectedDates() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -254,7 +254,7 @@ public void handSelectedDates() { } @Test - public void tylor() { + public void tylor() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -267,7 +267,7 @@ public void tylor() { } @Test - public void rhist() { + public void rhist() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -280,7 +280,7 @@ public void rhist() { } @Test - public void phist() { + public void phist() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -293,7 +293,7 @@ public void phist() { } @Test - public void servlet() { + public void servlet() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -306,7 +306,7 @@ public void servlet() { } @Test - public void rely() { + public void rely() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -319,7 +319,7 @@ public void rely() { } @Test - public void series_sum_stat() { + public void series_sum_stat() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); @@ -332,7 +332,7 @@ public void series_sum_stat() { } @Test - public void plot_afwa_thresh() { + public void plot_afwa_thresh() throws Exception { List argsList = new ArrayList<>(); argsList.add("-printSql"); argsList.add("mysql"); diff --git a/java/edu/ucar/metviewer/test/util/TestUtil.java b/java/edu/ucar/metviewer/test/util/TestUtil.java index 72bccedd..abcd5240 100644 --- a/java/edu/ucar/metviewer/test/util/TestUtil.java +++ b/java/edu/ucar/metviewer/test/util/TestUtil.java @@ -194,7 +194,7 @@ private TestUtil() { throw new IllegalAccessError("Utility class"); } - public static void runBatch(String testDataDir, String plotType) { + public static void runBatch(String testDataDir, String plotType) throws Exception { List argsList = new ArrayList<>(); if (list) { argsList.add("-list"); diff --git a/webapp/metviewer/plotJSP/series.jsp b/webapp/metviewer/plotJSP/series.jsp index d5078a3d..b7dfd733 100644 --- a/webapp/metviewer/plotJSP/series.jsp +++ b/webapp/metviewer/plotJSP/series.jsp @@ -394,7 +394,7 @@ $(this).prop("checked", true); $('#' + $(this).val()).show(); if($(this).val() === "revision_statistics"){ - $('#indy_var').val("fcst_valid_beg"); + //$('#indy_var').val("fcst_valid_beg"); $('#date_period_button').css("display", "block"); $("#indy_var").multiselect("refresh"); }