Skip to content

Commit

Permalink
add new and rearrange existing execution time measurements and log me…
Browse files Browse the repository at this point in the history
…ssages #86
  • Loading branch information
TatianaBurek committed Nov 6, 2018
1 parent 55bbbb4 commit 5c79af4
Show file tree
Hide file tree
Showing 27 changed files with 469 additions and 285 deletions.
16 changes: 10 additions & 6 deletions java/edu/ucar/metviewer/MVBatch.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import java.io.File;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
Expand All @@ -29,7 +28,7 @@ public class MVBatch {


private PrintStream printStream;
private PrintWriter printStreamSql;
private PrintStream printStreamSql;
private PrintStream printStreamEr;


Expand All @@ -46,7 +45,7 @@ public class MVBatch {


public MVBatch(
PrintStream log, PrintWriter printStreamSql, PrintStream printStreamEr,
PrintStream log, PrintStream printStreamSql, PrintStream printStreamEr,
AppDatabaseManager manager) {
super();

Expand All @@ -61,7 +60,7 @@ public MVBatch() {
this(IoBuilder.forLogger(MVBatch.class).setLevel(org.apache.logging.log4j.Level.INFO)
.buildPrintStream(),
IoBuilder.forLogger(MVBatch.class).setLevel(org.apache.logging.log4j.Level.INFO)
.buildPrintWriter(),
.buildPrintStream(),
IoBuilder.forLogger().setLevel(org.apache.logging.log4j.Level.INFO)
.setMarker(new MarkerManager.Log4jMarker("ERROR"))
.buildPrintStream(), null);
Expand Down Expand Up @@ -150,7 +149,7 @@ public void setRworkFolder(String rworkFolder) {
this.rworkFolder = rworkFolder;
}

public PrintWriter getPrintStreamSql() {
public PrintStream getPrintStreamSql() {
return printStreamSql;
}

Expand Down Expand Up @@ -212,7 +211,10 @@ public static String getUsage() {
+ "specification to run\n";
}

public static void main(String[] argv) {
public static void main(String[] argv) throws Exception {

StopWatch stopWatch = new StopWatch();
stopWatch.start();
MVBatch bat = new MVBatch();

bat.print("---- MVBatch ----\n");
Expand Down Expand Up @@ -393,6 +395,8 @@ public static void main(String[] argv) {
}

bat.print("---- MVBatch Done ----");
bat.print("\nTotal execution time " + stopWatch.getFormattedTotalDuration());

}

}
2 changes: 1 addition & 1 deletion java/edu/ucar/metviewer/MVLoad.java
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public static void main(String[] argv) {
MysqlLoadDatabaseManager.class)
.setLevel(
org.apache.logging.log4j.Level.INFO)
.buildPrintWriter());
.buildPrintStream());
}

verbose = job.getVerbose();
Expand Down
18 changes: 11 additions & 7 deletions java/edu/ucar/metviewer/MVServlet.java
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,8 @@ public int compare(String o1, String o2) {
public static String handlePlot(String strRequest, String[] currentDBName) throws Exception {

// extract the plot xml from the request
StopWatch stopWatch = new StopWatch();
stopWatch.start();
String strPlotXML = strRequest;
strPlotXML = strPlotXML.substring(strPlotXML.indexOf("</db_con>") + 9);
strPlotXML = strPlotXML.substring(0, strPlotXML.indexOf("</request>"));
Expand Down Expand Up @@ -463,7 +465,7 @@ public static String handlePlot(String strRequest, String[] currentDBName) throw

ByteArrayOutputStream logSql = null;
ByteArrayOutputStream logError = null;
PrintWriter printStreamSql = null;
PrintStream printStreamSql = null;
PrintStream printStreamError = null;


Expand All @@ -473,7 +475,7 @@ public static String handlePlot(String strRequest, String[] currentDBName) throw
logSql = new ByteArrayOutputStream();
logError = new ByteArrayOutputStream();
printStream = new PrintStream(log);
printStreamSql = new PrintWriter(logSql);
printStreamSql = new PrintStream(logSql);
printStreamError = new PrintStream(logError);
MVBatch mvBatch = new MVBatch(printStream, printStreamSql, printStreamError, databaseManager);
// configure the batch engine and run the job
Expand All @@ -495,7 +497,10 @@ public static String handlePlot(String strRequest, String[] currentDBName) throw
}
logSql.reset();

stopWatch.stop();
mvBatch.getPrintStream().println("\nTotal execution time " + stopWatch.getFormattedTotalDuration());
String strPlotterOutput = log.toString();

try (FileWriter fileWriter = new FileWriter(plotXml + DELIMITER + strPlotPrefix + ".log")) {
fileWriter.write(strPlotterOutput);
}
Expand All @@ -513,6 +518,7 @@ public static String handlePlot(String strRequest, String[] currentDBName) throw
throw new Exception("query returned no data");
}


} catch (Exception e) {

errorStream.print(
Expand Down Expand Up @@ -559,9 +565,7 @@ public static String handlePlot(String strRequest, String[] currentDBName) throw
+ "</r_error></response>" : "</response>");
}

private static void runTargetedJob(
MVPlotJob job,
MVBatch bat) throws Exception {
private static void runTargetedJob(MVPlotJob job, MVBatch bat) throws Exception {
JobManager jobManager;
switch (job.getPlotTmpl()) {
case "rhist.R_tmpl":
Expand Down Expand Up @@ -721,10 +725,10 @@ public void init() throws ServletException {
databaseInfo.setHost(bundle.getString("db.host"));
databaseInfo.setUser(bundle.getString("db.user"));
databaseInfo.setPassword(bundle.getString("db.password"));
PrintWriter printStreamSql = null;
PrintStream printStreamSql = null;
ByteArrayOutputStream logSql = null;
logSql = new ByteArrayOutputStream();
printStreamSql = new PrintWriter(logSql);
printStreamSql = new PrintStream(logSql);

if (bundle.getString("db.managementSystem").equals("mysql")) {
databaseManager = new MysqlAppDatabaseManager(databaseInfo, printStreamSql);
Expand Down
39 changes: 17 additions & 22 deletions java/edu/ucar/metviewer/MVUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -1521,38 +1521,36 @@ public static String getStatTable(final String strStat) {
}


public static boolean runRscript(
public static RscriptResponse runRscript(
final String rscript,
final String script,
final PrintStream printStream) throws Exception {
return runRscript(rscript, script, new String[]{}, printStream);
final String script
) throws Exception {
return runRscript(rscript, script, new String[]{});
}


/**
* Run the input R script named r using the Rscript command. The output and error output will be
* written to standard output.
*
* @param rscript Rscript command
* @param script R script to run
* @param rscriptCommand Rscript command
* @param scriptName R script to run
* @param args (optional) Arguments to pass to the R script
* @throws Exception
*/
public static boolean runRscript(
final String rscript, final String script,
final String[] args,
final PrintStream printStream) throws Exception {
public static RscriptResponse runRscript(
final String rscriptCommand, final String scriptName,
final String[] args
) throws Exception {

RscriptResponse rscriptResponse = new RscriptResponse();

// build a list of arguments
StringBuilder strArgList = new StringBuilder();
for (int i = 0; null != args && i < args.length; i++) {
strArgList.append(' ').append(args[i]);
}

// run the R script and wait for it to complete
printStream.println("\nRunning '" + rscript + " " + script + "'");


Process proc = null;
InputStreamReader inputStreamReader = null;
InputStreamReader errorInputStreamReader = null;
Expand All @@ -1568,7 +1566,7 @@ public static boolean runRscript(

try {

proc = Runtime.getRuntime().exec(rscript + " " + script + strArgList);
proc = Runtime.getRuntime().exec(rscriptCommand + " " + scriptName + strArgList);
inputStreamReader = new InputStreamReader(proc.getInputStream());
errorInputStreamReader = new InputStreamReader(proc.getErrorStream());

Expand Down Expand Up @@ -1612,17 +1610,14 @@ public static boolean runRscript(


if (strProcStd.length() > 0) {
printStream.println(
"\n==== Start Rscript output ====\n" + strProcStd + "==== End Rscript output ====\n");
rscriptResponse.setInfoMessage("==== Start Rscript output ====\n" + strProcStd + "==== End Rscript output ====");
}

if (strProcErr.length() > 0) {
printStream.println(
"\n==== Start Rscript error ====\n" + strProcErr + "==== End Rscript error ====\n");
rscriptResponse.setErrorMessage("==== Start Rscript error ====\n" + strProcErr + "==== End Rscript error ====");
}

// return the success flag
return 0 == intExitStatus;
rscriptResponse.setSuccess(0 == intExitStatus);
return rscriptResponse;
}

/**
Expand Down
80 changes: 80 additions & 0 deletions java/edu/ucar/metviewer/StopWatch.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/**
* StopWatch.java Copyright UCAR (c) 2018. University Corporation for Atmospheric Research (UCAR),
* National Center for Atmospheric Research (NCAR), Research Applications Laboratory (RAL), P.O. Box
* 3000, Boulder, Colorado, 80307-3000, USA.Copyright UCAR (c) 2018.
*/

package edu.ucar.metviewer;

/**
* @author : tatiana $
* @version : 1.0 : 05/11/18 15:02 $
*/
public class StopWatch {

private Long startTime;
private Long stopTime;
private Long duration;
private Long totalDuration;

public static void main(String args[]) throws Exception {
StopWatch stopWatch = new StopWatch();
stopWatch.start();

for (int i = 0; i < 10000000; i++) {
Object obj = new Object();
}
stopWatch.stop();
System.out.println(stopWatch.getFormattedDuration());

stopWatch.start();
for (int i = 0; i < 10000000; i++) {
Object obj = new Object();
}
stopWatch.stop();
System.out.println(stopWatch.getFormattedDuration());
System.out.println(stopWatch.getFormattedTotalDuration());
}


public void start() {
startTime = System.nanoTime();
stopTime = null;

duration = null;

}

public void stop() throws Exception {
if (startTime == null) {
throw new Exception("Start time was never set");
}
stopTime = System.nanoTime();
duration = stopTime - startTime;
if (totalDuration == null) {
totalDuration = duration;
} else {
totalDuration = totalDuration + duration;
}

}

public Long getTotalDuration() {
return totalDuration;
}

public String getFormattedDuration() throws Exception {
if (duration == null) {
throw new Exception("Start or stop time was never set");
}
return MVUtil.formatTimeSpan(duration / 1000000);
}

public String getFormattedTotalDuration() throws Exception {
if (totalDuration == null) {
throw new Exception("Start or stop time was never set");
}
return MVUtil.formatTimeSpan(totalDuration / 1000000);
}

}
34 changes: 17 additions & 17 deletions java/edu/ucar/metviewer/db/AppDatabaseManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
package edu.ucar.metviewer.db;

import java.io.PrintStream;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Map;
Expand All @@ -16,6 +15,7 @@
import edu.ucar.metviewer.MVNode;
import edu.ucar.metviewer.MVOrderedMap;
import edu.ucar.metviewer.MVPlotJob;
import edu.ucar.metviewer.RscriptResponse;

/**
* @author : tatiana $
Expand All @@ -24,31 +24,31 @@
public interface AppDatabaseManager {

List<String> buildPlotSql(
MVPlotJob job, MVOrderedMap mapPlotFixPerm,
PrintWriter printStreamSql) throws Exception;
MVPlotJob job, MVOrderedMap mapPlotFixPerm,
PrintStream printStreamSql) throws Exception;

List<String> buildPlotModeEventEqualizeSql(
MVPlotJob job, MVOrderedMap mapPlotFixPerm,
MVOrderedMap mapPlotFixVal) throws Exception;
MVPlotJob job, MVOrderedMap mapPlotFixPerm,
MVOrderedMap mapPlotFixVal) throws Exception;

boolean executeQueriesAndSaveToFile(
List<String> queries, String fileName, boolean isCalc,
String currentDBName, boolean isNewFile) throws Exception;
RscriptResponse executeQueriesAndSaveToFile(
List<String> queries, String fileName, boolean isCalc,
String currentDBName, boolean isNewFile) throws Exception;

String buildAndExecuteQueriesForHistJob(
MVPlotJob job, String strDataFile,
MVOrderedMap listPlotFixPerm, PrintStream printStream,
PrintWriter printStreamSql) throws Exception;
MVPlotJob job, String strDataFile,
MVOrderedMap listPlotFixPerm, PrintStream printStream,
PrintStream printStreamSql) throws Exception;

int buildAndExecuteQueriesForRocRelyJob(
MVPlotJob job, String strDataFile,
MVOrderedMap listPlotFixPerm, PrintStream printStream,
PrintWriter printStreamSql) throws Exception;
MVPlotJob job, String strDataFile,
MVOrderedMap listPlotFixPerm, PrintStream printStream,
PrintStream printStreamSql) throws Exception;

int buildAndExecuteQueriesForEclvJob(
MVPlotJob job, String strDataFile,
MVOrderedMap listPlotFixPerm, PrintStream printStream,
PrintWriter printStreamSql) throws Exception;
MVPlotJob job, String strDataFile,
MVOrderedMap listPlotFixPerm, PrintStream printStream,
PrintStream printStreamSql) throws Exception;

DatabaseInfo getDatabaseInfo();

Expand Down
Loading

0 comments on commit 5c79af4

Please sign in to comment.