diff --git a/build.xml b/build.xml index 8ba837e1..cbb36375 100644 --- a/build.xml +++ b/build.xml @@ -18,7 +18,7 @@ e.g. ant all -Dbuild.property.file=fullpath/buildMYSQL-model-vxtest.properties or - ant all -Dbuild.property.file=fullpath/buildCB-model-vxtest.properties + ant all -Dbuild.property.file=fullpath/build-model-vxtest.properties IMPORTANT! save your custom propertyfiles somewhere safe use -Dlog4j2.file=fullpath/customlog4j2.xml to provide a custom log4j2.xml file to replace @@ -26,7 +26,7 @@ use -Ddb.management.system=mariadb or - -Ddb.management.system=cb + -Ddb.management.system=mariadb to override the db.management.system property use -Dcontext.path.suffix to define a context path suffix. @@ -48,14 +48,14 @@ e.g -Dpython.env.path=/d1/python/METviewer_py3.6.3/ example build command... - ant -Dbuild.properties.file=$HOME/METViewer/webapp/metviewer/WEB-INF/classes/buildCB-model-vxtest.properties \ - -Dlog4j2.file=$HOME/METViewer/webapp/metviewer/WEB-INF/classes/buildCB-model-vxtest.log4j2.xml \ - -Ddb.management.system=cb -Dmetcalcpy.path=/d1/python/METcalcpy/ -Dmetplotpy.path=/d1/python/METplotpy/\ - -Dmetdataio.path=/d1/python/METdataio/ -Dpython.env.path=/d1/python/METviewer_py3.6.3/ -Dcontext.path.suffix="-cb" clean all + ant -Dbuild.properties.file=$HOME/METViewer/webapp/metviewer/WEB-INF/classes/build-model-vxtest.properties \ + -Dlog4j2.file=$HOME/METViewer/webapp/metviewer/WEB-INF/classes/build-model-vxtest.log4j2.xml \ + -Ddb.management.system=mariadb -Dmetcalcpy.path=/d1/python/METcalcpy/ -Dmetplotpy.path=/d1/python/METplotpy/\ + -Dmetdataio.path=/d1/python/METdataio/ -Dpython.env.path=/d1/python/METviewer_py3.6.3/ -Dcontext.path.suffix="-mariadb" clean all - will produce a war file named metviewer-cb.war with jar files appropriate for couchbase and the properties from the - $HOME/METViewer/webapp/metviewer/WEB-INF/classes/buildCB-model-vxtest.properties file, and a tar file - named metviewer-test.tar also with libraries appropriate for couchbase. + will produce a war file .war with jar files appropriate for mariadb and the properties from the + $HOME/METViewer/webapp/metviewer/WEB-INF/classes/build-model-vxtest.properties file, and a tar file + named metviewer-test.tar also with libraries appropriate for mariadb. --> @@ -109,53 +109,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -299,9 +255,7 @@ - - - + @@ -313,18 +267,12 @@ - - - - - - @@ -335,33 +283,21 @@ - - - - - - - - - - - - + path elements specified by refid i.e. mysql, mysql-all--> @@ -411,7 +347,6 @@ - @@ -441,18 +376,6 @@ - - - - - - - - - - - - @@ -482,7 +405,6 @@ - @@ -512,18 +434,6 @@ - - - - - - - - - - - - @@ -666,17 +576,7 @@ - - - - - - - - - - - + @@ -710,17 +610,7 @@ - - - - - - - - - - - + diff --git a/java/edu/ucar/metviewer/db/DatabaseManager.java b/java/edu/ucar/metviewer/db/DatabaseManager.java index f69b56f3..8d483b8a 100644 --- a/java/edu/ucar/metviewer/db/DatabaseManager.java +++ b/java/edu/ucar/metviewer/db/DatabaseManager.java @@ -29,7 +29,6 @@ public abstract class DatabaseManager { public static final String MYSQL = "mysql"; public static final String MARIADB = "mariadb"; - public static final String CB = "cb"; public static final String AURORA = "aurora"; public DatabaseManager(DatabaseInfo databaseInfo) { @@ -71,13 +70,7 @@ public static DatabaseManager getLoadManager( String.class) .newInstance(databaseInfo, password); break; - case CB: - databaseManager = (DatabaseManager) Class.forName( - "edu.ucar.metviewer.db.couchbase.CBLoadDatabaseManager") - .getDeclaredConstructor(DatabaseInfo.class, - String.class) - .newInstance(databaseInfo, password); - break; + case MARIADB: databaseManager = (DatabaseManager) Class.forName( "edu.ucar.metviewer.db.mariadb.MariaDbLoadDatabaseManager") @@ -107,11 +100,7 @@ public static DatabaseManager getAppManager( DatabaseInfo databaseInfo = new DatabaseInfo(host, user); databaseInfo.setDbName(database); DatabaseManager databaseManager; - // NOTE: the contstuctor using reflection is to enable us to build - // a mysql version without a couchbase dependency - // and a CB version without a mysql dependency. - // The build.xml will conditionally leave out the unwanted - // dependencies (jar files) based on db.management.system + switch (dbType) { case MYSQL: databaseManager = (DatabaseManager) Class.forName( @@ -120,13 +109,7 @@ public static DatabaseManager getAppManager( String.class) .newInstance(databaseInfo, password); break; - case CB: - databaseManager = (DatabaseManager) Class.forName( - "edu.ucar.metviewer.db.couchbase.CBAppDatabaseManager") - .getDeclaredConstructor(DatabaseInfo.class, - String.class) - .newInstance(databaseInfo, password); - break; + case MARIADB: databaseManager = (DatabaseManager) Class.forName( "edu.ucar.metviewer.db.mariadb.MariaDbAppDatabaseManager") diff --git a/java/edu/ucar/metviewer/db/couchbase/CBAppDatabaseManager.java b/java/edu/ucar/metviewer/db/couchbase/CBAppDatabaseManager.java deleted file mode 100644 index a568903b..00000000 --- a/java/edu/ucar/metviewer/db/couchbase/CBAppDatabaseManager.java +++ /dev/null @@ -1,3192 +0,0 @@ -/** - * CBAppDatabaseManager.java Copyright UCAR (c) 2017. University Corporation for Atmospheric - * Research (UCAR), National Center for Atmospheric Research (NCAR), Research Applications - * Laboratory (RAL), P.O. Box 3000, Boulder, Colorado, 80307-3000, USA.Copyright UCAR (c) 2017. - */ - -package edu.ucar.metviewer.db.couchbase; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.PrintStream; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.couchbase.client.core.CouchbaseException; -import com.couchbase.client.java.document.json.JsonObject; -import com.couchbase.client.java.query.N1qlQuery; -import com.couchbase.client.java.query.N1qlQueryResult; -import com.couchbase.client.java.query.N1qlQueryRow; -import edu.ucar.metviewer.BuildMysqlQueryStrings; -import edu.ucar.metviewer.MVNode; -import edu.ucar.metviewer.MVOrderedMap; -import edu.ucar.metviewer.MVPlotJob; -import edu.ucar.metviewer.MVUtil; -import edu.ucar.metviewer.MvResponse; -import edu.ucar.metviewer.StopWatch; -import edu.ucar.metviewer.StopWatchException; -import edu.ucar.metviewer.ValidationException; -import edu.ucar.metviewer.db.AppDatabaseManager; -import edu.ucar.metviewer.db.DatabaseInfo; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * @author : tatiana $ - * @version : 1.0 : 19/05/17 12:42 $ - */ -public class CBAppDatabaseManager extends CBDatabaseManager implements AppDatabaseManager { - - - private static final Logger logger = LogManager.getLogger("CBAppDatabaseManager"); - private final List statHeaderSqlType = new ArrayList<>(); - private final List modeHeaderSqlType = new ArrayList<>(); - private final Map mtd3dSingleStatField = new HashMap<>(); - - - private final List mtdHeaderSqlType = new ArrayList<>(); - - private StopWatch cbStopWatch = new StopWatch(); - - public CBAppDatabaseManager(DatabaseInfo databaseInfo, String password) throws Exception { - super(databaseInfo, password); - statHeaderSqlType.add("model"); - statHeaderSqlType.add("descr"); - statHeaderSqlType.add("fcst_lead"); - statHeaderSqlType.add("fcst_valid_beg"); - statHeaderSqlType.add("fcst_valid_end"); - statHeaderSqlType.add("fcst_init_beg"); - statHeaderSqlType.add("obs_lead"); - statHeaderSqlType.add("obs_valid_beg"); - statHeaderSqlType.add("obs_valid_end"); - statHeaderSqlType.add("init_hour"); - statHeaderSqlType.add("valid_hour"); - statHeaderSqlType.add("fcst_var"); - statHeaderSqlType.add("fcst_lev"); - statHeaderSqlType.add("obs_var"); - statHeaderSqlType.add("obs_lev"); - statHeaderSqlType.add("obtype"); - statHeaderSqlType.add("vx_mask"); - statHeaderSqlType.add("interp_mthd"); - statHeaderSqlType.add("interp_pnts"); - statHeaderSqlType.add("fcst_thresh"); - statHeaderSqlType.add("obs_thresh"); - statHeaderSqlType.add("fcst_perc"); - statHeaderSqlType.add("obs_perc"); - - modeHeaderSqlType.add("model"); - modeHeaderSqlType.add("descr"); - modeHeaderSqlType.add("fcst_lead"); - modeHeaderSqlType.add("fcst_valid"); - modeHeaderSqlType.add("fcst_accum"); - modeHeaderSqlType.add("fcst_init"); - modeHeaderSqlType.add("obs_lead"); - modeHeaderSqlType.add("obs_valid"); - modeHeaderSqlType.add("obs_accum"); - modeHeaderSqlType.add("init_hour"); - modeHeaderSqlType.add("valid_hour"); - modeHeaderSqlType.add("fcst_rad"); - modeHeaderSqlType.add("fcst_thr"); - modeHeaderSqlType.add("obs_rad"); - modeHeaderSqlType.add("obs_thr"); - modeHeaderSqlType.add("fcst_var"); - modeHeaderSqlType.add("fcst_lev"); - modeHeaderSqlType.add("obs_var"); - modeHeaderSqlType.add("obs_lev"); - - - mtd3dSingleStatField.put("3D_CENTROID_X", "centroid_x"); - mtd3dSingleStatField.put("3D_CENTROID_Y", "centroid_y"); - mtd3dSingleStatField.put("3D_CENTROID_T", "centroid_t"); - mtd3dSingleStatField.put("3D_CENTROID_LAT", "centroid_lat"); - mtd3dSingleStatField.put("3D_CENTROID_LON", "centroid_lon"); - mtd3dSingleStatField.put("3D_X_DOT", "x_dot"); - mtd3dSingleStatField.put("3D_Y_DOT", "y_dot"); - mtd3dSingleStatField.put("3D_AXIS_ANG", "axis_ang"); - mtd3dSingleStatField.put("3D_VOLUME", "volume"); - mtd3dSingleStatField.put("3D_START_TIME", "start_time"); - mtd3dSingleStatField.put("3D_END_TIME", "end_time"); - mtd3dSingleStatField.put("3D_DURATION", "end_time-start_time"); - mtd3dSingleStatField.put("3D_CDIST_TRAVELLED", "cdist_travelled"); - mtd3dSingleStatField.put("3D_INTENSITY_10", "intensity_10"); - mtd3dSingleStatField.put("3D_INTENSITY_25", "intensity_25"); - mtd3dSingleStatField.put("3D_INTENSITY_50", "intensity_50"); - mtd3dSingleStatField.put("3D_INTENSITY_75", "intensity_75"); - mtd3dSingleStatField.put("3D_INTENSITY_90", "intensity_90"); - - - mtdHeaderSqlType.add("model"); - mtdHeaderSqlType.add("descr"); - mtdHeaderSqlType.add("fcst_lead"); - mtdHeaderSqlType.add("fcst_valid"); - mtdHeaderSqlType.add("fcst_accum"); - mtdHeaderSqlType.add("fcst_init"); - mtdHeaderSqlType.add("obs_lead"); - mtdHeaderSqlType.add("t_delta"); - mtdHeaderSqlType.add("fcst_rad"); - mtdHeaderSqlType.add("fcst_thr"); - mtdHeaderSqlType.add("fcst_var"); - mtdHeaderSqlType.add("fcst_lev"); - mtdHeaderSqlType.add("obs_var"); - mtdHeaderSqlType.add("obs_lev"); - - } - - - @Override - public List getListStat(String strFcstVar, String[] currentDBName) { - List listStatName = new ArrayList<>(); - - N1qlQueryResult queryResult = null; - List queryList = null; - String queryString = ""; - String dbList = "["; - - for (String cdbName : currentDBName) { - if (dbList.length() > 1) { - dbList += ", "; - } - dbList += "\'" + cdbName + "\'"; - } - dbList += "]"; - - // find the line_data documents where the header has the selected forecast variable - queryString = "SELECT DISTINCT ld.line_type" - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld on ld.header_id = meta(h).id" - + " WHERE ld.type = \'line\' AND h.type = \'header\' AND h.header_type = \'stat\' " - + " AND h.fcst_var = \'" + strFcstVar + "\' " - + " AND h.dbname IN " + dbList; - - try { - cbStopWatch.start(); - queryResult = getBucket().query(N1qlQuery.simple(queryString)); - cbStopWatch.stop(); - logger.error("getListStat QUERYSTRING: " + queryString + "\nDuration: " + cbStopWatch - .getFormattedTotalDuration()); - boolean boolCnt = false; - boolean boolCts = false; - boolean boolVcnt = false; - for (N1qlQueryRow row : queryResult) { - // System.out.println(row); - switch (row.value().get("line_type").toString()) { - case "cnt": - case "sl1l2": - case "sal1l2": - case "grad": - if (!boolCnt) { - listStatName.addAll(MVUtil.statsCnt.keySet()); - } - boolCnt = true; - break; - case "cts": - case "ctc": - if (!boolCts) { - listStatName.addAll(MVUtil.statsCts.keySet()); - } - boolCts = true; - break; - case "nbrcnt": - listStatName.addAll(MVUtil.statsNbrcnt.keySet()); - break; - case "nbrcts": - listStatName.addAll(MVUtil.statsNbrcts.keySet()); - break; - case "pstd": - listStatName.addAll(MVUtil.statsPstd.keySet()); - break; - case "mcts": - listStatName.addAll(MVUtil.statsMcts.keySet()); - break; - case "rhist": - listStatName.addAll(MVUtil.statsRhist.keySet()); - break; - case "vl1l2": - //case val1l2: - listStatName.addAll(MVUtil.statsVl1l2.keySet()); - listStatName.addAll(MVUtil.statsVcnt.keySet()); - boolVcnt = true; - break; - case "phist": - listStatName.addAll(MVUtil.statsPhist.keySet()); - break; - case "enscnt": - listStatName.addAll(MVUtil.statsEnscnt.keySet()); - break; - case "mpr": - listStatName.addAll(MVUtil.statsMpr.keySet()); - break; - case "orank": - listStatName.addAll(MVUtil.statsOrank.keySet()); - break; - case "ssvar": - listStatName.addAll(MVUtil.statsSsvar.keySet()); - break; - case "val1l2": - listStatName.addAll(MVUtil.statsVal1l2.keySet()); - break; - case "vcnt": - if (!boolVcnt) { - listStatName.addAll(MVUtil.statsVcnt.keySet()); - } - break; - case "ecnt": - listStatName.addAll(MVUtil.statsEcnt.keySet()); - break; - default: - - } - } - } catch (Exception e) { - logger.error(e.getMessage()); - } - - Collections.sort(listStatName); - return listStatName; - } - - @Override - public List getListValues(MVNode nodeCall, String strField, String[] currentDBName) { - List listRes = new ArrayList<>(); - N1qlQueryResult queryResult = null; - List queryList = null; - String queryString = ""; - - boolean boolMode = nodeCall.children[0].tag.equals("mode_field"); - boolean boolMtd = nodeCall.children[0].tag.equals("mtd_field"); - boolean boolRhist = nodeCall.children[0].tag.equals("rhist_field"); - boolean boolPhist = nodeCall.children[0].tag.equals("phist_field"); - boolean boolROC = nodeCall.children[0].tag.equals("roc_field"); - boolean boolRely = nodeCall.children[0].tag.equals("rely_field"); - boolean boolEnsSS = nodeCall.children[0].tag.equals("ensss_field"); - boolean boolPerf = nodeCall.children[0].tag.equals("perf_field"); - boolean boolTaylor = nodeCall.children[0].tag.equals("taylor_field"); - boolean boolEclv = nodeCall.children[0].tag.equals("eclv_field"); - String strHeaderTable; - if (boolMode) { - strHeaderTable = "mode"; - } else if (boolMtd) { - strHeaderTable = "mtd"; - } else { - strHeaderTable = "stat"; - } - boolean boolNRank = strField.equalsIgnoreCase("N_RANK"); - boolean boolNBin = strField.equalsIgnoreCase("N_BIN"); - - // parse the fcst_var/stat constraint to build a list of line_data tables and fcst_var values - Map tableFcstVarStat = new HashMap<>(); - Map tableLineDataTables = new HashMap<>(); - boolean boolFcstVar = false; - if (boolRhist) { - tableLineDataTables.put("rhist", "true"); - } else if (boolPhist) { - tableLineDataTables.put("phist", "true"); - } else if (boolROC) { - tableLineDataTables.put("pct", "true"); - tableLineDataTables.put("ctc", "true"); - } else if (boolRely) { - tableLineDataTables.put("pct", "true"); - } else if (boolEnsSS) { - tableLineDataTables.put("ssvar", "true"); - } else if (boolPerf) { - tableLineDataTables.put("cts", "true"); - } else if (boolTaylor) { - tableLineDataTables.put("sl1l2", "true"); - } else if (boolEclv) { - tableLineDataTables.put("eclv", "true"); - } else if (1 < nodeCall.children.length) { - boolFcstVar = true; - MVNode nodeFcstVarStat = nodeCall.children[1]; - for (int i = 0; i < nodeFcstVarStat.children.length; i++) { - MVNode nodeFcstVar = nodeFcstVarStat.children[i]; - tableFcstVarStat.put(nodeFcstVar.name, "true"); - for (int j = 0; j < nodeFcstVar.children.length; j++) { - String strStat = nodeFcstVar.children[j].value; - String strLineDataTable = MVUtil.getStatTable(strStat).substring(10); - tableLineDataTables.put(strLineDataTable, "true"); - if (strLineDataTable.equals("cnt")) { - tableLineDataTables.put("sl1l2", "true"); - tableLineDataTables.put("sal1l2", "true"); - } else if (strLineDataTable.equals("cts")) { - tableLineDataTables.put("ctc", "true"); - } - } - } - } - String[] listTables = tableLineDataTables.keySet().toArray(new String[]{}); - String tableList = ""; - if (listTables.length == 0) { - boolFcstVar = false; - } else { - tableList = "["; - for (String tableName : listTables) { - if (tableList.length() > 1) { - tableList += ", "; - } - tableList += "\'" + tableName + "\'"; - } - tableList += "]"; - } - - String strWhere = ""; - //we need to get all values of obs_var for all variables - if (strField.equalsIgnoreCase("obs_var")) { - boolFcstVar = false; - } - if (boolFcstVar) { - String strFcstVarList = ""; - String[] listFcstVar = tableFcstVarStat.keySet().toArray(new String[]{}); - boolean boolRegEx = false; - for (int i = 0; i < listFcstVar.length; i++) { - if (listFcstVar[i].contains("*")) { - boolRegEx = true; - } - if (listFcstVar[i].length() > 0 && !listFcstVar[i].equals("NA")) { - strFcstVarList += (0 < i ? "," : "") + "'" + listFcstVar[i].replace("*", "%") + "'"; - } - } - if (strFcstVarList.length() > 0) { - strWhere += " WHERE h.fcst_var " + (boolRegEx ? "LIKE" : "IN") + " [" + strFcstVarList + "]"; - strWhere += " AND h.type = \'header\' AND h.header_type = \'stat\'"; - } - } - - // parse the list of constraints into a SQL where clause - String strWhereTime = ""; - for (int i = 1; i < nodeCall.children.length; i++) { - if (nodeCall.children[i].tag.equals("stat")) { - continue; - } - - // determine if the field should be used as criteria - MVNode nodeField = nodeCall.children[i]; - String strFieldCrit = nodeField.name.toLowerCase(Locale.ENGLISH); - boolean boolTimeCritField = false; - boolean boolTimeCritCur = false; - if (strFieldCrit.contains("valid") - || strFieldCrit.contains("init") - || strFieldCrit.contains("lead")) { - boolTimeCritField = strField.equals(strFieldCrit) - || (strField.contains("fcst_init") - && strFieldCrit.equals("init_hour")) - || (strField.contains("fcst_valid") - && strFieldCrit.equals("valid_hour")); - boolTimeCritCur = true; - } - // if so, build a where clause for the criteria - String strFieldDBCrit = formatField(strFieldCrit, boolMode || boolMtd, false); - if (strFieldDBCrit.contains("n_rank") || strFieldDBCrit.contains("n_bin")) { - continue; - } - String strSqlOp = "IN"; - String strValList = ""; - for (int j = 0; j < nodeField.children.length; j++) { - String strVal = nodeField.children[j].value; - if (strVal.contains("*")) { - strSqlOp = "LIKE"; - } - strValList += (0 < j ? "," : "") + "\'" + strVal.replace("*", "%") + "\'"; - } - - // add the where clause to the criteria, if appropriate - if (boolTimeCritField) { - if (boolMode || boolMtd) { - strWhere += (strWhere.equals("") ? " WHERE " : " AND ") - + strFieldDBCrit + " " + strSqlOp + " [" + strValList + "]"; - } else { - strWhereTime += (strWhereTime.equals("") ? " WHERE " : " AND ") - + strFieldDBCrit + " " + strSqlOp + " [" + strValList + "]"; - } - } else if (!boolTimeCritCur) { - strWhere += (strWhere.equals("") ? " WHERE " : " AND ") - + strFieldDBCrit + " " + strSqlOp + " [" + strValList + "]"; - } - } - - // build a query for the values - String strFieldDB = formatField(strField, boolMode || boolMtd).trim(); - if (boolFcstVar) { - if (strField.contains("valid") || strField.contains("init") || strField.contains("lead")) { - strFieldDB = "ld." + strField; - } else { - strFieldDB = "h." + strField; - } - } else { - strFieldDB = strFieldDB.replaceAll("ld\\.", ""); - strFieldDB = strFieldDB.replaceAll("h\\.", ""); - } - - String dbList = "["; - - for (String cdbName : currentDBName) { - if (dbList.length() > 1) { - dbList += ", "; - } - dbList += "\'" + cdbName + "\'"; - } - dbList += "]"; - - if (boolNRank) { - queryString = "SELECT DISTINCT ld.n_rank " - + " FROM `" + getBucket().name() + "` as h " - + " INNER JOIN `" + getBucket() - .name() + "` as ld on ld.header_id = meta(h).id " - + strWhere + (strWhere.equals("") ? " WHERE" : " AND") - + " ld.type = \'line\' AND ld.line_type = \'rhist\'" - + " AND h.type = \'header\' AND h.header_type = \'" + strHeaderTable + "\'" - + " AND h.dbname IN " + dbList - + " ORDER BY ld.n_rank"; - } else if (boolNBin) { - queryString = "SELECT DISTINCT ld.n_bin " - + " FROM `" + getBucket().name() + "` as h " - + " INNER JOIN `" + getBucket() - .name() + "` as ld on ld.header_id = meta(h).id " - + strWhere + (strWhere.equals("") ? " WHERE" : " AND") - + " ld.type = \'line\' AND ld.line_type = \'phist\'" - + " AND h.type = \'header\' AND h.header_type = \'" + strHeaderTable + "\'" - + " AND h.dbname IN " + dbList - + " ORDER BY ld.n_bin"; - } else if (!boolMode && !boolMtd && boolFcstVar) { - queryString = "SELECT DISTINCT " + strFieldDB - + " FROM `" + getBucket().name() + "` as h " - + " INNER JOIN `" + getBucket() - .name() + "` as ld on ld.header_id = meta(h).id " - + strWhere + (strWhere.equals("") ? " WHERE" : " AND") - + " ld.type = \'line\' AND ld.line_type IN " + tableList - + " AND h.type = \'header\' AND h.header_type = \'" + strHeaderTable + "\'" - + " AND h.dbname IN " + dbList - + " ORDER BY " + strFieldDB; - } else if (!boolMode && !boolMtd - && (strField.equals("fcst_lead") - || strField.contains("valid") - || strField.contains("init"))) { - // build a query to list all distinct, - // ordered values of the list field from the selected line documents - strWhere = strWhere.replaceAll("ld\\.", ""); - queryString = "SELECT DISTINCT " + strFieldDB - + " FROM `" + getBucket().name() + "` " - + strWhere + (strWhere.equals("") ? " WHERE" : " AND") - + " line_type IN " + tableList - + " AND type = \'line\'" + strWhereTime - + " AND dbname IN " + dbList - + " ORDER BY " + strFieldDB; - } else { - strWhere = strWhere.replaceAll("h\\.", ""); - queryString = "SELECT DISTINCT " + strFieldDB - + " FROM `" + getBucket().name() + "` " - + strWhere + (strWhere.equals("") ? " WHERE" : " AND") - + " header_type = \'" + strHeaderTable - + "\' AND type = \'header\'" - + " AND dbname IN " + dbList - + " ORDER BY " + strFieldDB; - } - // execute the query - try { - cbStopWatch.start(); - queryResult = getBucket().query(N1qlQuery.simple(queryString)); - cbStopWatch.stop(); - logger.error("getListStat QUERYSTRING: " + queryString + "\nDuration: " + cbStopWatch - .getFormattedTotalDuration()); - - for (N1qlQueryRow row : queryResult) { - listRes.add(row.value().get(strField).toString()); - } - - } catch (Exception e) { - logger.error(e.getMessage()); - } - - Collections.sort(listRes); - return listRes; - } - - @Override - public MvResponse executeQueriesAndSaveToFile( - List queries, String fileName, - boolean isCalc, String currentDBName, - boolean isNewFile) throws ValidationException { - N1qlQueryResult queryResult = null; - List queryList = null; - String queryString = ""; - MvResponse mvResponse = new MvResponse(); - - - List listSqlBeforeSelect = new ArrayList<>(); - List listSqlLastSelectTemp = new ArrayList<>(); - List listSqlLastSelect = new ArrayList<>(); - for (int i = queries.size() - 1; i >= 0; i--) { - if (queries.get(i).startsWith("SELECT")) { - listSqlLastSelectTemp.add(queries.get(i)); - } else { - break; - } - } - for (String sql : queries) { - if (listSqlLastSelectTemp.contains(sql)) { - listSqlLastSelect.add(sql); - } else { - listSqlBeforeSelect.add(sql); - } - } - StopWatch dbStopWatch = new StopWatch(); - StopWatch saveToFileStopWatch = new StopWatch(); - - try { - dbStopWatch.start(); - for (String aListSqlBeforeSelect : listSqlBeforeSelect) { - try { - queryResult = getBucket().query(N1qlQuery.simple(aListSqlBeforeSelect)); - queryList = queryResult.allRows(); - } catch (CouchbaseException e) { - logger.error(e.getMessage()); - } - } - try { - dbStopWatch.stop(); - } catch (StopWatchException e) { - logger.error(e.getMessage()); - } - - for (int i = 0; i < listSqlLastSelect.size(); i++) { - boolean append = !isNewFile || i != 0; - boolean printHeader = !append; - try (FileWriter fstream = new FileWriter(new File(fileName), append); - BufferedWriter out = new BufferedWriter(fstream);) { - - queryString = listSqlLastSelect.get(i); - dbStopWatch.start(); - queryResult = getBucket().query(N1qlQuery.simple(queryString)); - dbStopWatch.stop(); - saveToFileStopWatch.start(); - queryList = queryResult.allRows(); - if (queryList.size() > 0) { - printFormattedTable(queryList, queryString, out, printHeader); - saveToFileStopWatch.stop(); - out.flush(); - mvResponse.setSuccess(true); - } else { - throw new Exception("Error: Query returned no data"); - } - } catch (Exception e) { - logger.error(e.getMessage()); - String stat = "This"; - if (e.getMessage().contains("Unknown column")) { - String[] queryArr = listSqlLastSelect.get(i).split(","); - for (String str : queryArr) { - if (str.contains("stat_name")) { - stat = str.replaceAll("stat_name", "").trim(); - break; - } - } - if (stat.equals("This")) { - Pattern pattern = Pattern.compile("'(.*?)'"); - Matcher matcher = pattern.matcher(e.getMessage()); - - if (matcher.find()) { - stat = matcher.group(1); - if (stat.contains(".")) { - stat = stat.split("\\.")[1]; - } - } - } - logger.error(stat + " statistic can only be plotted as an aggregation of lines"); - - //rethrow the exception to be printed as a error popup on UI - throw new ValidationException( - stat + " statistic can only be plotted as an aggregation of lines"); - } - - } - } - - } catch (CouchbaseException e) { - logger.error(e.getMessage()); - } - String message = null; - try { - message = "Database query time for " + currentDBName + " " - + dbStopWatch.getFormattedTotalDuration(); - if (saveToFileStopWatch.getTotalDuration() != null) { - message = message + "\nSave to file time for " + currentDBName + " " - + saveToFileStopWatch.getFormattedTotalDuration(); - } - } catch (StopWatchException e) { - logger.error(e.getMessage()); - } - - mvResponse.setInfoMessage(message); - - return mvResponse; - } - - /** - * Prints a textual representation of the input with the field names in the first row to the - * specified {@link BufferedWriter} destination. - * - * @param res The ResultSet to print - * @param bufferedWriter The stream to write the formatted results to (defaults to printStream) - */ - - private void printFormattedTable( - List res, String queryString, BufferedWriter bufferedWriter, boolean isHeader) { - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - ArrayList docFields = new ArrayList(); - String fieldString = ""; - String strVal = ""; - Boolean firstLine; - - // determine the field names in the correct order from the select portion of the query statement - fieldString = queryString - .substring(queryString.indexOf(" ") + 1, queryString.indexOf("FROM") - 1) - .trim(); - String[] fieldArr = fieldString.split(","); - for (String str : fieldArr) { - str = str.trim(); - while (str.contains(" ")) { - str = str.substring(str.indexOf(" ") + 1); - } - if (str.contains(".")) { - str = str.substring(str.indexOf(".") + 1); - } - - docFields.add(str); - } - char delim = '\t'; - try { - // print out the column headers - firstLine = true; - for (String docHeader : docFields) { - if (firstLine) { - bufferedWriter.write(docHeader); - firstLine = false; - } else { - bufferedWriter.write(delim + docHeader); - } - } - bufferedWriter.write(System.getProperty("line.separator")); - - // print out the table of values - for (N1qlQueryRow row : res) { - firstRowObject = row.value(); - firstLine = true; - for (String docLine : docFields) { - strVal = firstRowObject.get(docLine).toString(); - strVal = strVal.equalsIgnoreCase("null") ? "NA" : strVal; - strVal = strVal.equalsIgnoreCase("-9999") ? "NA" : strVal; - if (firstLine) { - bufferedWriter.write(strVal); - firstLine = false; - } else { - bufferedWriter.write(delim + strVal); - } - } - bufferedWriter.write(System.getProperty("line.separator")); - } - -// if (objectType.equals("DATETIME")) { -// Calendar cal = Calendar.getInstance(); -// cal.setTimeZone(TimeZone.getTimeZone("UTC")); -// Timestamp ts = res.getTimestamp(i, cal); -// strVal = DATE_FORMAT.format(ts); - - } catch (Exception e) { - logger.error( - " ** ERROR: Caught " + e.getClass() - + " in printFormattedTable(ResultSet res): " + e.getMessage()); - } - } - - private Map getPctThreshInfo(String query, String currentDBName) { - int numPctThresh = 0; - int pctThresh = -1; - Map result = new HashMap<>(); -// try (Connection con = getConnection(currentDBName); -// Statement stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, -// ResultSet.CONCUR_READ_ONLY); -// ResultSet resultSet = stmt.executeQuery(query) -// ) { -// -// // validate and save the number of thresholds -// while (resultSet.next()) { -// pctThresh = resultSet.getInt(1); -// numPctThresh++; -// } -// -// } catch (SQLException e) { -// logger.error(e.getMessage()); -// } - result.put("numPctThresh", numPctThresh); - result.put("pctThresh", pctThresh); - - return result; - } - - private List getNumbers(String queryString, String currentDBName) { - List result = new ArrayList<>(); - String fieldName; - - N1qlQueryResult queryResult = null; - List queryList = null; - - fieldName = queryString.substring(queryString.indexOf(".") + 1); - fieldName = fieldName.substring(0, fieldName.indexOf(" ")); - - try { - queryResult = getBucket().query(N1qlQuery.simple(queryString)); - } catch (Exception e) { - logger.error(e.getMessage()); - } - for (N1qlQueryRow row : queryResult) { - - result.add(row.value().get(fieldName).toString()); - } - return result; - } - - /** - * Returns a list of all available database names - * - * @return list of database names - */ - @Override - public List getAllDatabases() { - return new ArrayList<>(listDB.keySet()); - } - - @Override - public Map getAllDatabasesWithDescription() { - return Collections.unmodifiableMap(listDB); - } - - @Override - public Map> getAllGroups() { - return Collections.unmodifiableMap(groupToDatabases); - } - - - @Override - public SimpleDateFormat getDateFormat() { - return DATE_FORMAT; - } - - @Override - public Pattern getDateRangePattern() { - - return Pattern.compile("(?i)\\s*between\\s+'([^']+)'\\s+and\\s+'([^']+)'\\s*"); - } - - @Override - public void closeDataSource() { - - } - - - /** - * The input job and plot_fix information is used to build a list of SQL queries that result in - * the temp table plot_data being filled with formatted plot data for a single plot. Several job - * validation checks are performed, and an Exception is thrown in case of error. - * - * @param job contains plot job information - * @param mapPlotFixPerm permutation of plot_fix values for current plot - * @return list of SQL statements that result in plot data - * @throws Exception - */ - @Override - public List buildPlotSql( - MVPlotJob job, MVOrderedMap mapPlotFixPerm, - PrintStream printStreamSql) throws ValidationException { - MVOrderedMap _mapFcstVarPat = new MVOrderedMap(); - MVOrderedMap mapPlotFixVal = job.getPlotFixVal(); - // determine if the plot job is for stat data or MODE data - List tableHeaderSqlType; - if (job.isModeJob()) { - tableHeaderSqlType = modeHeaderSqlType; - } else if (job.isMtdJob()) { - tableHeaderSqlType = mtdHeaderSqlType; - } else { - tableHeaderSqlType = statHeaderSqlType; - } - - // populate the plot template values with plot_fix values - Map.Entry[] listPlotFixVal = MVUtil.buildPlotFixTmplMap(mapPlotFixPerm, mapPlotFixVal); - - // build the sql where clauses for the current permutation of fixed variables and values - String strPlotFixWhere = buildPlotFixWhere(listPlotFixVal, job, - job.isModeJob() || job.isMtdJob()); - - // add the user-specified condition clause, if present - if (null != job.getPlotCond() && !job.getPlotCond().isEmpty()) { - strPlotFixWhere += " AND " + job.getPlotCond() + " "; - } - - // determine if the plot requires data aggregation or calculations - boolean boolAggStat = job.isAggStat(); - - boolean boolCalcStat = job.isModeRatioJob() - || job.isMtdRatioJob() - || job.isCalcStat(); - boolean boolEnsSs = job.getPlotTmpl().equals("ens_ss.R_tmpl"); - - // remove multiple dep group capability - MVOrderedMap[] listDep = job.getDepGroups(); - MVOrderedMap mapDepGroup; - if (1 != listDep.length && !job.getPlotTmpl().equals("eclv.R_tmpl")) { - throw new ValidationException("unexpected number of groups: " + listDep.length); - } - if (job.getPlotTmpl().equals("eclv.R_tmpl")) { - mapDepGroup = new MVOrderedMap(); - } else { - mapDepGroup = listDep[0]; - } - - List listSql = new ArrayList<>(); - String strSelectSql = ""; - for (int intY = 1; intY <= 2; intY++) { - - // get the dep values for the current dep group - MVOrderedMap mapDep = (MVOrderedMap) mapDepGroup.get("dep" + intY); - if (mapDep == null) { - mapDep = new MVOrderedMap(); - } - - - // establish lists of entries for each group of variables and values - Map.Entry[] listSeries = (1 == intY ? job.getSeries1Val() : job.getSeries2Val()) - .getOrderedEntriesForSqlSeries(); - Map.Entry[] listDepPlot = mapDep.getOrderedEntries(); - - // if there is a mis-match between the presence of series and dep values, bail - if (0 < listDepPlot.length && 1 > listSeries.length) { - throw new ValidationException("dep values present, but no series values for Y" + intY); - } - if (1 > listDepPlot.length && 0 < listSeries.length - && !job.getPlotTmpl().equals("eclv.R_tmpl")) { - throw new ValidationException("series values present, but no dep values for Y" + intY); - } - - // there must be at least one y1 series and stat, but not for y2 - - if (!job.getPlotTmpl().equals( - "eclv.R_tmpl") && 1 == intY && 1 > listDepPlot.length && 1 > listSeries.length) { - throw new ValidationException("no Y1 series stat found"); - } - if (2 == intY && 1 > listDepPlot.length && 1 > listSeries.length) { - continue; - } - - - // build the select list and where clauses for the series variables and values - - String strWhere = strPlotFixWhere; - BuildMysqlQueryStrings buildMysqlQueryStrings = build(job.isModeJob() || job.isMtdJob(), - tableHeaderSqlType, - listSeries, strWhere, true); - String strSelectList = buildMysqlQueryStrings.getSelectList(); - strWhere = buildMysqlQueryStrings.getWhere(); - - // if the fcst_valid or fcst_init fields are not present - // in the select list and temp table list, add them - if (!strSelectList.contains("fcst_init")) { - if (job.isModeJob() || job.isMtdJob()) { - strSelectList += ",h.fcst_init"; - } else { - if (strSelectList.length() > 0) { - strSelectList += ","; - } - strSelectList += "ld.fcst_init_beg"; - - - } - } - if (!strSelectList.contains("fcst_valid")) { - if (job.isModeJob() || job.isMtdJob()) { - strSelectList += ",h.fcst_valid"; - } else { - if (strSelectList.length() > 0) { - strSelectList += ","; - } - strSelectList += "ld.fcst_valid_beg"; - - } - } - BuildMysqlQueryStrings buildQueryPlotStrings = build(job.isModeJob() || job.isMtdJob(), - tableHeaderSqlType, - listSeries, strWhere, false); - String selectPlotList = buildQueryPlotStrings.getSelectList().trim(); - // if the fcst_valid or fcst_init fields are not present - // in the select list and temp table list, add them - if (!selectPlotList.contains("fcst_init") && !selectPlotList.contains("init_hour")) { - if (job.isModeJob() || job.isMtdJob()) { - selectPlotList += ",h.fcst_init"; - } else { - selectPlotList += ",ld.fcst_init_beg"; - } - } - if (!selectPlotList.contains("fcst_valid")) { - if (job.isModeJob() || job.isMtdJob()) { - selectPlotList += ",h.fcst_valid"; - } else { - selectPlotList += ",ld.fcst_valid_beg"; - } - } - - if (!boolEnsSs && !strSelectList.contains("fcst_lead")) { - if (job.isModeJob() || job.isMtdJob()) { - - if (job.getEventEqual()) { - strSelectList += "," + " if( (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) is NULL , h.fcst_lead , h.fcst_lead " - + "+ (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) ) fcst_lead"; - } else { - strSelectList += ",h.fcst_lead"; - } - selectPlotList += ",h.fcst_lead"; - } else { - if (job.getEventEqual()) { - strSelectList += "," + " if( (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) is NULL , ld.fcst_lead , ld.fcst_lead " - + "+ (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) ) fcst_lead"; - } else { - strSelectList += ",ld.fcst_lead"; - } - selectPlotList += ",h.fcst_lead"; - } - } - - - // for MODE, build the group by list - String[] listGroupBy = new String[]{}; - if ((job.isModeJob() && !job.isModeRatioJob()) - || (job.isMtdJob() && !job.isMtdRatioJob())) { - ArrayList listGroupFields = new ArrayList<>(); - listGroupFields.add(job.getIndyVar()); - for (Map.Entry listSery : listSeries) { - listGroupFields.add(listSery.getKey().toString()); - } - - Collections.addAll(listGroupFields, job.getPlotFixVal().getKeyList()); - listGroupBy = listGroupFields.toArray(new String[]{}); - } - - // for ensemble spread/skill, add the ssvar line data and bail - if (boolEnsSs) { - - listSql.add("SELECT " - + "h." + selectPlotList + ",h.fcst_var," - + "ld.total,ld.bin_n,ld.var_min,ld.var_max,ld.var_mean," - + "ld.fbar,ld.obar,ld.fobar,ld.ffbar,ld.oobar" - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld ON ld.header_id = meta(h).id " - + " WHERE " + strWhere + " AND ld.type = \'line\' AND ld.line_type = \'ssvar\' " - + " AND h.type = \'header\' AND h.header_type = \'stat\' " - + " AND h.dbname IN [\'" + job.getCurrentDBName().get(0) + "\']"); - - return listSql; - } - - - // validate and get the type and values for the independent variable - String strIndyVarType = ""; - String strIndyVar = job.getIndyVar(); - String strIndyVarFormatted = ""; - if (!strIndyVar.isEmpty()) { - String[] listIndyVal = job.getIndyVal(); - if (!tableHeaderSqlType.contains(strIndyVar)) { - throw new ValidationException("unrecognized indep " - + (job.isModeJob() ? "mode" : "stat") - + "_header field: " + strIndyVar); - } - if (1 > listIndyVal.length) { - throw new ValidationException("no independent variable values specified"); - } - - // construct the select list item, where clause - // and temp table entry for the independent variable - if (!strSelectList.contains(strIndyVar)) { - strSelectList += "," + formatField(strIndyVar, job.isModeJob() || job.isMtdJob(), - true); - selectPlotList += "," + formatField(strIndyVar, job.isModeJob() || job.isMtdJob(), - true); - } - strIndyVarFormatted = formatField(strIndyVar, job.isModeJob() || job.isMtdJob(), false); - if (strIndyVar.equals("fcst_lead") && job.getEventEqual()) { - strIndyVarFormatted = " if( (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) is NULL , " - + strIndyVarFormatted + " , " + strIndyVarFormatted - + " + (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) ) "; - } - strWhere += (!strWhere.isEmpty() ? " AND " : "") + strIndyVarFormatted - + " IN [" + MVUtil.buildValueList(job.getIndyVal()) + "] "; - } - // add fcst_var to the select list and temp table entries - // strSelectList += ",h.fcst_var"; - selectPlotList += ",h.fcst_var"; - - if (listPlotFixVal.length > 0) { - for (int i = 0; i < listPlotFixVal.length; i++) { - String strField = (String) listPlotFixVal[i].getKey(); - - } - } - - /* - * For agg_stat PCT plots, retrieve the sizes of PCT threshold lists for each series - */ - Map pctThreshInfo = new HashMap<>(); - if (job.getAggPct()) { - MVOrderedMap[] series = MVUtil.permute(job.getSeries1Val().convertFromSeriesMap()) - .getRows(); - MVOrderedMap[] forecastVars; - if (job.getPlotTmpl().equals("eclv.R_tmpl") && job.getDepGroups().length == 0) { - MVOrderedMap m = new MVOrderedMap(); - m.put("NA", "ECLV"); - forecastVars = new MVOrderedMap[]{m}; - } else { - forecastVars = MVUtil.permute((MVOrderedMap) job.getDepGroups()[0].get("dep" + intY)) - .getRows(); - } - for (int forecastVarsInd = 0; forecastVarsInd < forecastVars.length; forecastVarsInd++) { - MVOrderedMap stats = forecastVars[forecastVarsInd]; - String[] vars = stats.getKeyList(); - for (int varsInd = 0; varsInd < vars.length; varsInd++) { - int[] seriesNthresh = new int[series.length]; - for (int seriesInd = 0; seriesInd < series.length; seriesInd++) { - MVOrderedMap ser = series[seriesInd]; - String[] serName = ser.getKeyList(); - for (int serNameInd = 0; serNameInd < serName.length; serNameInd++) { - String strSelPctThresh = "SELECT DISTINCT ld.n_thresh FROM " - + "stat_header h, line_data_pct ld "; - strSelPctThresh = strSelPctThresh + " WHERE "; - if (strIndyVarFormatted.length() > 0 && job.getIndyVal().length > 0) { - strSelPctThresh = strSelPctThresh + strIndyVarFormatted; - strSelPctThresh = strSelPctThresh + " IN [" + MVUtil.buildValueList( - job.getIndyVal()) + "] " + " AND "; - } - strSelPctThresh = strSelPctThresh + serName[serNameInd] - + " = '" + ser.getStr(serName[serNameInd]) + "'"; - if (!vars[varsInd].equals("NA")) { - strSelPctThresh = strSelPctThresh + " AND fcst_var='" + vars[varsInd] + "' "; - } - if (strPlotFixWhere.length() > 0) { - strSelPctThresh = strSelPctThresh + " AND " + strPlotFixWhere; - } - strSelPctThresh = strSelPctThresh + " AND ld.stat_header_id = h.stat_header_id;"; - printStreamSql.println(strSelPctThresh + " "); - printStreamSql.flush(); - - // run the PCT thresh query - List errors = new ArrayList<>(); - for (int i = 0; i < job.getCurrentDBName().size(); i++) { - pctThreshInfo = getPctThreshInfo(strSelPctThresh, job.getCurrentDBName().get(i)); - if (1 != pctThreshInfo.get("numPctThresh")) { - String error = "number of PCT thresholds (" + pctThreshInfo.get( - "numPctThresh") + ") not distinct for " + serName[serNameInd] - + " = '" + ser.getStr(serName[serNameInd]) - + "' AND database " + job.getCurrentDBName().get(i) + "'"; - if (!vars[varsInd].equals("NA")) { - error = error + "' AND fcst_var='" + vars[varsInd] + "'"; - } - errors.add(error); - } else if (1 > pctThreshInfo.get("numPctThresh")) { - String error = "invalid number of PCT thresholds (" - + pctThreshInfo.get("numPctThresh") + ") found for " - + serName[serNameInd] + " = '" - + ser.getStr(serName[serNameInd]) - + "' AND database " + job.getCurrentDBName().get(i) + "'"; - if (!vars[varsInd].equals("NA")) { - error = error + "' AND fcst_var='" + vars[varsInd] + "'"; - } - errors.add(error); - } else { - errors.add(null); - seriesNthresh[seriesInd] = pctThreshInfo.get("pctThresh"); - } - } - boolean noErrors = false; - for (String error : errors) { - if (error == null) { - noErrors = true; - break; - } - } - if (!noErrors) { - for (String error : errors) { - if (error != null) { - throw new ValidationException(error); - } - } - } - //seriesNthresh[seriesInd] = pctThreshInfo.get("pctThresh"); - } - - } - boolean allEqual = true; - for (Integer s : seriesNthresh) { - if (!s.equals(seriesNthresh[0])) { - allEqual = false; - break; - } - } - if (!allEqual) { - String error = "Different value for PCT thresholds for individual series!"; - throw new ValidationException(error); - } else { - pctThreshInfo.put("pctThresh", seriesNthresh[0]); - } - } - } - - } else { - pctThreshInfo.put("pctThresh", -1); - } - - - // determine how many queries are needed to gather that stat information - int intNumQueries; - String[][] listFcstVarStat; - if (!job.getPlotTmpl().equals("eclv.R_tmpl")) { - listFcstVarStat = MVUtil.buildFcstVarStatList(mapDep); - intNumQueries = listFcstVarStat.length; - } else { - intNumQueries = 1; - listFcstVarStat = new String[0][0]; - } - - // build a query for each fcst_var/stat pair - // or a just single query for contingency tables or partial sums - for (int intFcstVarStat = 0; intFcstVarStat < intNumQueries; intFcstVarStat++) { - String strFcstVarClause = ""; - String strStat = ""; - if (listFcstVarStat.length > 0) { - // get the current fcst_var/stat pair - String strFcstVar = listFcstVarStat[intFcstVarStat][0]; - strStat = listFcstVarStat[intFcstVarStat][1]; - - // build the fcst_var where clause criteria - strFcstVarClause = "= '" + strFcstVar + "'"; - Matcher matProb = MVUtil.prob.matcher(strFcstVar); - if (matProb.matches() && strFcstVar.contains("*")) { - Pattern patFcstVar = Pattern.compile( - strFcstVar.replace("*", ".*").replace("(", "\\(").replace(")", "\\)")); - if (!_mapFcstVarPat.containsKey(patFcstVar)) { - _mapFcstVarPat.put(patFcstVar, MVUtil.replaceSpecialChars(strFcstVar)); - } - strFcstVarClause = "LIKE '" + strFcstVar.replace("*", "%") + "'"; - } - } else { - if (job.getPlotTmpl().equals("eclv.R_tmpl")) { - strStat = "ECLV"; - } - } - if (!strSelectList.contains("fcst_var")) { - strSelectList += ",\n'" + listFcstVarStat[intFcstVarStat][0] + "' fcst_var"; - } else { - strSelectList = strSelectList - .replace(listFcstVarStat[intFcstVarStat - 1][0] + "' fcst_var" - , listFcstVarStat[intFcstVarStat][0] + "' fcst_var"); - } - - // determine the table containing the current stat - Map tableStats; - String strStatTable = ""; - String strStatField = strStat.toLowerCase(Locale.US); - if (job.isModeJob()) { - String strStatMode = MVUtil.parseModeStat(strStat)[0]; - if (MVUtil.modeSingleStatField.containsKey(strStatMode)) { - tableStats = MVUtil.modeSingleStatField; - } else if (MVUtil.modePairStatField.containsKey(strStatMode)) { - tableStats = MVUtil.modeSingleStatField; - } else if (MVUtil.modeRatioField.contains(strStat)) { - tableStats = MVUtil.modeSingleStatField; - } else { - throw new ValidationException("unrecognized mode stat: " + strStatMode); - } - } else if (job.isMtdJob()) { - - String[] listStatParse = strStat.split("_"); - String stat = strStat.replace("_" + listStatParse[listStatParse.length - 1], ""); - - if (mtd3dSingleStatField.containsKey(stat)) { - tableStats = mtd3dSingleStatField; - } else if (MVUtil.mtd3dPairStatField.containsKey(stat)) { - tableStats = MVUtil.mtd3dPairStatField; - } else if (MVUtil.mtdRatioField.contains(strStat)) { - tableStats = mtd3dSingleStatField; - } else if (MVUtil.mtd2dStatField.containsKey(stat)) { - tableStats = MVUtil.mtd2dStatField; - } else { - throw new ValidationException("unrecognized mode stat: " + stat); - } - } else { - - String aggType = null; - if (job.getCalcCtc() || job.getAggCtc()) { - aggType = MVUtil.CTC; - } else if (job.getCalcSl1l2() || job.getAggSl1l2()) { - aggType = MVUtil.SL1L2; - } else if (job.getCalcGrad() || job.getAggGrad()) { - aggType = MVUtil.GRAD; - } else if (job.getCalcSal1l2() || job.getAggSal1l2()) { - aggType = MVUtil.SAL1L2; - } else if (job.getAggNbrCnt()) { - aggType = MVUtil.NBRCNT; - } else if (job.getAggPct()) { - aggType = MVUtil.PCT; - } else if (job.getAggSsvar()) { - aggType = MVUtil.SSVAR; - } else if (job.getCalcVl1l2() || job.getAggVl1l2()) { - aggType = MVUtil.VL1L2; - } else if (job.getCalcVal1l2() || job.getAggVal1l2()) { - aggType = MVUtil.VAL1L2; - } else if (job.getAggEcnt()) { - aggType = MVUtil.ECNT; - } - - - if (MVUtil.statsCnt.containsKey(strStat)) { - tableStats = MVUtil.statsCnt; - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsCnt, strStat, aggType); - strStatTable = " AND ld.line_type = \'" + aggType + "\' "; - } else { - strStatTable = " AND ld.line_type = \'cnt\' "; - } - } else if (MVUtil.statsSsvar.containsKey(strStat)) { - tableStats = MVUtil.statsSsvar; - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsSsvar, strStat, aggType); - strStatTable = " AND ld.line_type = \'" + aggType + "\' "; - } else { - strStatTable = " AND ld.line_type = \'ssvar\' "; - } - } else if (MVUtil.statsCts.containsKey(strStat)) { - tableStats = MVUtil.statsCts; - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsCts, strStat, aggType); - strStatTable = " AND ld.line_type = \'ctc\' "; - } else { - strStatTable = " AND ld.line_type = \'cts\' "; - } - } else if (MVUtil.statsNbrcnt.containsKey(strStat)) { - tableStats = MVUtil.statsNbrcnt; - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsNbrcnt, strStat, aggType); - } - strStatTable = " AND ld.line_type = \'nbrcnt\' "; - strStatField = strStat.replace("NBR_", "").toLowerCase(); - } else if (MVUtil.statsEnscnt.containsKey(strStat)) { - tableStats = MVUtil.statsEnscnt; - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsEnscnt, strStat, aggType); - } - strStatTable = " AND ld.line_type = \'enscnt\' "; - strStatField = strStat.replace("ENS_", "").toLowerCase(); - } else if (MVUtil.statsNbrcts.containsKey(strStat)) { - tableStats = MVUtil.statsNbrcts; - MVUtil.isAggTypeValid(MVUtil.statsNbrcts, strStat, aggType); - strStatTable = " AND ld.line_type = \'nbrcts\' "; - strStatField = strStat.replace("NBR_", "").toLowerCase(); - } else if (MVUtil.statsPstd.containsKey(strStat)) { - tableStats = MVUtil.statsPstd; - strStatTable = " AND ld.line_type = \'pstd\' "; - if (aggType != null) { - strStatTable = " AND ld.line_type = \'pct\' "; - MVUtil.isAggTypeValid(MVUtil.statsPstd, strStat, aggType); - for (int i = 1; i < pctThreshInfo.get("pctThresh"); i++) { - strStatTable += ", line_data_pct_thresh ldt" + i; - } - strStatTable += " "; - } - strStatField = strStat.replace("PSTD_", "").toLowerCase(); - } else if (MVUtil.statsMcts.containsKey(strStat)) { - tableStats = MVUtil.statsMcts; - MVUtil.isAggTypeValid(MVUtil.statsMcts, strStat, aggType); - strStatTable = " AND ld.line_type = \'mcts\' "; - strStatField = strStat.replace("MCTS_", "").toLowerCase(); - } else if (MVUtil.statsRhist.containsKey(strStat)) { - tableStats = MVUtil.statsRhist; - strStatTable = " AND ld.line_type = \'rhist\' "; - strStatField = strStat.replace("RHIST_", "").toLowerCase(); - } else if (MVUtil.statsPhist.containsKey(strStat)) { - tableStats = MVUtil.statsPhist; - MVUtil.isAggTypeValid(MVUtil.statsPhist, strStat, aggType); - strStatTable = " AND ld.line_type = \'phist\' "; - strStatField = strStat.replace("PHIST_", "").toLowerCase(); - } else if (MVUtil.statsVl1l2.containsKey(strStat)) { - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsVl1l2, strStat, aggType); - } - tableStats = MVUtil.statsVl1l2; - strStatTable = " AND ld.line_type = \'vl1l2\' "; - strStatField = strStat.replace("VL1L2_", "").toLowerCase(); - } else if (MVUtil.statsVal1l2.containsKey(strStat)) { - MVUtil.isAggTypeValid(MVUtil.statsVal1l2, strStat, aggType); - tableStats = MVUtil.statsVal1l2; - strStatTable = " AND ld.line_type = \'val1l2\' "; - strStatField = strStat.replace("VAL1L2_", "").toLowerCase(); - } else if (MVUtil.statsMpr.containsKey(strStat)) { - tableStats = MVUtil.statsMpr; - strStatTable = " AND ld.line_type = \'mpr\' "; - } else if (MVUtil.statsOrank.containsKey(strStat)) { - tableStats = MVUtil.statsOrank; - strStatTable = " AND ld.line_type = \'orank\' "; - strStatField = strStat.replace("ORANK_", "").toLowerCase(); - } else if (MVUtil.statsVcnt.containsKey(strStat)) { - tableStats = MVUtil.statsVcnt; - strStatField = strStat.replace("VCNT_", "").toLowerCase(); - if (aggType != null) { - MVUtil.isAggTypeValid(MVUtil.statsVcnt, strStat, aggType); - strStatTable = " AND ld.line_type = \'" + aggType + "\' AND"; - } else { - strStatTable = " AND ld.line_type = \'vcnt\' "; - } - } else if (strStat.equals("ECLV") && job.getPlotTmpl().equals("eclv.R_tmpl")) { - if (aggType != null && aggType.equals(MVUtil.CTC)) { - tableStats = MVUtil.statsCts; - strStatTable = " AND ld.line_type = \'ctc\' "; - } else { - tableStats = MVUtil.statsPstd; - strStatTable = "line_data_pct ld, line_data_pct_thresh ldt "; - } - } else if (MVUtil.statsEcnt.containsKey(strStat)) { - tableStats = MVUtil.statsEcnt; - strStatTable = " AND ld.line_type = \'ecnt\' "; - strStatField = strStat.replace("ECNT_", "").toLowerCase(); - } else { - throw new ValidationException("unrecognized stat: " + strStat); - } - } - - // build the SQL for the current fcst_var and stat - if (job.isModeJob()) { - - // build the mode SQL - String strWhereFcstVar = strWhere + " AND fcst_var " + strFcstVarClause; - - listSql.addAll(buildModeStatSql(strSelectList, strWhereFcstVar, strStat, listGroupBy, - job.getEventEqual(), listSeries)); - } else if (job.isMtdJob()) { - - // build the mtd SQL - String strWhereFcstVar = strWhere + " AND fcst_var " + strFcstVarClause; - listSql.addAll(buildMtdStatSql(strSelectList, strWhereFcstVar, strStat)); - } else { - boolean boolBCRMSE = false; - String strSelectStat = strSelectList; - - // build the select list and temp table elements for the stat and CIs - if (strStat.equals("BCRMSE")) { - boolBCRMSE = true; - strStatField = "bcmse"; - } - strSelectStat += ", \'" + strStat + "\' stat_name"; - - // add the appropriate stat table members, depending - // on the use of aggregation and stat calculation - if (job.getAggCtc()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.fy_oy, ld.fy_on, " - + "ld.fn_oy, ld.fn_on"; - } else if (job.getAggSl1l2()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.fbar, ld.obar, " - + "ld.fobar, ld.ffbar, ld.oobar, ld.mae"; - } else if (job.getAggGrad()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.fgbar, ld.ogbar, " - + "ld.mgbar, ld.egbar, ld.s1, ld.s1_og, ld.fgog_ratio"; - } else if (job.getAggSsvar()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.fbar, ld.obar, " - + "ld.fobar, ld.ffbar, ld.oobar, " - + "ld.var_mean, ld.bin_n"; - } else if (job.getAggSal1l2()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.fabar, ld.oabar, " - + "ld.foabar, ld.ffabar, ld.ooabar, ld.mae"; - - } else if (job.getAggEcnt()) { - - strSelectStat += ", 'NA' stat_value," - + " ld.total, ld.me, ld.rmse, ld.crps, ld.crpss," - + " ld.ign, " - + " ld.spread, ld.me_oerr, ld.rmse_oerr," - + " ld.spread_oerr," - + " ld.spread_plus_oerr"; - } else if (job.getAggPct()) { - if (!job.getPlotTmpl().equals("eclv.R_tmpl")) { - strSelectStat += ", 'NA' stat_value, ld.total, (ld.n_thresh - 1)"; - for (int i = 1; i < pctThreshInfo.get("pctThresh"); i++) { - strSelectStat += ", "; - if (i < pctThreshInfo.get("pctThresh") - 1) { - strSelectStat += " FORMAT((ldt" + i + ".thresh_i + ldt" - + (i + 1) + ".thresh_i)/2, 3), "; - } else { - strSelectStat += " FORMAT((ldt" + i + ".thresh_i + 1)/2, 3), "; - } - strSelectStat += " ldt" + i + ".oy_i, " - + " ldt" + i + ".on_i"; - } - } else { - strSelectStat += ", 'NA' stat_value, ld.n_thresh, ldt.thresh_i, ldt.oy_i ," - + " ldt.on_i"; - } - } else if (job.getAggNbrCnt()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.fbs, ld.fss"; - } else if (job.getAggVl1l2()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.ufbar, ld.vfbar, ld.uobar," - + " ld.vobar, ld.uvfobar, ld.uvffbar, ld.uvoobar," - + " ld.f_speed_bar, ld.o_speed_bar"; - } else if (job.getAggVal1l2()) { - strSelectStat += ", 'NA' stat_value, ld.total, ld.ufabar, ld.vfabar, " - + "ld.uoabar, ld.voabar, ld.uvfoabar, ld.uvffabar, " - + " ld.uvooabar"; - } else if (job.getCalcCtc()) { - strSelectStat += ", ld.total, ld.fy_oy, ld.fy_on, ld.fn_oy, ld.fn_on, " - + "'NA' stat_value, " - + " 'NA' stat_ncl, 'NA' stat_ncu, " - + "'NA' stat_bcl, 'NA' stat_bcu"; - } else if (job.getCalcSl1l2()) { - if (strStat.equalsIgnoreCase("mae")) { - strSelectStat += ", ld.mae, 'NA' stat_value, " - + " 'NA' stat_ncl, 'NA' stat_ncu, 'NA' stat_bcl, " - + "'NA' stat_bcu"; - } else { - strSelectStat += ", ld.total, ld.fbar, ld.obar, ld.fobar, ld.ffbar, ld.oobar," - + " 'NA' stat_value, " - + " 'NA' stat_ncl, 'NA' stat_ncu, 'NA' stat_bcl, " - + "'NA' stat_bcu"; - } - } else if (job.getCalcGrad()) { - strSelectStat += ", ld.total, ld.fgbar, ld.ogbar, ld.mgbar, ld.egbar, " - + "'NA' stat_value, " - + " 'NA' stat_ncl, 'NA' stat_ncu, 'NA' stat_bcl, " - + "'NA' stat_bcu"; - } else if (job.getCalcSal1l2()) { - strSelectStat += ", ld.total, ld.fabar, ld.oabar, ld.foabar, ld.ffabar, " - + "ld.ooabar, 'NA' stat_value, " - + " 'NA' stat_ncl, 'NA' stat_ncu, 'NA' stat_bcl, " - + "'NA' stat_bcu"; - } else if (job.getCalcVl1l2()) { - strSelectStat += ", ld.total, ld.ufbar, ld.vfbar, ld.uobar, ld.vobar, " - + "ld.uvfobar, ld.uvffbar, ld.uvoobar," - + " ld.f_speed_bar, ld.o_speed_bar, 'NA' stat_value, " - + " 'NA' stat_ncl, 'NA' stat_ncu, 'NA' stat_bcl, " - + "'NA' stat_bcu"; - - } else if (job.getCalcVal1l2()) { - strSelectStat += ", ld.total, ld.ufabar, ld.vfabar, ld.uoabar, ld.voabar, " - + "ld.uvfoabar, ld.uvffabar, ld.uvooabar," - + " 'NA' stat_value," - + " 'NA' stat_ncl, 'NA' stat_ncu, 'NA' stat_bcl, " - + "'NA' stat_bcu"; - } else { - if (boolBCRMSE) { - strSelectStat += ", IF(ld." + strStatField + "=-9999,'NA',CAST(sqrt(ld." - + strStatField + ") as DECIMAL(30, 5))) stat_value"; - - } else { - strSelectStat += ", IF(ld." + strStatField + "=-9999,'NA',ld." - + strStatField + " ) stat_value"; - - } - - // determine if the current stat has normal or bootstrap CIs - String[] listStatCI = (String[]) tableStats.get(strStat); - boolean boolHasNorm = false; - boolean boolHasBoot = false; - for (String aListStatCI : listStatCI) { - if (aListStatCI.equals("nc")) { - boolHasNorm = true; - } else if (aListStatCI.equals("bc")) { - boolHasBoot = true; - } - } - - // add the CIs to the select list, if present, otherwise, invalid data - if (boolHasNorm) { - strSelectStat += ", IF(ld." + strStatField + "_ncl=-9999,'NA',ld." - + strStatField + "_ncl ) stat_ncl" - + ", IF(ld." + strStatField + "_ncu=-9999,'NA',ld." - + strStatField + "_ncu ) stat_ncu"; - } else { - strSelectStat += ", 'NA' stat_ncl, 'NA' stat_ncu"; - } - - if (boolHasBoot && !boolAggStat) { - if (boolBCRMSE) { - strSelectStat += ", IF(ld." + strStatField + "_bcl=-9999,'NA',CAST(sqrt(ld." - + strStatField + "_bcl) as DECIMAL(30, 5))) stat_bcl" - + ", IF(ld." + strStatField + "_bcu=-9999,'NA',CAST(sqrt(ld." - + strStatField + "_bcu) as DECIMAL(30, 5))) stat_bcu"; - } else { - strSelectStat += ", IF(ld." + strStatField + "_bcl=-9999,'NA',ld." - + strStatField + "_bcl) stat_bcl" - + ", IF(ld." + strStatField + "_bcu=-9999,'NA',ld." - + strStatField + "_bcu ) stat_bcu"; - } - } else { - strSelectStat += ", 'NA' stat_bcl, 'NA' stat_bcu"; - } - } - - String strStatNaClause = ""; - if (!boolAggStat && !boolCalcStat) { - strStatNaClause = " AND ld." + strStatField + " != -9999"; - } - if (job.getAggPct()) { - if (!job.getPlotTmpl().equals("eclv.R_tmpl")) { - for (int i = 1; i < pctThreshInfo.get("pctThresh"); i++) { - strStatNaClause += " AND ld.line_data_id = ldt" + i + ".line_data_id " - + " AND ldt" + i + ".i_value = " + i; - } - } else { - strStatNaClause = " AND ld.line_data_id = ldt.line_data_id "; - } - } - - // build the query - strSelectSql += (strSelectSql.isEmpty() ? "" : " UNION ALL ") - + "SELECT " + strSelectStat - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld ON ld.header_id = meta(h).id "; - strSelectSql += " WHERE " + strWhere; - if (strFcstVarClause.length() > 0) { - strSelectSql += " AND h.fcst_var " + strFcstVarClause + " "; - } - strSelectSql += " AND h.type = \'header\' AND ld.type = \'line\'" + strStatTable - + " AND h.dbname = \'" + job.getCurrentDBName().get(0) + "\'" - + strStatNaClause; - } - - } - - } - - // add the stat plot query to the list - if (!job.isModeJob() && !job.isMtdJob()) { - listSql.add(strSelectSql + ";"); - } - - //remove duplicated queries - listSql = new ArrayList<>(new LinkedHashSet<>(listSql)); - - return listSql; - } - - /** - * Build where clauses for each of the input aggregation field/value entries and return the - * clauses as a String - * - * @param listPlotFixFields list of <plot_fix> field/value pairs - * @param boolModePlot specifies MODE plot - * @return generated SQL where clauses - */ - - private String buildPlotFixWhere( - Map.Entry[] listPlotFixFields, MVPlotJob job, - boolean boolModePlot) { - String strWhere = ""; - - // build the aggregate fields where clause - for (int i = 0; i < listPlotFixFields.length; i++) { - String strField = (String) listPlotFixFields[i].getKey(); - String strCondition = ""; - Object objValue = listPlotFixFields[i].getValue(); - if (objValue instanceof String[]) { - strCondition = "IN [" + MVUtil.buildValueList((String[]) objValue) + "]"; - } else if (objValue instanceof MVOrderedMap) { - MVOrderedMap mapTmpl = job.getTmplVal(); - String strSetName = mapTmpl.get(strField + "_set").toString(); - String[] listValues = (String[]) ((MVOrderedMap) objValue).get(strSetName); - strCondition = "IN [" + MVUtil.buildValueList(listValues) + "]"; - - } else if (objValue instanceof String) { - if (objValue.toString().startsWith("BETWEEN")) { - strCondition = objValue.toString(); - } else { - strCondition = "IN ['" + objValue.toString() + "']"; - } - } - String strIndyVarFormatted = formatField(strField, boolModePlot, false); - if (strField.equals("fcst_lead") && job.getEventEqual()) { - strIndyVarFormatted = " if( (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) is NULL , " - + strIndyVarFormatted + " , " + strIndyVarFormatted - + " + (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) ) "; - } - strWhere += (0 < i ? " AND " : " ") + strIndyVarFormatted + " " + strCondition + " "; - } - - return strWhere; - } - - /** - * Use the input query components to build a list of select statements to gather plot data. This - * function is a switchboard for the different types of MODE statistics: single, pair, derived, - * difference and ratios. - * - * @param strSelectList list of select fields - * @param strWhere list of where clauses - * @param strStat MODE stat - * @param listGroupBy list of fields to group by - * @param listSeries - * @return list of SQL queries for gathering plot data - */ - private List buildModeStatSql( - String strSelectList, String strWhere, String strStat, - String[] listGroupBy, boolean isEventEqualization, - Map.Entry[] listSeries) { - - List listQuery = new ArrayList<>(); - - // build the appropriate type of query, depending on the statistic - String[] listStatComp = MVUtil.parseModeStat(strStat); - if (listStatComp[0].equals("ACOV")) { - strWhere = strWhere.replace("h.", ""); - listQuery.add(buildModeSingleAcovTable(strSelectList, strWhere, strStat, listGroupBy, - isEventEqualization)); - } else if (MVUtil.modeSingleStatField.containsKey(listStatComp[0])) { - if (!listStatComp[1].startsWith("D")) { - strWhere = strWhere.replace("h.", ""); - listQuery.add(buildModeSingleStatTable(strSelectList, strWhere, strStat, listGroupBy, - isEventEqualization)); - } else { - String strWhereForQuery = strWhere.replace("h.", ""); - String newStat = strStat.replace("_D", "_F"); - String query1 = buildModeSingleStatTable(strSelectList, strWhereForQuery, newStat, - listGroupBy, - isEventEqualization); - newStat = strStat.replace("_D", "_O"); - String query2 = buildModeSingleStatTable(strSelectList, strWhereForQuery, newStat, - listGroupBy, - isEventEqualization); - strWhere = strWhere.replace("h.", "s."); - listQuery - .add(buildModeSingleStatDiffTable(strSelectList, strWhere, strStat, query1, query2, - listSeries)); - } - } else if (MVUtil.modePairStatField.containsKey(listStatComp[0])) { - - - if (listStatComp[0].equals("MAXINT")) { - String[] listMaxintQueries = { - buildModePairStatTable(strSelectList, strWhere, "MAXINTF_" + listStatComp[1]), - buildModePairStatTable(strSelectList, strWhere, "MAXINTO_" + listStatComp[1]) - }; - listMaxintQueries[0] = listMaxintQueries[0].replace("MAXINTF", "MAXINT"); - listMaxintQueries[1] = listMaxintQueries[1].replace("MAXINTO", "MAXINT"); - listQuery.addAll(Arrays.asList(listMaxintQueries)); - } else { - listQuery.add(buildModePairStatTable(strSelectList, strWhere, strStat)); - } - } else if ((listStatComp[0].equals("RATIO") - || listStatComp[0].equals("AREARAT") - || strStat.startsWith("OBJ"))) { - listQuery.add(buildModeSingleStatRatioTable(strSelectList, strWhere)); - } - return listQuery; - } - - /** - * Use the input query components to build a list of select statements to gather plot data. This - * function is a switchboard for the different types of MODE statistics: single, pair, derived, - * difference and ratios. - * - * @param strSelectList list of select fields - * @param strWhere list of where clauses - * @param strStat MTD stat - * @return list of SQL queries for gathering plot data - */ - private List buildMtdStatSql( - String strSelectList, String strWhere, String strStat) { - - List listQuery = new ArrayList<>(); - - // build the appropriate type of query, depending on the statistic - String[] listStatParse = strStat.split("_"); - String stat = strStat.replace("_" + listStatParse[listStatParse.length - 1], ""); - String strStatFlag = listStatParse[listStatParse.length - 1]; - if (mtd3dSingleStatField.containsKey(stat)) { - if (!strStatFlag.startsWith("D")) { - strWhere = strWhere.replace("h.", ""); - listQuery.add(buildMtd3dSingleStatTable(strSelectList, strWhere, strStat)); - } else { - String strWhereForQuery = strWhere.replace("h.", ""); - String newStat = strStat.replace("_D", "_F"); - String query1 = buildMtd3dSingleStatTable(strSelectList, strWhereForQuery, newStat); - newStat = strStat.replace("_D", "_O"); - String query2 = buildMtd3dSingleStatTable(strSelectList, strWhereForQuery, newStat); - strWhere = strWhere.replace("h.", "s."); - listQuery - .add(buildMtd3dSingleStatDiffTable(strSelectList, strWhere, strStat, query1, query2)); - } - } else if (MVUtil.mtd2dStatField.containsKey(stat)) { - if (!strStatFlag.startsWith("D")) { - strWhere = strWhere.replace("h.", ""); - listQuery.add(buildMtd2dStatTable(strSelectList, strWhere, strStat)); - } else { - String strWhereForQuery = strWhere.replace("h.", ""); - String newStat = strStat.replace("_D", "_F"); - String query1 = buildMtd2dStatTable(strSelectList, strWhereForQuery, newStat); - newStat = strStat.replace("_D", "_O"); - String query2 = buildMtd2dStatTable(strSelectList, strWhereForQuery, newStat); - strWhere = strWhere.replace("h.", "s."); - listQuery.add(buildMtd2dStatDiffTable(strSelectList, strWhere, strStat, query1, query2)); - } - } else if (MVUtil.mtd3dPairStatField.containsKey(stat)) { - strWhere = strWhere.replace("h.", ""); - listQuery.add(buildMtd3dPairStatTable(strSelectList, strWhere, strStat)); - } else { - strWhere = strWhere.replace("h.", ""); - if (stat.startsWith("2d")) { - listQuery.add(buildMtdSingleStatRatio2dTable(strSelectList, strWhere)); - } else if (stat.startsWith("3d")) { - listQuery.add(buildMtdSingleStatRatio3dTable(strSelectList, strWhere)); - } - } - return listQuery; - } - - private String buildModeSingleAcovTable( - String selectList, String strWhere, String stat, - String[] groups, boolean isEventEqualization) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = MVUtil.parseModeStat(stat); - if (2 != listStatParse.length) { - return ""; - } - String strStatFlag = listStatParse[1]; - String strGroupBy = " GROUP BY "; - for (int i = 0; i < groups.length; i++) { - strGroupBy += (0 < i ? ", " : "") + " " + groups[i]; - } - - // build the query components - String strStat = "SUM(area) / (2*total)"; - if (strStatFlag.charAt(0) != 'A') { - strStat = "SUM(area) / total"; - strGroupBy += " , fcst_flag"; - strWhere += " AND fcst_flag = " + ('F' == strStatFlag.charAt(0) ? "1" : "0"); - } - - - if (!strGroupBy.contains("fcst_valid")) { - if (groups.length > 0) { - strGroupBy += " ,"; - } - strGroupBy += " fcst_valid"; - - } - - //mandatory group by fcst_valid and fcst_lead for EE - if (isEventEqualization && !strGroupBy.contains("fcst_lead")) { - if (groups.length > 0) { - strGroupBy += " ,"; - } - strGroupBy += " fcst_lead"; - } - - String selectListStat = selectList.replaceAll("h\\.", "").replaceAll(",\\s+$", ""); - - // build the query - return - "SELECT " + selectListStat + ", " - + " '' object_id, " - + " '' object_cat, " - + " '" + stat + "' stat_name, " - + " " + strStat + " stat_value " - + " FROM " - + " mode_header , " - + " mode_obj_single , " - + " mode_cts " - + " WHERE " + strWhere + " " - + " AND simple_flag = 1 " - + " AND mode_obj_single.mode_header_id = mode_header.mode_header_id " - + " AND mode_cts.mode_header_id = mode_obj_single.mode_header_id " - + " AND mode_cts.field = 'OBJECT' " - + strGroupBy + ";"; - } - - /** - * Build SQL to gather mode pair data - * - * @param strSelectList - * @param stat - * @return - */ - private String buildModePairStatTable(String strSelectList, String strWhere, String stat) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = MVUtil.parseModeStat(stat); - if (2 != listStatParse.length) { - return ""; - } - String strStatFlag = listStatParse[1]; - - // build the object flag where clause - if (strStatFlag.charAt(0) != 'A') { - strWhere += " AND simple_flag = " + ('S' == strStatFlag.charAt(0) ? "1" : "0"); - } - if (strStatFlag.charAt(1) != 'A') { - strWhere += " AND matched_flag = " + ('M' == strStatFlag.charAt(1) ? "1" : "0"); - } - - // build the list of fields involved in the computations - String selectListStat = strSelectList.replaceAll("h\\.", ""); - String strGroupListMMI = selectListStat.replaceAll("HOUR\\([^\\)]+\\) ", ""); - strGroupListMMI = strGroupListMMI.replaceAll("if\\D+fcst_lead", "fcst_lead"); - // set the object_id field, depending on the stat - String objectId = "object_id"; - String objectIdName = "object_id"; - String strGroupBy = ""; - String statName = listStatParse[0]; - - if (statName.startsWith("MAXINT")) { - if (statName.equals("MAXINTF")) { - objectId = "SUBSTR(object_id, 1, LOCATE('_', object_id)-1) fcst_id"; - objectIdName = "fcst_id"; - } else if (statName.equals("MAXINTO")) { - objectId = "SUBSTR(object_id, LOCATE('_', object_id)+1) obs_id"; - objectIdName = "obs_id"; - } - strGroupBy = " GROUP BY " + strGroupListMMI + ", " + objectIdName; - } - - // set the table stat field, object_id pattern and group by clause, depending on the stat - String strTableStat = MVUtil.modePairStatField.get(statName); - strWhere = strWhere.replace("h.", ""); - // build the query - return - "SELECT " + selectListStat + ", " - + " " + objectId + ", " - + " object_cat, " - + " '" + stat + "' stat_name, " - + " " + strTableStat + " stat_value " - + " FROM " - + " mode_header, " - + " mode_obj_pair " - + " WHERE " + strWhere - + " AND mode_header.mode_header_id = mode_obj_pair.mode_header_id " - + strGroupBy + ";"; - } - - - /** - * Build SQL to gather mode pair data - * - * @param strSelectList - * @param stat - * @return - */ - private String buildMtd3dPairStatTable(String strSelectList, String strWhere, String stat) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = stat.split("_"); - String strStatFlag = listStatParse[listStatParse.length - 1]; - - // build the object flag where clause - if (strStatFlag.charAt(0) != 'A') { - strWhere += " AND simple_flag = " + ('S' == strStatFlag.charAt(0) ? "1" : "0"); - } - if (strStatFlag.charAt(1) != 'A') { - strWhere += " AND matched_flag = " + ('M' == strStatFlag.charAt(1) ? "1" : "0"); - } - - // build the list of fields involved in the computations - String strSelectListStat = strSelectList.replaceAll("h\\.", ""); - String strGroupListMMI = strSelectListStat.replaceAll("HOUR\\([^\\)]+\\) ", ""); - // set the object_id field, depending on the stat - String objectId = "object_id"; - String statName = stat.replace("_" + listStatParse[listStatParse.length - 1], ""); - - // set the table stat field, object_id pattern and group by clause, depending on the stat - String strTableStat = MVUtil.mtd3dPairStatField.get(statName); - - // build the query - return - "SELECT " + strSelectListStat + ", " - + " " + objectId + ", " - + " cluster_id, " - + " '" + stat + "' stat_name, " - + " " + strTableStat + " stat_value " - + " FROM mtd_header, mtd_3d_obj_pair " - + " WHERE " + strWhere - + " AND mtd_header.mtd_header_id = mtd_3d_obj_pair.mtd_header_id"; - } - - private String buildModeSingleStatTable( - String selectList, String strWhere, String stat, - String[] groups, boolean isEventEqualization) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = MVUtil.parseModeStat(stat); - if (2 != listStatParse.length) { - return ""; - } - - String strStatFlag = listStatParse[1]; - - - // build the object flag where clause - if (strStatFlag.charAt(0) != 'A') { - strWhere += " AND fcst_flag = " + ('F' == strStatFlag.charAt(0) ? "1" : "0"); - } - if (strStatFlag.charAt(1) != 'A') { - strWhere += " AND simple_flag = " + ('S' == strStatFlag.charAt(1) ? "1" : "0"); - } - if (strStatFlag.charAt(2) != 'A') { - strWhere += " AND matched_flag = " + ('M' == strStatFlag.charAt(2) ? "1" : "0"); - } - - // build the group by clause - String strGroupBy = ""; - String statName = listStatParse[0]; - if (statName.startsWith("CNT")) { - strGroupBy = " GROUP BY "; - for (int i = 0; i < groups.length; i++) { - strGroupBy += (0 < i ? ", " : "") + " " + groups[i]; - } - if (!statName.equals("CNTSUM") && !strGroupBy.contains("fcst_valid")) { - if (groups.length > 0) { - strGroupBy += " ,"; - } - strGroupBy += " fcst_valid"; - } - - //mandatory group by fcst_valid and fcst_lead for EE - if (isEventEqualization) { - if (!strGroupBy.contains("fcst_valid")) { - if (groups.length > 0) { - strGroupBy += " ,"; - } - strGroupBy += " fcst_valid"; - } - if (!strGroupBy.contains("fcst_lead")) { - if (groups.length > 0) { - strGroupBy += " ,"; - } - strGroupBy += " fcst_lead"; - } - } - } - String selectListStat = selectList.replaceAll("h\\.", ""); - // build the query - return - "SELECT " + selectListStat + ", " - + " object_id, " - + " object_cat, " - + " '" + stat + "' stat_name, " - + " " + MVUtil.modeSingleStatField.get(statName) + " stat_value " - + " FROM " - + " mode_header , " - + " mode_obj_single , " - + " mode_cts " - + " WHERE " + strWhere - + " AND mode_obj_single.mode_header_id = mode_header.mode_header_id " - + " AND mode_cts.mode_header_id = mode_obj_single.mode_header_id " - + " AND mode_cts.field = 'OBJECT'" - + strGroupBy; - } - - - private String buildMtd3dSingleStatTable( - String selectList, String strWhere, String stat) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = stat.split("_"); - String statFlag = listStatParse[listStatParse.length - 1]; - // build the list of fields involved in the computations - - - // build the object flag where clause - if (statFlag.charAt(0) != 'A') { - strWhere += " AND fcst_flag = " + ('F' == statFlag.charAt(0) ? "1" : "0"); - } - if (statFlag.charAt(1) != 'A') { - strWhere += " AND simple_flag = " + ('S' == statFlag.charAt(1) ? "1" : "0"); - } - if (statFlag.charAt(2) != 'A') { - strWhere += " AND matched_flag = " + ('M' == statFlag.charAt(2) ? "1" : "0"); - } - String statName = stat.replace("_" + listStatParse[listStatParse.length - 1], ""); - String selectListStat = selectList.replaceAll("h\\.", ""); - - - // build the query - return - "SELECT " + selectListStat + ", " - + " object_id, " - + " cluster_id, " - + " '" + stat + "' stat_name, " - + " " + mtd3dSingleStatField.get(statName) + " stat_value " - + " FROM mtd_header, mtd_3d_obj_single " - + " WHERE " + strWhere - + " AND mtd_header.mtd_header_id = mtd_3d_obj_single.mtd_header_id"; - } - - - private String buildMtd2dStatTable( - String selectList, String strWhere, String stat) { - - - String[] listStatParse = stat.split("_"); - String strStatFlag = listStatParse[listStatParse.length - 1]; - - // build the object flag where clause - if (strStatFlag.charAt(0) != 'A') { - strWhere += " AND fcst_flag = " + ('F' == strStatFlag.charAt(0) ? "1" : "0"); - } - if (strStatFlag.charAt(1) != 'A') { - strWhere += " AND simple_flag = " + ('S' == strStatFlag.charAt(1) ? "1" : "0"); - } - if (strStatFlag.charAt(2) != 'A') { - strWhere += " AND matched_flag = " + ('M' == strStatFlag.charAt(2) ? "1" : "0"); - } - - String strStatName = stat.replace("_" + listStatParse[listStatParse.length - 1], ""); - String strSelectListStat = selectList.replaceAll("h\\.", ""); - // build the query - return - "SELECT " + strSelectListStat + ", " - + " object_id, " - + " cluster_id, " - + " '" + stat + "' stat_name, " - + " " + MVUtil.mtd2dStatField.get(strStatName) + " stat_value " - + " FROM mtd_header, mtd_2d_obj " - + " WHERE " + strWhere - + " AND mtd_header.mtd_header_id = mtd_2d_obj.mtd_header_id"; - } - - private String buildModeSingleStatRatioTable(String selectList, String strWhere) { - - // build the list of fields involved in the computations - String strSelectListStat = selectList.replaceAll("h\\.", ""); - strWhere = strWhere.replaceAll("h\\.", ""); - - return - "SELECT " + strSelectListStat + ", " - + " object_id, " - + " object_cat, " - + " area, " - + " total, " - + " fcst_flag, " - + " simple_flag, " - + " matched_flag " - + " FROM " - + " mode_header , " - + " mode_obj_single , " - + " mode_cts " - + " WHERE " + strWhere - + " AND mode_obj_single.mode_header_id = mode_header.mode_header_id " - + " AND mode_cts.mode_header_id = mode_obj_single.mode_header_id" - + " AND mode_cts.field = 'OBJECT'"; - } - - private String buildMtdSingleStatRatio2dTable(String selectList, String strWhere) { - - // build the list of fields involved in the computations - String strSelectListStat = selectList.replaceAll("h\\.", ""); - - - return - "SELECT " + strSelectListStat + ", " - + " object_id, " - + " cluster_id, " - + " area, " - + " fcst_flag, " - + " simple_flag, " - + " matched_flag " - + " FROM mtd_header, mtd_2d_obj " - + " WHERE " + strWhere - + " AND mtd_header.mtd_header_id = mtd_2d_obj.mtd_header_id"; - - } - - private String buildMtdSingleStatRatio3dTable(String selectList, String strWhere) { - - // build the list of fields involved in the computations - String strSelectListStat = selectList.replaceAll("h\\.", ""); - - - return - "SELECT " + strSelectListStat + ", " - + " object_id, " - + " cluster_id, " - + " volume, " - + " fcst_flag, " - + " simple_flag, " - + " matched_flag " - + " FROM mtd_header, mtd_3d_obj_single " - + " WHERE " + strWhere - + " AND mtd_header.mtd_header_id = mtd_3d_obj_single.mtd_header_id"; - - } - - private String buildModeSingleStatDiffTable( - String strSelectList, String strWhere, String stat, - String table1, String table2, - Map.Entry[] listSeries) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = MVUtil.parseModeStat(stat); - if (2 != listStatParse.length) { - return ""; - } - String strStatName = listStatParse[0]; - - // build the list of fields involved in the computations - String strSelectListStat = strSelectList.replaceAll("h\\.", "s."); - - // modify the where clause to suit two tables - strWhere = strWhere.replaceAll("fcst_var", "s.fcst_var") + " AND s.fcst_var = s2.fcst_var"; - - - // set the table stat field, object_id pattern and group by clause, depending on the stat - String strTableStat = MVUtil.modeSingleStatField.get(strStatName); - String statName = strTableStat.split("\\(")[0]; - String[] strTableStats = new String[2]; - if (strTableStat.contains("object_id")) { - strTableStats[0] = statName + "( s.object_id)"; - strTableStats[1] = statName + "( s2.object_id)"; - } else { - strTableStats[0] = "s." + "stat_value"; - strTableStats[1] = "s2." + "stat_value"; - - } - - // build the query COUNT(object_id) - String result = - "SELECT " + strSelectListStat + ", " - + " s.object_id, " - + " s.object_cat, " - + " '" + stat + "' stat_name, " - + " " + strTableStats[0] + " - " + strTableStats[1] + " stat_value " - + " FROM (" - + table1 - + " ) s, ( " + table2 + " ) s2 " - + " WHERE " - + strWhere + " " - + " AND SUBSTRING(s.object_id, -3) = SUBSTRING(s2.object_id, -3) "; - if (!strTableStat.contains("object_id")) { - result = result + " AND " + "s.stat_value" + " != -9999" - + " AND " + "s2.stat_value" + " != -9999" - + " AND s.fcst_valid = s2.fcst_valid " - + " AND s.fcst_lead = s2.fcst_lead"; - for (int i = 0; i < listSeries.length; i++) { - result = result + " AND s." + listSeries[i].getKey() - + " = s2." + listSeries[i].getKey(); - } - result = result + ";"; - } - return result; - } - - private String buildMtd3dSingleStatDiffTable( - String strSelectList, String strWhere, - String stat, - String table1, String table2) { - - // parse the stat into the stat name and the object flags - // parse the stat into the stat name and the object flags - String[] listStatParse = stat.split("_"); - String strStatName = stat.replace("_" + listStatParse[listStatParse.length - 1], ""); - - // build the list of fields involved in the computations - String strSelectListStat = strSelectList.replaceAll("h\\.", "s."); - - // modify the where clause to suit two tables - strWhere = strWhere.replaceAll("fcst_var", "s.fcst_var") + " AND s.fcst_var = s2.fcst_var"; - - - // set the table stat field, object_id pattern and group by clause, depending on the stat - String strTableStat = mtd3dSingleStatField.get(strStatName); - String statName = strTableStat.split("\\(")[0]; - String[] strTableStats = new String[2]; - if (strTableStat.contains("object_id")) { - strTableStats[0] = statName + "( s.object_id)"; - strTableStats[1] = statName + "( s2.object_id)"; - } else { - strTableStats[0] = "s." + "stat_value"; - strTableStats[1] = "s2." + "stat_value"; - - } - - String result = - "SELECT " + strSelectListStat + ", " - + " s.object_id, " - + " s.cluster_id, " - + " '" + stat + "' stat_name, " - + " " + strTableStats[0] + " - " + strTableStats[1] + " stat_value " - + " FROM (" - + table1 - + " ) s, ( " + table2 + " ) s2 " - + " WHERE " - + strWhere + " " - + " AND SUBSTRING(s.object_id, LOCATE('_', s.object_id)+1) = SUBSTRING(s2.object_id, " - + "LOCATE('_', s " - + ".object_id)+1) "; - if (!strTableStat.contains("object_id")) { - result = result + " AND " + strTableStats[0] - + " != -9999 AND " + strTableStats[1] + " != -9999;"; - } - return result; - } - - private String buildMtd2dStatDiffTable( - String strSelectList, String strWhere, String stat, - String table1, String table2) { - - // parse the stat into the stat name and the object flags - String[] listStatParse = stat.split("_"); - String strStatName = stat.replace("_" + listStatParse[listStatParse.length - 1], ""); - - // build the list of fields involved in the computations - String strSelectListStat = strSelectList.replaceAll("h\\.", "s."); - - // modify the where clause to suit two tables - strWhere = strWhere.replaceAll("fcst_var", "s.fcst_var") + " AND s.fcst_var = s2.fcst_var"; - - - // set the table stat field, object_id pattern and group by clause, depending on the stat - String strTableStat = MVUtil.mtd2dStatField.get(strStatName); - String statName = strTableStat.split("\\(")[0]; - String[] strTableStats = new String[2]; - if (strTableStat.contains("object_id")) { - strTableStats[0] = statName + "( s.object_id)"; - strTableStats[1] = statName + "( s2.object_id)"; - } else { - strTableStats[0] = "s." + "stat_value"; - strTableStats[1] = "s2." + "stat_value"; - - } - - String result = - "SELECT " + strSelectListStat + ", " - + " s.object_id, " - + " s.cluster_id, " - + " '" + stat + "' stat_name, " - + " " + strTableStats[0] + " - " + strTableStats[1] + " stat_value " - + " FROM (" - + table1 - + " ) s, ( " + table2 + " ) s2 " - + " WHERE " - + strWhere + " " - + " AND SUBSTRING(s.object_id, LOCATE('_', s.object_id)+1) " - + "= SUBSTRING(s2.object_id, LOCATE('_', s.object_id)+1) "; - if (!strTableStat.contains("object_id")) { - result = result + " AND " + strTableStats[0] - + " != -9999 AND " + strTableStats[1] + " != -9999;"; - } - return result; - } - - - @Override - public List buildPlotModeEventEqualizeSql( - MVPlotJob job, MVOrderedMap mapPlotFixPerm, - MVOrderedMap mapPlotFixVal) - throws ValidationException { - MVOrderedMap fcstVarPat = new MVOrderedMap(); - - // determine if the plot job is for stat data or MODE data - List tableHeaderSqlType; - if (job.isModeJob()) { - tableHeaderSqlType = modeHeaderSqlType; - } else { - tableHeaderSqlType = statHeaderSqlType; - } - - - // populate the plot template values with plot_fix values - Map.Entry[] listPlotFixVal = MVUtil.buildPlotFixTmplMap(mapPlotFixPerm, mapPlotFixVal); - - // build the sql where clauses for the current permutation of fixed variables and values - String strPlotFixWhere = buildPlotFixWhere(listPlotFixVal, job, - job.isModeJob() || job.isMtdJob()); - - // add the user-specified condition clause, if present - if (null != job.getPlotCond() && job.getPlotCond().length() > 0) { - strPlotFixWhere += " AND " + job.getPlotCond() + " "; - } - - - // remove multiple dep group capability - MVOrderedMap[] listDep = job.getDepGroups(); - if (1 != listDep.length) { - throw new ValidationException("unexpected number of groups: " + listDep.length); - } - MVOrderedMap mapDepGroup = listDep[0]; - - - /* - * Build queries for statistics on both the y1 and y2 axes - */ - - List listSql = new ArrayList<>(); - for (int intY = 1; intY <= 2; intY++) { - - // get the dep values for the current dep group - MVOrderedMap mapDep = (MVOrderedMap) mapDepGroup.get("dep" + intY); - - // establish lists of entires for each group of variables and values - Map.Entry[] listSeries = (1 == intY ? job.getSeries1Val() : job.getSeries2Val()) - .getOrderedEntriesForSqlSeries(); - Map.Entry[] listDepPlot = mapDep.getOrderedEntries(); - - // if there is a mis-match between the presence of series and dep values, bail - if (0 < listDepPlot.length && 1 > listSeries.length) { - throw new ValidationException("dep values present, but no series values for Y" + intY); - } - if (1 > listDepPlot.length && 0 < listSeries.length) { - throw new ValidationException("series values present, but no dep values for Y" + intY); - } - - // there must be at least one y1 series and stat, but not for y2 - if (!job.getPlotTmpl().equals( - "eclv.R_tmpl") && 1 == intY && 1 > listDepPlot.length && 1 > listSeries.length) { - throw new ValidationException("no Y1 series stat found"); - } - if (2 == intY && 1 > listDepPlot.length && 1 > listSeries.length) { - continue; - } - - - /* - * Construct query components from the series variable/value pairs - */ - - // build the select list and where clauses for the series variables and values - - String strWhere = strPlotFixWhere; - BuildMysqlQueryStrings buildMysqlQueryStrings = build(job.isModeJob() || job.isMtdJob(), - tableHeaderSqlType, - listSeries, strWhere, true); - String selectList = buildMysqlQueryStrings.getSelectList(); - strWhere = buildMysqlQueryStrings.getWhere(); - - // if the fcst_valid or fcst_init fields - // are not present in the select list and temp table list, add them - if (!selectList.contains("fcst_init")) { - selectList += ", h.fcst_init"; - } - if (!selectList.contains("fcst_valid")) { - selectList += ", h.fcst_valid"; - } - BuildMysqlQueryStrings buildQueryPlotStrings = build(job.isModeJob() || job.isMtdJob(), - tableHeaderSqlType, - listSeries, strWhere, false); - String selectPlotList = buildQueryPlotStrings.getSelectList(); - // if the fcst_valid or fcst_init fields - // are not present in the select list and temp table list, add them - if (!selectPlotList.contains("fcst_init") && !selectPlotList.contains("init_hour")) { - if (job.isModeJob()) { - selectPlotList += ", h.fcst_init"; - } else { - selectPlotList += ", " + " ld.fcst_init_beg"; - } - } - if (!selectPlotList.contains("fcst_valid")) { - if (job.isModeJob()) { - selectPlotList += ", h.fcst_valid"; - } else { - selectPlotList += ", " + " ld.fcst_valid_beg"; - } - } - - if (!selectList.contains("fcst_lead")) { - if (job.getEventEqual()) { - selectList += ", " + " if( (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) is NULL , h.fcst_lead , h.fcst_lead " - + "+ (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) ) fcst_lead"; - } else { - selectList += ", " + " h.fcst_lead"; - } - selectPlotList += ", " + " h.fcst_lead"; - } - - /* - * Construct the query components for the independent variable and values - */ - - // validate and get the type and values for the independent variable - String strIndyVarType; - String strIndyVar = job.getIndyVar(); - String[] listIndyVal = job.getIndyVal(); - if (!tableHeaderSqlType.contains(strIndyVar)) { - throw new ValidationException("unrecognized indep " + (job.isModeJob() ? "mode" : "stat") - + "_header field: " + strIndyVar); - } - if (1 > listIndyVal.length) { - throw new ValidationException("no independent variable values specified"); - } - - // construct the select list item, where clause - // and temp table entry for the independent variable - if (!selectList.contains(strIndyVar)) { - selectList += ", " + formatField(strIndyVar, job.isModeJob() || job.isMtdJob(), true); - selectPlotList += ", " + formatField(strIndyVar, job.isModeJob() || job.isMtdJob(), - true); - } - String strIndyVarFormatted = formatField(strIndyVar, job.isModeJob() || job.isMtdJob(), - false); - if (strIndyVar.equals("fcst_lead") && job.getEventEqual()) { - strIndyVarFormatted = " if( (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) is NULL , " - + strIndyVarFormatted + " , " + strIndyVarFormatted - + " + (select fcst_lead_offset FROM model_fcst_lead_offset " - + " WHERE model = h.model) ) "; - } - strWhere += (!strWhere.isEmpty() ? " AND " : "") + strIndyVarFormatted + - " IN [" + MVUtil.buildValueList(job.getIndyVal()) + "] "; - - // add fcst_var to the select list and temp table entries - selectList += ", h.fcst_var"; - selectPlotList += ", h.fcst_var"; - - if (listPlotFixVal.length > 0) { - for (int i = 0; i < listPlotFixVal.length; i++) { - String strField = (String) listPlotFixVal[i].getKey(); - - } - } - - /* - * Construct a query for each fcst_var/stat pair - */ - - // determine how many queries are needed to gather that stat information - int intNumQueries; - String[][] listFcstVarStat = MVUtil.buildFcstVarStatList(mapDep); - - intNumQueries = listFcstVarStat.length; - - // build a query for each fcst_var/stat pair - // or a just single query for contingency tables or partial sums - for (int intFcstVarStat = 0; intFcstVarStat < intNumQueries; intFcstVarStat++) { - - // get the current fcst_var/stat pair - String strFcstVar = listFcstVarStat[intFcstVarStat][0]; - - // build the fcst_var where clause criteria - String strFcstVarClause = "= '" + strFcstVar + "'"; - Matcher matProb = MVUtil.prob.matcher(strFcstVar); - if (matProb.matches() && strFcstVar.contains("*")) { - Pattern patFcstVar = Pattern.compile( - strFcstVar.replace("*", ".*").replace("(", "\\(").replace(")", "\\)")); - if (!fcstVarPat.containsKey(patFcstVar)) { - fcstVarPat.put(patFcstVar, MVUtil.replaceSpecialChars(strFcstVar)); - } - strFcstVarClause = "LIKE '" + strFcstVar.replace("*", "%") + "'"; - } - - // build the mode SQL - String strWhereFcstVar = strWhere + " AND fcst_var " + strFcstVarClause; - strWhereFcstVar = strWhereFcstVar.replace("h.", ""); - selectList = selectList.replace("h.", ""); - listSql.addAll(buildModeStatEventEqualizeSql(selectList, strWhereFcstVar)); - - } - - } - - - //remove duplicated queries - listSql = new ArrayList<>(new LinkedHashSet<>(listSql)); - - return listSql; - } - - - private List buildModeStatEventEqualizeSql(String strSelectList, String strWhere) { - - List listQuery = new ArrayList(); - - listQuery.add(buildModeSingleStatRatioEventEqualizeTable(strSelectList, strWhere)); - - return listQuery; - } - - - private String buildModeSingleStatRatioEventEqualizeTable(String selectList, String strWhere) { - - // build the list of fields involved in the computations - String strSelectListStat = selectList.replaceAll("h\\.", ""); - - - return - "SELECT " + strSelectListStat + ", " - + " object_id, " - + " object_cat, " - + " area, " - + " total, " - + " fcst_flag, " - + " simple_flag, " - + " matched_flag " - + " FROM " - + " mode_header , " - + " mode_obj_single, " - + " mode_cts " - + " WHERE " + strWhere - + " AND mode_obj_single.mode_header_id = mode_header.mode_header_id" - + " AND mode_cts.mode_header_id = mode_obj_single.mode_header_id" - + " AND mode_cts.field = 'OBJECT'"; - } - - @Override - public String buildAndExecuteQueriesForHistJob( - MVPlotJob job, String strDataFile, - MVOrderedMap listPlotFixPerm, - PrintStream printStream, - PrintStream printStreamSql) throws ValidationException { - String strTempList = ""; - String strSelectList = ""; - String strWhereSeries = ""; - Map.Entry[] listSeries = job.getSeries1Val().getOrderedEntriesForSqlSeries(); - - - for (Map.Entry listSery : listSeries) { - // get the current series field and values - String strSeriesField = listSery.getKey().toString(); - String[] listSeriesVal = (String[]) listSery.getValue(); - // validate the series field and get its type - if (!statHeaderSqlType.contains(strSeriesField)) { - throw new ValidationException("unrecognized " + "stat" + "_header field: " + strSeriesField); - } - // build the select list element, where clause and temp table list element - strSelectList += (strSelectList.isEmpty() ? "" : ",") - + " " + formatField(strSeriesField, false, true); - strWhereSeries += " AND " + formatField(strSeriesField, false, false) - + " IN [" + MVUtil.buildValueList(listSeriesVal) + "] "; - - } - - // populate the template map with fixed values - Map.Entry[] listPlotFixVal = MVUtil.buildPlotFixTmplMap(listPlotFixPerm, job.getPlotFixVal()); - - - // build the stat_header where clauses of the sql - String strWhere = buildPlotFixWhere(listPlotFixVal, job, false); - String strNumSelect = ""; - String type = ""; - String table = ""; - String tableBins = ""; - String binColumnName = null; - - if (job.getPlotTmpl().startsWith("relp")) { - type = "ens"; - table = "relp"; - tableBins = "relp_ens"; - } else if (job.getPlotTmpl().startsWith("rhist")) { - type = "rank"; - table = "rhist"; - tableBins = "rhist_rank"; - } else if (job.getPlotTmpl().startsWith("phist")) { - type = "bin"; - table = "phist"; - tableBins = "phist_bin"; - binColumnName = "bin_size"; - } - strWhere = strWhere.replaceAll("h\\.n_" + type, "ld.n_" + type); - - String dbList = "["; - - for (String cdbName : job.getCurrentDBName()) { - if (dbList.length() > 1) { - dbList += ", "; - } - dbList += "\'" + cdbName + "\'"; - } - dbList += "]"; - - // find the line_data documents where the header has the selected forecast variable - strNumSelect = "SELECT DISTINCT ld.n_" + type - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld on ld.header_id = meta(h).id" - + " WHERE " + strWhere - + " AND ld.type = \'line\' AND ld.line_type = \'" + table + "\'" - + " AND h.type = \'header\' AND h.header_type = \'stat\' " - + " AND h.dbname IN " + dbList; - - if (printStreamSql != null) { - printStreamSql.println(strNumSelect + " "); - printStreamSql.flush(); - } - - - // run the rank number query and warn, if necessary - String strMsg = ""; - List listNum = getNumbers(strNumSelect, dbList); - - - if (listNum.isEmpty()) { - throw new ValidationException("no " + type + " data found"); - } else if (1 < listNum.size()) { - strMsg = " ** WARNING: multiple n_" + type + " values found for search criteria: "; - for (int i = 0; i < listNum.size(); i++) { - strMsg += (0 < i ? ", " : "") + listNum.get(i); - } - printStream.println(strMsg); - } - - - // build a query for the rank data - strWhere = strWhere + strWhereSeries; - String strPlotDataSelect = - "SELECT hbin.i_value, "; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + strSelectList + ", "; - } - - - strPlotDataSelect = strPlotDataSelect + " SUM(TONUMBER(hbin." + type + "_i)) AS stat_value"; - - if (binColumnName != null) { - strPlotDataSelect = strPlotDataSelect + ", ld." + binColumnName; - } - strPlotDataSelect = strPlotDataSelect - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld on ld.header_id = meta(h).id" - + " UNNEST ld." + tableBins + "[0:" + listNum.get(0) + "] AS hbin" - + " WHERE " + strWhere - + " AND ld.type = \'line\' AND ld.line_type = \'" + table + "\'" - + " AND h.type = \'header\' AND h.header_type = \'stat\'" - + " AND h.dbname IN " + dbList - + " GROUP BY hbin.i_value"; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + ", " + strSelectList; - } - // for phist plots with bin_size, add to group by as well as select - if (binColumnName != null) { - strPlotDataSelect = strPlotDataSelect + ", ld." + binColumnName; - } - strPlotDataSelect = strPlotDataSelect + " ORDER BY TONUMBER(hbin.i_value);"; - if (printStreamSql != null) { - printStreamSql.println(strPlotDataSelect + " "); - printStreamSql.flush(); - } - - // get the data for the current plot from the plot_data temp table and write it to a data file - List queries = new ArrayList<>(1); - queries.add(strPlotDataSelect); - for (int i = 0; i < job.getCurrentDBName().size(); i++) { - MvResponse mvResponse = executeQueriesAndSaveToFile(queries, strDataFile, - job.getCalcCtc() || job.getCalcSl1l2() - || job.getCalcSal1l2() || job.getCalcGrad(), - job.getCurrentDBName().get(i), i == 0); - if (mvResponse.getInfoMessage() != null) { - printStream.println(mvResponse.getInfoMessage()); - } - printStream.println(); - } - return strMsg; - } - - - @Override - public int buildAndExecuteQueriesForRocRelyJob( - MVPlotJob job, String strDataFile, - MVOrderedMap listPlotFixPerm, - PrintStream printStream, - PrintStream printStreamSql) throws ValidationException { - String strSelectList = ""; - String strTempList = ""; - String strWhereSeries = ""; - - Map.Entry[] listSeries = job.getSeries1Val().getOrderedEntriesForSqlSeries(); - - for (Map.Entry listSery : listSeries) { - // get the current series field and values - String strSeriesField = listSery.getKey().toString(); - String[] listSeriesVal = (String[]) listSery.getValue(); - // validate the series field and get its type - if (!statHeaderSqlType.contains(strSeriesField)) { - throw new ValidationException("unrecognized " + "stat" + "_header field: " + strSeriesField); - } - // build the select list element, where clause and temp table list element - strSelectList += (strSelectList.isEmpty() ? "" : ",") - + " " + formatField(strSeriesField, false, true); - strWhereSeries += " AND " + formatField(strSeriesField, false, false) - + " IN [" + MVUtil.buildValueList(listSeriesVal) + "] "; - - } - - - // populate the template map with fixed values - Map.Entry[] listPlotFixVal = MVUtil.buildPlotFixTmplMap(listPlotFixPerm, job.getPlotFixVal()); - - boolean boolRelyPlot = job.getPlotTmpl().startsWith("rely"); - - // build the stat_header where clauses of the sql - String strWhere = buildPlotFixWhere(listPlotFixVal, job, false); - strWhere = strWhere + strWhereSeries; - - String dbList = "["; - - for (String cdbName : job.getCurrentDBName()) { - if (dbList.length() > 1) { - dbList += ", "; - } - dbList += "\'" + cdbName + "\'"; - } - dbList += "]"; - - // check to ensure only a single obs_thresh is used - String strObsThreshSelect = - "SELECT DISTINCT h.obs_thresh " - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket().name() + "` AS ld ON ld.header_id = meta(h).id" - + " WHERE " + strWhere + (strWhere.equals("") ? "" : " AND ") - + " ld.type = \'line\' AND ld.line_type = " - + (boolRelyPlot || job.getRocPct() ? "\'pct\'" : "\'ctc\'") - + " AND h.type = \'header\' AND h.header_type = \'stat\'" - + " AND h.dbname IN " + dbList - + " ORDER BY h.obs_thresh;"; - - if (printStreamSql != null) { - printStreamSql.println(strObsThreshSelect + " "); - } - - - // build the query depending on the type of data requested - String strPlotDataSelect = ""; - List listFcstThresh = new ArrayList<>(); - if (boolRelyPlot || job.getRocPct()) { - - // check to ensure only a single fcst_thresh is used - String strFcstThreshSelect = "SELECT DISTINCT h.fcst_thresh "; - - strFcstThreshSelect = strFcstThreshSelect - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld ON ld.header_id = meta(h).id" - + " WHERE " + strWhere + (strWhere.equals("") ? "" : " AND ") - + " ld.type = \'line\' AND ld.line_type = " - + (boolRelyPlot || job.getRocPct() ? "\'pct\'" : "\'ctc\'") - + " AND h.type = \'header\' AND h.header_type = \'stat\'" - + " AND h.dbname IN " + dbList - + " ORDER BY h.fcst_thresh;"; - - - if (printStreamSql != null) { - printStreamSql.println(strFcstThreshSelect + " "); - } - - listFcstThresh = getNumbers(strFcstThreshSelect, job.getCurrentDBName().get(0)); - - - // build the plot data sql - strPlotDataSelect = - "SELECT "; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + strSelectList + ", "; - } - if (boolRelyPlot) { - strPlotDataSelect = strPlotDataSelect - + " thresh.i_value," - + " TONUMBER(thresh.thresh_i) AS thresh_i," - + " thresh.oy_i," - + " thresh.on_i"; - - strPlotDataSelect = strPlotDataSelect - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld on ld.header_id = meta(h).id" - + " UNNEST ld.pct_thresh AS thresh" - + " WHERE " + strWhere + (strWhere.equals("") ? "" : " AND ") - + " ld.type = \'line\' AND ld.line_type = \'pct\'" - + " AND h.type = \'header\' AND h.header_type = \'stat\'" - + " AND h.dbname IN " + dbList - + " ORDER BY thresh_i"; - } else { - strPlotDataSelect = strPlotDataSelect - + " thresh.i_value, " - + " thresh.thresh_i, " - + " SUM(TONUMBER(thresh.oy_i)) oy_i, " - + " SUM(TONUMBER(thresh.on_i)) on_i "; - - strPlotDataSelect = strPlotDataSelect - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld on ld.header_id = meta(h).id" - + " UNNEST ld.pct_thresh AS thresh" - + " WHERE " + strWhere + (strWhere.equals("") ? "" : " AND ") - + " ld.type = \'line\' AND ld.line_type = \'pct\'" - + " AND h.type = \'header\' AND h.header_type = \'stat\'" - + " AND h.dbname IN " + dbList - + " GROUP BY thresh.i_value, thresh.thresh_i" - + " ORDER BY thresh.i_value"; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + ", " + strSelectList; - } - strPlotDataSelect = strPlotDataSelect + ";"; - } - - } else if (job.getRocCtc()) { - - strPlotDataSelect = - "SELECT h.fcst_thresh thresh, "; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + strSelectList + ", "; - } - strPlotDataSelect = strPlotDataSelect - + " SUM(tonumber(ld.fy_oy)) fy_oy, " - + " SUM(tonumber(ld.fy_on)) fy_on, " - + " SUM(tonumber(ld.fn_oy)) fn_oy, " - + " SUM(tonumber(ld.fn_on)) fn_on " - + " FROM `" + getBucket().name() + "` AS h " - + " INNER JOIN `" + getBucket() - .name() + "` AS ld ON ld.header_id = meta(h).id " - + " WHERE " + strWhere + " AND ld.type = \'line\' AND ld.line_type = \'ctc\' " - + " AND h.type = \'header\' AND h.header_type = \'stat\' " - + " AND h.dbname IN " + dbList - + " GROUP BY " - + " h.fcst_thresh"; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + ", " + strSelectList; - } - strPlotDataSelect = strPlotDataSelect + ";"; - - } - - if (printStreamSql != null) { - printStreamSql.println(strPlotDataSelect + " "); - } - - - // if the query does not return data from a expected obs_thresh, throw an error - int intNumDepSeries = 1; - Map.Entry[] listSeries1Val = job.getSeries1Val().getOrderedEntriesForSqlSeries(); - for (Map.Entry aListSeries1Val : listSeries1Val) { - String[] listVal = (String[]) aListSeries1Val.getValue(); - intNumDepSeries *= listVal.length; - } - List listObsThresh = getNumbers(strObsThreshSelect, job.getCurrentDBName().get(0)); - if (intNumDepSeries < listObsThresh.size()) { - String obsThreshMsg = "ROC/Reliability plots must contain data from only a single obs_thresh," - + " instead found " + listObsThresh.size(); - for (int i = 0; i < listObsThresh.size(); i++) { - obsThreshMsg += (0 == i ? ": " : ", ") + listObsThresh.toString(); - } - throw new ValidationException(obsThreshMsg); - } - - if (listObsThresh.isEmpty()) { - String strObsThreshMsg = "ROC/Reliability plots must contain data " - + "from at least one obs_thresh "; - throw new ValidationException(strObsThreshMsg); - } - - // if the query for a PCT plot does not return data from a single fcst_thresh, throw an error - if (job.getRocPct() && intNumDepSeries < listFcstThresh.size()) { - String fcstThreshMsg = "ROC/Reliability plots using PCTs must contain data " - + "from only a single fcst_thresh, " - + "instead found " + listFcstThresh.size(); - for (int i = 0; i < listFcstThresh.size(); i++) { - fcstThreshMsg += (0 == i ? ":" : "") + " " + listFcstThresh.toString(); - } - throw new ValidationException(fcstThreshMsg.toString()); - } - if (job.getRocPct() && listObsThresh.isEmpty()) { - String strObsThreshMsg = "ROC/Reliability plots must contain data " - + "from at least one obs_thresh "; - throw new ValidationException(strObsThreshMsg); - } - - // get the data for the current plot from the plot_data temp table and write it to a data file - List queries = new ArrayList<>(1); - queries.add(strPlotDataSelect); - for (int i = 0; i < job.getCurrentDBName().size(); i++) { - MvResponse mvResponse = executeQueriesAndSaveToFile(queries, strDataFile, - job.isCalcStat(), - job.getCurrentDBName().get(i), - i == 0); - if (mvResponse.getInfoMessage() != null) { - printStream.println(mvResponse.getInfoMessage()); - } - } - - return intNumDepSeries; - } - - @Override - public int buildAndExecuteQueriesForEclvJob( - MVPlotJob job, String strDataFile, - MVOrderedMap listPlotFixPerm, - PrintStream printStream, - PrintStream printStreamSql) throws ValidationException { - String strSelectList = ""; - String strTempList = ""; - String strWhereSeries = ""; - - Map.Entry[] listSeries = job.getSeries1Val().getOrderedEntriesForSqlSeries(); - - for (Map.Entry listSery : listSeries) { - // get the current series field and values - String strSeriesField = listSery.getKey().toString(); - String[] listSeriesVal = (String[]) listSery.getValue(); - // validate the series field and get its type - if (!statHeaderSqlType.contains(strSeriesField)) { - throw new ValidationException("unrecognized " + "stat" + "_header field: " + strSeriesField); - } - // build the select list element, where clause and temp table list element - strSelectList += (strSelectList.isEmpty() ? "" : ",") - + " " + formatField(strSeriesField, false, true); - strWhereSeries += " AND " + formatField(strSeriesField, false, false) - + " IN [" + MVUtil.buildValueList(listSeriesVal) + "] "; - - - } - if (!strSelectList.toString().contains("fcst_valid")) { - strSelectList += ", " + " ld.fcst_valid_beg"; - } - if (!strSelectList.toString().contains("fcst_lead")) { - strSelectList += ", " + " ld.fcst_lead"; - } - - - // populate the template map with fixed values - Map.Entry[] listPlotFixVal = MVUtil.buildPlotFixTmplMap(listPlotFixPerm, job.getPlotFixVal()); - - - // build the stat_header where clauses of the sql - String strWhere = buildPlotFixWhere(listPlotFixVal, job, false); - - strWhere = strWhere.replaceAll("h\\.n_pnt", "ld.n_pnt"); - String strNumSelect = - "SELECT DISTINCT " - + " ld.n_pnt " - + " FROM " - + " stat_header h, " - + " line_data_eclv ld " - + " WHERE " - + strWhere - + " AND h.stat_header_id = ld.stat_header_id;"; - - if (printStreamSql != null) { - printStreamSql.println(strNumSelect + " "); - } - // run the rank number query and warn, if necessary - String strMsg = ""; - List listNum = getNumbers(strNumSelect, job.getCurrentDBName().get(0)); - - - if (listNum.isEmpty()) { - throw new ValidationException("no pnt data found"); - } else if (1 < listNum.size()) { - strMsg = " ** WARNING: multiple n_pnt values found for search criteria: "; - for (int i = 0; i < listNum.size(); i++) { - strMsg += (0 < i ? ", " : "") + listNum.get(i); - } - printStream.println(strMsg); - } - - - Map pctThreshInfo; - MVOrderedMap[] series = MVUtil.permute(job.getSeries1Val().convertFromSeriesMap()).getRows(); - for (int seriesInd = 0; seriesInd < series.length; seriesInd++) { - MVOrderedMap ser = series[seriesInd]; - String[] serName = ser.getKeyList(); - for (int serNameInd = 0; serNameInd < serName.length; serNameInd++) { - String strSelPctThresh = "SELECT DISTINCT ld.n_pnt FROM stat_header h, " - + "line_data_eclv ld, line_data_eclv_pnt ldt WHERE " - + serName[serNameInd] + " = '" - + ser.getStr(serName[serNameInd]) + "' AND " - + strWhere - + " AND ld.stat_header_id = h.stat_header_id " - + "AND ld.line_data_id = ldt.line_data_id;"; - printStreamSql.println(strSelPctThresh + " "); - - // run the PCT thresh query - pctThreshInfo = getPctThreshInfo(strSelPctThresh, job.getCurrentDBName().get(0)); - if (1 != pctThreshInfo.get("numPctThresh")) { - throw new ValidationException("number of ECLV pnts (" + pctThreshInfo.get( - "numPctThresh") + ") not distinct for " + serName[serNameInd] + " = '" + ser.getStr( - serName[serNameInd])); - } else if (1 > pctThreshInfo.get("numPctThresh")) { - throw new ValidationException("invalid number of ECLV pnts (" + pctThreshInfo.get( - "numPctThresh") + ") found for" + serName[serNameInd] - + " = '" + ser.getStr(serName[serNameInd]) + "'"); - } - } - } - strWhere = strWhere + strWhereSeries; - - // build the query depending on the type of data requested - String strPlotDataSelect = ""; - - // build the plot data sql - strPlotDataSelect = - "SELECT "; - if (listSeries.length > 0) { - strPlotDataSelect = strPlotDataSelect + strSelectList + ", "; - } - if (listPlotFixVal.length > 0) { - for (Map.Entry aListPlotFixVal : listPlotFixVal) { - String strField = (String) aListPlotFixVal.getKey(); - strPlotDataSelect = strPlotDataSelect + strField + ", "; - } - } - - strPlotDataSelect = strPlotDataSelect - + " ldt.x_pnt_i, " - + " ldt.y_pnt_i "; - - strPlotDataSelect = strPlotDataSelect + " FROM " - + " stat_header h, " - + " line_data_eclv ld, " - + " line_data_eclv_pnt ldt " - + " WHERE " - + strWhere - + " AND h.stat_header_id = ld.stat_header_id " - + " AND ld.line_data_id = ldt.line_data_id"; - - strPlotDataSelect = strPlotDataSelect + ";"; - - - if (printStreamSql != null) { - printStreamSql.println(strPlotDataSelect + " "); - } - - - // if the query does not return data from a expected obs_thresh, throw an error - int intNumDepSeries = 1; - Map.Entry[] listSeries1Val = job.getSeries1Val().getOrderedEntriesForSqlSeries(); - for (Map.Entry aListSeries1Val : listSeries1Val) { - String[] listVal = (String[]) aListSeries1Val.getValue(); - intNumDepSeries *= listVal.length; - } - - - // get the data for the current plot from the plot_data temp table and write it to a data file - List queries = new ArrayList<>(1); - queries.add(strPlotDataSelect); - for (int i = 0; i < job.getCurrentDBName().size(); i++) { - MvResponse mvResponse = executeQueriesAndSaveToFile(queries, strDataFile, - job.isCalcStat(), - job.getCurrentDBName().get(i), - i == 0); - if (mvResponse.getInfoMessage() != null) { - printStream.println(mvResponse.getInfoMessage()); - } - } - - return intNumDepSeries; - } - - /** - * Format the input field according to the table that it is stored in, and whether or not it needs - * to be derived or formatted as a date. Stat fields must be differentiated from MODE fields. - * Also, fields intended for the select list, as opposed to a where clause, must be specified. - * - * @param field MET output header field to format - * @param mode specifies whether or not the field is a stat_header or mode_header field - * @param fmtSel specifies whether the formatted field will be used for the select list (true) or - * a where clause (false) - * @return the formatted field - */ - private String formatField(String field, boolean mode, boolean fmtSel) { - if (field.equals("init_hour")) { - return (mode ? "HOUR(h.fcst_init)" : "HOUR(ld.fcst_init_beg)") + (fmtSel ? " init_hour" : ""); - } else if (field.equals("valid_hour")) { - return (mode ? "HOUR(h.fcst_valid)" : "HOUR(ld.fcst_valid_beg)") - + (fmtSel ? " valid_hour" : ""); - } else if (field.equals("fcst_init") && fmtSel) { - return " fcst_init"; - } else if (field.equals("fcst_init_beg") && fmtSel) { - return " fcst_init_beg"; - } else if (field.equals("fcst_init_beg") && !fmtSel) { - return "ld.fcst_init_beg"; - } else if (field.equals("fcst_valid") && fmtSel) { - return " fcst_valid"; - } else if (field.equals("fcst_valid_beg") && fmtSel) { - return " fcst_valid_beg"; - } else if (field.equals("fcst_valid_beg") && !fmtSel) { - return "ld.fcst_valid_beg"; - } else if (field.equals("fcst_lead")) { - return mode ? "h.fcst_lead" : "ld.fcst_lead"; - } else { - return "h." + field; - } - } - - - private String formatField(String field, boolean mode) { - return formatField(field, mode, true); - } - - private BuildMysqlQueryStrings build( - boolean boolModePlot, - List tableHeaderSqlType, - Map.Entry[] listSeries, String strWhere, - boolean isFormatSelect) throws ValidationException { - - BuildMysqlQueryStrings buildMysqlQueryStrings = new BuildMysqlQueryStrings(boolModePlot, - tableHeaderSqlType, - listSeries, strWhere, - isFormatSelect); - for (Map.Entry entry : listSeries) { - - // get the current series field and values - String field = entry.getKey().toString(); - - // validate the series field and get its type - String strTempType; - if (!tableHeaderSqlType.contains(field)) { - throw new ValidationException("unrecognized " + (boolModePlot ? "mode" : "stat") - + "_header field: " + field); - } - //strTempType = tableHeaderSqlType.get(field); - - // build the select list element, where clause and temp table list element - if (buildMysqlQueryStrings.getSelectList().length() == 0) { - if (isFormatSelect) { - buildMysqlQueryStrings.setSelectList( - buildMysqlQueryStrings.getSelectList() + " " - + formatField(field, boolModePlot, true)); - } else { - buildMysqlQueryStrings - .setSelectList(buildMysqlQueryStrings.getSelectList() + " " + field); - - } - } else { - if (isFormatSelect) { - buildMysqlQueryStrings.setSelectList( - buildMysqlQueryStrings.getSelectList() + ", " - + " " + formatField(field, boolModePlot, true)); - - } else { - buildMysqlQueryStrings - .setSelectList(buildMysqlQueryStrings.getSelectList() + ", " + " " + field); - } - } - buildMysqlQueryStrings.setWhere( - buildMysqlQueryStrings.getWhere() - + (buildMysqlQueryStrings.getWhere().isEmpty() ? " " : " AND ") - + formatField(field, boolModePlot, false) - + " IN [" + MVUtil.buildValueList((String[]) entry.getValue()) + "] "); - - - } - return buildMysqlQueryStrings; - } - -} - diff --git a/java/edu/ucar/metviewer/db/couchbase/CBDatabaseManager.java b/java/edu/ucar/metviewer/db/couchbase/CBDatabaseManager.java deleted file mode 100644 index 9be9bed4..00000000 --- a/java/edu/ucar/metviewer/db/couchbase/CBDatabaseManager.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * MysqlDatabaseManager.java Copyright UCAR (c) 2017. University Corporation for Atmospheric Research (UCAR), National Center for Atmospheric Research (NCAR), - * Research Applications Laboratory (RAL), P.O. Box 3000, Boulder, Colorado, 80307-3000, USA.Copyright UCAR (c) 2017. - */ - -package edu.ucar.metviewer.db.couchbase; - - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.TreeMap; - -import com.couchbase.client.core.CouchbaseException; -import com.couchbase.client.java.Bucket; -import com.couchbase.client.java.Cluster; -import com.couchbase.client.java.CouchbaseCluster; -import com.couchbase.client.java.document.json.JsonObject; -import com.couchbase.client.java.env.CouchbaseEnvironment; -import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; -import com.couchbase.client.java.query.N1qlQuery; -import com.couchbase.client.java.query.N1qlQueryResult; -import com.couchbase.client.java.query.N1qlQueryRow; -import edu.ucar.metviewer.MVUtil; -import edu.ucar.metviewer.db.DatabaseInfo; -import edu.ucar.metviewer.db.DatabaseManager; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * @author : tatiana $ - * @version : 1.0 : 23/05/17 09:51 $ - */ -public class CBDatabaseManager extends DatabaseManager { - - private static final Logger logger = LogManager.getLogger("CBDatabaseManager"); - protected static final String DB_PREFIX_MV = "mv_"; - protected static Map listDB = new TreeMap<>(); - protected static Map> groupToDatabases = new HashMap<>(); - private static Bucket bucket = null; - protected static final java.text.SimpleDateFormat DATE_FORMAT = - new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US); - protected static final java.time.format.DateTimeFormatter DATE_FORMAT_1 - = java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - - public CBDatabaseManager(DatabaseInfo databaseInfo, String password) throws CouchbaseException { - super(databaseInfo); - - // hardcoded bucket for now - change to command line option? XML tag? - String bucketName = "testvsdb"; - CouchbaseEnvironment env; - try { - env = DefaultCouchbaseEnvironment.builder() - .connectTimeout(40000) //20000ms = 20s, default is 5s - .queryTimeout(300000) //20000ms = 20s, default is 75000ms - .build(); - Cluster cluster = CouchbaseCluster.create(env, getDatabaseInfo().getHost()); - cluster.authenticate(getDatabaseInfo().getUser(), password); - bucket = cluster.openBucket(bucketName); - } catch (CouchbaseException e) { - logger.debug(e); - logger.error("Open bucket connection for a Couchbase database did not succeed."); - logger.error(e.getMessage()); - } - boolean updateGroups = false; - if (databaseInfo.getDbName() == null) { - updateGroups = true; - } - initDBList(updateGroups); - } - - public void initDBList(boolean updateGroups) { - N1qlQueryResult queryResult = null; - List results = null; - JsonObject firstRowObject = null; - - listDB.clear(); - - // get list of database names from data file documents - String nquery = "select distinct dbname as database_name " + - "from `" + - getBucket().name() + - "` where type = \'file\';"; - try { - queryResult = getBucket().query(N1qlQuery.simple(nquery)); - results = queryResult.allRows(); - - for (N1qlQueryRow row : results) { - firstRowObject = row.value(); - listDB.put(firstRowObject.get("database_name").toString(), ""); - } - } catch (Exception e) { - logger.error(e.getMessage()); - } - if (updateGroups) { - - //init groups - groupToDatabases.clear(); - - //for each database find a group - for (Map.Entry database : listDB.entrySet()) { - String[] metadata = getDatabaseMetadata(database.getKey()); - database.setValue(metadata[1]); - - if (!groupToDatabases.containsKey(metadata[0])) { - groupToDatabases.put(metadata[0], new ArrayList<>()); - } - - groupToDatabases.get(metadata[0]).add(database.getKey()); - } - - } - } - - private String[] getDatabaseMetadata(String database) { - N1qlQueryResult queryResult = null; - List queryList = null; - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - String group = MVUtil.DEFAULT_DATABASE_GROUP; - String description = ""; - - String strDataFileQuery = "SELECT " + - "meta().id as groupId, " + - "type, " + - "`group`, " + - "description " + - "FROM `" + - getBucket().name() + - "` WHERE " + - "type = \'category\' AND " + - "dbname = \'" + database + "\';"; - - try { - queryResult = getBucket().query(N1qlQuery.simple(strDataFileQuery)); - queryList = queryResult.allRows(); - - // if a category document is present for the database, get the database group and description and the ID - if (queryList.size() > 0) { - firstRow = queryList.get(0); - firstRowObject = firstRow.value(); - group = firstRowObject.get("group").toString(); - description = firstRowObject.get("description").toString(); - } - - } catch (CouchbaseException e) { - System.out.println(e.getMessage()); - } - - return new String[]{group, description}; - } - - - /** - * checks if a database with specified name exists - * - * @param db - name of the database to check - * @return - is database valid - */ - public boolean validate(String db) { - boolean result = false; - for (String availableDB : listDB.keySet()) { - if (availableDB.equals(db)) { - result = true; - break; - } - } - return result; - } - - /** - * Returns a connection to couchbase - * - * @return - connection (for couchbase a bucket) - */ - protected Bucket getBucket() { - /* CouchbaseEnvironment env; - if (bucket == null) { -// CouchbaseEnvironment env = DefaultCouchbaseEnvironment.builder() -// .connectTimeout(40000) //20000ms = 20s, default is 5s -// .build(); - // hardcoded bucket for now - change to command line option? XML tag? - String bucketName = "testvsdb"; - try { - env = DefaultCouchbaseEnvironment.builder() - .connectTimeout(40000) //20000ms = 20s, default is 5s - .queryTimeout(300000) //20000ms = 20s, default is 75000ms - .build(); - Cluster cluster = CouchbaseCluster.create(env, getDatabaseInfo().getHost()); - cluster.authenticate(getDatabaseInfo().getUser(), password); - bucket = cluster.openBucket(bucketName); - } catch (CouchbaseException e) { - logger.debug(e); - logger.error("Open bucket connection for a Couchbase database did not succeed."); - logger.error(e.getMessage()); - } - }*/ - return bucket; - } -} diff --git a/java/edu/ucar/metviewer/db/couchbase/CBLoadDatabaseManager.java b/java/edu/ucar/metviewer/db/couchbase/CBLoadDatabaseManager.java deleted file mode 100644 index 16d29067..00000000 --- a/java/edu/ucar/metviewer/db/couchbase/CBLoadDatabaseManager.java +++ /dev/null @@ -1,3110 +0,0 @@ -/** - * CBLoadDatabaseManager.java Copyright UCAR (c) 2017. University Corporation for Atmospheric - * Research (UCAR), National Center for Atmospheric Research (NCAR), Research Applications - * Laboratory (RAL), P.O. Box 3000, Boulder, Colorado, 80307-3000, USA.Copyright UCAR (c) 2017. - */ - -package edu.ucar.metviewer.db.couchbase; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.couchbase.client.core.CouchbaseException; -import com.couchbase.client.java.document.JsonDocument; -import com.couchbase.client.java.document.json.JsonArray; -import com.couchbase.client.java.document.json.JsonObject; -import com.couchbase.client.java.query.N1qlQuery; -import com.couchbase.client.java.query.N1qlQueryResult; -import com.couchbase.client.java.query.N1qlQueryRow; -import edu.ucar.metviewer.DataFileInfo; -import edu.ucar.metviewer.DatabaseException; -import edu.ucar.metviewer.MVLoadJob; -import edu.ucar.metviewer.MVLoadStatInsertData; -import edu.ucar.metviewer.MVOrderedMap; -import edu.ucar.metviewer.MVUtil; -import edu.ucar.metviewer.db.DatabaseInfo; -import edu.ucar.metviewer.db.LoadDatabaseManager; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * @author : tatiana $ - * @version : 1.0 : 06/06/17 11:19 $ - */ -public class CBLoadDatabaseManager extends CBDatabaseManager implements LoadDatabaseManager { - - private static final Logger logger = LogManager.getLogger("CBLoadDatabaseManager"); - protected static final DateTimeFormatter DB_DATE_STAT_FORMAT = DateTimeFormatter - .ofPattern("yyyyMMdd_HHmmss"); - - private final Pattern patIndexName = Pattern.compile("#([\\w\\d]+)#([\\w\\d]+)"); - private final Map statHeaders = new HashMap<>(); - - private static final int INDEX_LINE_DATA = 1; - private static final int INDEX_VAR_LENGTH = 3; - - // entity separator. Something not contained in the data. cannot be comma. - private static final String ESEP = "~"; - - private static final double[] X_POINTS_FOR_ECON = new double[]{ - 0.952380952, 0.909090909, 0.800000000, 0.666666667, 0.500000000, 0.333333333, - 0.200000000, 0.125000000, 0.100000000, 0.055555556, 0.037037037, 0.025000000, - 0.016666667, 0.011111111, 0.007142857, 0.004761905, 0.002857143, 0.002000000 - }; - - private final Map modeHeaders = new HashMap<>(); - private final Map mtdHeaders = new HashMap<>(); - private final String[] modeObjSingleColumns = new String[]{ - "OBJECT_CAT", "CENTROID_X", "CENTROID_Y", "CENTROID_LAT", "CENTROID_LON", "AXIS_ANG", "LENGTH", "WIDTH", "AREA", "AREA_THRESH", "CURVATURE", "CURVATURE_X", "CURVATURE_Y", "COMPLEXITY", "INTENSITY_10", "INTENSITY_25", "INTENSITY_50", "INTENSITY_75", "INTENSITY_90", "INTENSITY_50", "INTENSITY_SUM"}; - - private final String[] mtdObj2dColumns = new String[]{ - "OBJECT_CAT", "TIME_INDEX", "AREA", "CENTROID_X", "CENTROID_Y", "CENTROID_LAT", - "CENTROID_LON", "AXIS_ANG" - }; - private final String[] mtdObj3dSingleColumns = new String[]{ - "OBJECT_CAT", "CENTROID_X", "CENTROID_Y", "CENTROID_T", "CENTROID_LAT", - "CENTROID_LON", "X_DOT", "Y_DOT ", "AXIS_ANG", "VOLUME", "START_TIME", "END_TIME", "CDIST_TRAVELLED", "INTENSITY_10", "INTENSITY_25", "INTENSITY_50", "INTENSITY_75", "INTENSITY_90" - }; - private final String[] listLineDataTables = { - "line_data_fho", "line_data_ctc", "line_data_cts", "line_data_cnt", "line_data_pct", - "line_data_pstd", "line_data_pjc", "line_data_prc", "line_data_sl1l2", "line_data_sal1l2", - "line_data_vl1l2", "line_data_val1l2", "line_data_mpr", "line_data_nbrctc", "line_data_nbrcts", - "line_data_nbrcnt", "line_data_isc", "line_data_mctc", "line_data_rhist", "line_data_orank", "line_data_relp", "line_data_eclv", - "line_data_ssvar", "line_data_enscnt", "line_data_grad" - }; - private MVOrderedMap mapIndexes; - /* - * variable length table names for each variable length output line type - */ - private final Map tableVarLengthTable; - /* - * names of the fields for each line type - */ - private final Map tableLineDataFieldsTable; - /* - * data_file_lu_id values for each MET output type - */ - private final Map tableDataFileLU; - - public CBLoadDatabaseManager(DatabaseInfo databaseInfo, String password) throws Exception { - super(databaseInfo, password); - mapIndexes = new MVOrderedMap(); - mapIndexes.put("#stat_header#_model_idx", "model"); - mapIndexes.put("#stat_header#_fcst_var_idx", "fcst_var"); - mapIndexes.put("#stat_header#_fcst_lev_idx", "fcst_lev"); - mapIndexes.put("#stat_header#_obtype_idx", "obtype"); - mapIndexes.put("#stat_header#_vx_mask_idx", "vx_mask"); - mapIndexes.put("#stat_header#_interp_mthd_idx", "interp_mthd"); - mapIndexes.put("#stat_header#_interp_pnts_idx", "interp_pnts"); - mapIndexes.put("#stat_header#_fcst_thresh_idx", "fcst_thresh"); - - mapIndexes.put("#mode_header#_model_idx", "model"); - mapIndexes.put("#mode_header#_fcst_lead_idx", "fcst_lead"); - mapIndexes.put("#mode_header#_fcst_valid_idx", "fcst_valid"); - mapIndexes.put("#mode_header#_fcst_init_idx", "fcst_init"); - mapIndexes.put("#mode_header#_fcst_rad_idx", "fcst_rad"); - mapIndexes.put("#mode_header#_fcst_thr_idx", "fcst_thr"); - mapIndexes.put("#mode_header#_fcst_var_idx", "fcst_var"); - mapIndexes.put("#mode_header#_fcst_lev_idx", "fcst_lev"); - - mapIndexes.put("#mtd_header#_model_idx", "model"); - mapIndexes.put("#mtd_header#_fcst_lead_idx", "fcst_lead"); - mapIndexes.put("#mtd_header#_fcst_valid_idx", "fcst_valid"); - mapIndexes.put("#mtd_header#_fcst_init_idx", "fcst_init"); - mapIndexes.put("#mtd_header#_fcst_rad_idx", "fcst_rad"); - mapIndexes.put("#mtd_header#_fcst_thr_idx", "fcst_thr"); - mapIndexes.put("#mtd_header#_fcst_var_idx", "fcst_var"); - mapIndexes.put("#mtd_header#_fcst_lev_idx", "fcst_lev"); - - - for (String listLineDataTable : listLineDataTables) { - mapIndexes.put("#" + listLineDataTable + "#_fcst_lead_idx", "fcst_lead"); - mapIndexes.put("#" + listLineDataTable + "#_fcst_valid_beg_idx", "fcst_valid_beg"); - mapIndexes.put("#" + listLineDataTable + "#_fcst_init_beg_idx", "fcst_init_beg"); - } - tableVarLengthTable = new HashMap<>(); - tableVarLengthTable.put("PCT", "pct_thresh"); - tableVarLengthTable.put("PSTD", "pstd_thresh"); - tableVarLengthTable.put("PJC", "pjc_thresh"); - tableVarLengthTable.put("PRC", "prc_thresh"); - tableVarLengthTable.put("MCTC", "mctc_cnt"); - tableVarLengthTable.put("RHIST", "rhist_rank"); - tableVarLengthTable.put("RELP", "relp_ens"); - tableVarLengthTable.put("PHIST", "phist_bin"); - tableVarLengthTable.put("ORANK", "orank_ens"); - tableVarLengthTable.put("ECLV", "eclv_pnt"); - - tableLineDataFieldsTable = new HashMap<>(); - tableLineDataFieldsTable.put("ctc", "total,fy_oy,fy_on,fn_oy,fn_on"); - tableLineDataFieldsTable.put("fho", "total,f_rate,h_rate,o_rate"); - tableLineDataFieldsTable.put("pct", "cov_thresh,total,n_thresh,pct_thresh"); - tableLineDataFieldsTable.put("pct_thresh", "i_value,thresh_i,oy_i,on_i"); - tableLineDataFieldsTable.put("pjc", "cov_thresh,total,n_thresh,pjc_thresh"); - tableLineDataFieldsTable.put("pjc_thresh", "i_value,thresh_i,oy_tp_i,on_tp_i,calibration_i," + - "refinement_i,likelihood_i,baser_i"); - tableLineDataFieldsTable.put("prc", "cov_thresh,total,n_thresh,prc_thresh"); - tableLineDataFieldsTable.put("prc_thresh", "i_value,thresh_i,pody_i,pofd_i"); - tableLineDataFieldsTable - .put("pstd", "cov_thresh,alpha,total,n_thresh,baser,baser_ncl,baser_ncu," + - "reliability,resolution,uncertainty,roc_auc,brier,brier_ncl,brier_ncu,briercl,briercl_ncl," + - "briercl_ncu,bss,bss_smpl,pstd_thresh"); - tableLineDataFieldsTable.put("pstd_thresh", "i_value,thresh_i"); - tableLineDataFieldsTable.put("relp", "total,n_ens,relp_ens"); - tableLineDataFieldsTable.put("relp_ens", "i_value,ens_i"); - tableLineDataFieldsTable.put("eclv", "total,baser,value_baser,n_pnt,eclv_pnt"); - tableLineDataFieldsTable.put("eclv_pnt", "i_value,x_pnt_i,y_pnt_i"); - tableLineDataFieldsTable.put("enscnt", "rpsf,rpsf_ncl,rpsf_ncu,rpsf_bcl,rpsf_bcu," + - "rpscl,rpscl_ncl,rpscl_ncu,rpscl_bcl,rpscl_bcu," + - "rpss,rpss_ncl,rpss_ncu,rpss_bcl,rpss_bcu," + - "crpsf,crpsf_ncl,crpsf_ncu,crpsf_bcl,crpsf_bcu," + - "crpscl,crpscl_ncl,crpscl_ncu,crpscl_bcl,crpscl_bcu," + - "crpss,crpss_ncl,crpss_ncu,crpss_bcl,crpss_bcu"); - tableLineDataFieldsTable.put("grad", "total,fgbar,ogbar,mgbar,egbar,s1,s1_og,fgog_ratio"); - tableLineDataFieldsTable.put("sl1l2", "total,fbar,obar,fobar,ffbar,oobar,mae"); - tableLineDataFieldsTable.put("sal1l2", "total,fabar,oabar,foabar,ffabar,ooabar,mae"); - tableLineDataFieldsTable.put("vl1l2", "total,ufbar,vfbar,uobar,vobar,uvfobar,uvffbar,uvoobar," + - "f_speed_bar,o_speed_bar"); - tableLineDataFieldsTable - .put("val1l2", "total,ufabar,vfabar,uoabar,voabar,uvfoabar,uvffabar,uvooabar"); - tableLineDataFieldsTable - .put("cts", "alpha,total,baser,baser_ncl,baser_ncu,baser_bcl,baser_bcu," + - "fmean,fmean_ncl,fmean_ncu,fmean_bcl,fmean_bcu,acc,acc_ncl,acc_ncu,acc_bcl," + - "acc_bcu,fbias,fbias_bcl,fbias_bcu,pody,pody_ncl,pody_ncu,pody_bcl," + - "pody_bcu,podn,podn_ncl,podn_ncu,podn_bcl,podn_bcu,pofd,pofd_ncl," + - "pofd_ncu,pofd_bcl,pofd_bcu,far,far_ncl,far_ncu,far_bcl,far_bcu," + - "csi,csi_ncl,csi_ncu,csi_bcl,csi_bcu,gss,gss_bcl,gss_bcu," + - "hk,hk_ncl,hk_ncu,hk_bcl,hk_bcu,hss,hss_bcl,hss_bcu," + - "odds,odds_ncl,odds_ncu,odds_bcl,odds_bcu,lodds,lodds_ncl," + - "lodds_ncu,lodds_bcl,lodds_bcu,orss,orss_ncl,orss_ncu,orss_bcl,orss_bcu," + - "eds,eds_ncl,eds_ncu,eds_bcl,eds_bcu,seds,seds_ncl,seds_ncu,seds_bcl,seds_bcu," + - "edi,edi_ncl,edi_ncu,edi_bcl,edi_bcu,sedi,sedi_ncl,sedi_ncu,sedi_bcl,sedi_bcu," + - "bagss,bagss_bcl,bagss_bcu"); - tableLineDataFieldsTable.put("cnt", "alpha,total,fbar,fbar_ncl,fbar_ncu,fbar_bcl,fbar_bcu," + - "fstdev,fstdev_ncl,fstdev_ncu,fstdev_bcl,fstdev_bcu,obar,obar_ncl,obar_ncu,obar_bcl,obar_bcu," + - "ostdev,ostdev_ncl,ostdev_ncu,ostdev_bcl,ostdev_bcu," + - "pr_corr,pr_corr_ncl,pr_corr_ncu,pr_corr_bcl,pr_corr_bcu,sp_corr,dt_corr," + - "ranks,frank_ties,orank_ties,me,me_ncl,me_ncu,me_bcl,me_bcu," + - "estdev,estdev_ncl,estdev_ncu,estdev_bcl,estdev_bcu," + - "mbias,mbias_bcl,mbias_bcu,mae,mae_bcl,mae_bcu,mse,mse_bcl,mse_bcu," + - "bcmse,bcmse_bcl,bcmse_bcu,rmse,rmse_bcl,rmse_bcu," + - "e10,e10_bcl,e10_bcu,e25,e25_bcl,e25_bcu,e50,e50_bcl,e50_bcu,e75,e75_bcl,e75_bcu," + - "e90,e90_bcl,e90_bcu,iqr,iqr_bcl,iqr_bcu,mad,mad_bcl,mad_bcu," + - "anom_corr,anom_corr_ncl,anom_corr_ncu,anom_corr_bcl,anom_corr_bcu," + - "me2,me2_bcl,me2_bcu,msess,msess_bcl,msess_bcu," + - "rmsfa,rmsfa_bcl,rmsfa_bcu,rmsoa,rmsoa_bcl,rmsoa_bcu"); - tableLineDataFieldsTable.put("mctc", "total,n_cat,mctc_cnt"); - tableLineDataFieldsTable.put("mctc_cnt", "i_value,j_value,fi_oj"); - tableLineDataFieldsTable.put("mpr", "total,mp_index,obs_sid,obs_lat,obs_lon,obs_lvl,obs_elv," + - "mpr_fcst,mpr_obs,mpr_climo,obs_qc,climo_mean,climo_stdev,climo_cdf"); - tableLineDataFieldsTable - .put("orank", "total,orank_index,obs_sid,obs_lat,obs_lon,obs_lvl,obs_elv," + - "orank_obs,pit,rank,n_ens_vld,n_ens,obs_qc,ens_mean,orank_climo,orank_enc_spread,orank_ens"); - tableLineDataFieldsTable.put("orank_ens", "i_value,ens_i"); - tableLineDataFieldsTable.put("isc", "total,tile_dim,time_xll,tile_yll,nscale,iscale,mse,isc," + - "fenergy2,oenergy2,baser,fbias"); - tableLineDataFieldsTable.put("nbrcnt", "alpha,total,fbs,fbs_bcl,fbs_bcu,fss,fss_bcl,fss_bcu," + - "afss,afss_bcl,afss_bcu,ufss,ufss_bcl,ufss_bcu,f_rate,f_rate_bcl,f_rate_bcu," + - "o_rate,o_rate_bcl,o_rate_bcu"); - tableLineDataFieldsTable.put("nbrctc", "cov_thresh,total,fy_oy,fy_on,fn_oy,fn_on"); - tableLineDataFieldsTable - .put("nbrcts", "cov_thresh,alpha,total,baser,baser_ncl,baser_ncu,baser_bcl,baser_bcu," + - "fmean,fmean_ncl,fmean_ncu,fmean_bcl,fmean_bcu,acc,acc_ncl,acc_ncu,acc_bcl," + - "acc_bcu,fbias,fbias_bcl,fbias_bcu,pody,pody_ncl,pody_ncu,pody_bcl," + - "pody_bcu,podn,podn_ncl,podn_ncu,podn_bcl,podn_bcu,pofd,pofd_ncl," + - "pofd_ncu,pofd_bcl,pofd_bcu,far,far_ncl,far_ncu,far_bcl,far_bcu," + - "csi,csi_ncl,csi_ncu,csi_bcl,csi_bcu,gss,gss_bcl,gss_bcu," + - "hk,hk_ncl,hk_ncu,hk_bcl,hk_bcu,hss,hss_bcl,hss_bcu," + - "odds,odds_ncl,odds_ncu,odds_bcl,odds_bcu,lodds,lodds_ncl," + - "lodds_ncu,lodds_bcl,lodds_bcu,orss,orss_ncl,orss_ncu,orss_bcl,orss_bcu," + - "eds,eds_ncl,eds_ncu,eds_bcl,eds_bcu,seds,seds_ncl,seds_ncu,seds_bcl,seds_bcu," + - "edi,edi_ncl,edi_ncu,edi_bcl,edi_bcu,sedi,sedi_ncl,sedi_ncu,sedi_bcl,sedi_bcu," + - "bagss,bagss_bcl,bagss_bcu"); - tableLineDataFieldsTable - .put("ssvar", "alpha,total,n_bin,bin_i,bin_n,var_min,var_max,var_mean," + - "fbar,obar,fobar,ffbar,oobar,fbar_ncl,fbar_ncu,fstdev,fstdev_ncl,fstdev_ncu," + - "obar_ncl,obar_ncu,ostdev,ostdev_ncl,ostdev_ncu," + - "pr_corr,pr_corr_ncl,pr_corr_ncu,me,me_ncl,me_ncu," + - "estdev,estdev_ncl,estdev_ncu,mbias,mse,bcmse,rmse"); - tableLineDataFieldsTable.put("ecnt", "total,n_ens,crps,crpss,ign,me,rmse,spread," + - "me_oerr,rmse_oerr,spread_oerr,spread_plus_oerr"); - tableLineDataFieldsTable.put("rhist", "total,n_rank,rhist_rank"); - tableLineDataFieldsTable.put("rhist_rank", "i_value,rank_i"); - tableLineDataFieldsTable.put("phist", "total,bin_size,n_bin,phist_bin"); - tableLineDataFieldsTable.put("phist_bin", "i_value,bin_i"); - - tableDataFileLU = new HashMap<>(); - tableDataFileLU.put("point_stat", 0); - tableDataFileLU.put("grid_stat", 1); - tableDataFileLU.put("mode_cts", 2); - tableDataFileLU.put("mode_obj", 3); - tableDataFileLU.put("wavelet_stat", 4); - tableDataFileLU.put("ensemble_stat", 5); - tableDataFileLU.put("vsdb_point_stat", 6); - tableDataFileLU.put("stat", 7); - tableDataFileLU.put("mtd_2d", 8); - tableDataFileLU.put("mtd_3d_pc", 9); - tableDataFileLU.put("mtd_3d_ps", 10); - tableDataFileLU.put("mtd_3d_sc", 11); - tableDataFileLU.put("mtd_3d_ss", 12); - - } - - @Override - public void applyIndexes() { - // what indexes do we need to apply in couchbase? - // do nothing for now - } - - @Override - public void dropIndexes() { - // what indexes do we need to drop in couchbase? - // do nothing for now - } - -// /** -// * Executes the input update statement against the database underlying the input Connection and -// * cleans up any resources upon completion. -// * -// * @param update SQL UPDATE statement to execute -// * @return Number of records affected (output of Statement.executeUpdate() call) -// * @throws Exception -// */ -// private int executeUpdate(String update) throws Exception { -// -// int intRes; -// /* try ( -// Connection con = getConnection(); -// Statement stmt = con.createStatement() ) { -// intRes = stmt.executeUpdate(update); -// -// } catch (Exception e) { -// logger.error(update); -// throw new Exception("caught Exception calling executeUpdate: " + e.getMessage()); -// } -// */ -// intRes = -1; -// return intRes; -// } - - - /** - * Build and execute a query that retrieves the next table record id, whose name is specified by - * the input field, from the specified input table. The statement is run against the input - * Connection and the next available id is returned. // * @param con - * - * @param table Database table whose next available id is returned - * @param field Field name of the table id record - * @return Next available id - * @throws Exception - */ - private int getNextId(String table, String field) throws DatabaseException { - int intId = -1; - /* PreparedStatement pstmt = null; - ResultSet res = null; - try (Connection con = getConnection()) { - pstmt = con.prepareStatement("SELECT MAX(" + field + ") FROM " + table); - res = pstmt.executeQuery(); - if (!res.next()) { - throw new Exception("METViewer load error: getNextId(" + table + ", " + field + ") unable to find max id"); - } - String strId = res.getString(1); - if (null == strId) { - intId = 0; - } else { - intId = Integer.parseInt(strId) + 1; - } - - } catch (Exception e) { - throw new Exception(e.getMessage()); - } finally { - if (pstmt != null) { - pstmt.close(); - } - if (res != null) { - res.close(); - } - } -*/ - return intId; - } - - /** - * Load the MET output data from the data file underlying the input DataFileInfo object into the - * database underlying the input Connection. The header information can be checked in two - * different ways: using a table for the current file (specified by _boolStatHeaderTableCheck) or - * by searching the stat_header table for a duplicate (specified by _boolStatHeaderDBCheck). - * Records in line_data tables, stat_group tables and line_data_thresh tables are created from the - * data in the input file. If necessary, records in the stat_header table are created as well. - * - * @param info Contains MET output data file information //* @param con Connection to the target - * database - * @throws Exception - */ - @Override - public Map loadStatFile(DataFileInfo info) throws DatabaseException { - Map timeStats = new HashMap<>(); - // initialize the insert data structure - MVLoadStatInsertData insertData = new MVLoadStatInsertData(); - // performance counters - long intStatHeaderLoadStart = System.currentTimeMillis(); - long headerSearchTime = 0; - long headerRecords = 0; - long headerInserts = 0; - long dataRecords = 0; - long dataInserts = 0; - long intLineDataSkipped = 0; - long lengthRecords = 0; - long lengthInserts = 0; - String modelName; - N1qlQueryResult queryResult = null; - List queryList = null; - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - long nextIdNumber = 0; - String headerIdString = ""; - JsonObject headerFile; - JsonDocument response; - JsonDocument doc; - timeStats.put("headerSearchTime", 0L); - - // set up the input file for reading - String strFilename = info.path + "/" + info.filename; - int intLine = 0; - List headerNames = new ArrayList<>(); - try ( - FileReader fileReader = new FileReader(strFilename); - BufferedReader reader = new BufferedReader(fileReader)) { - // read in each line of the input file - while (reader.ready()) { - String[] listToken = reader.readLine().split("\\s+"); - intLine++; - - // the first line is the header line - if (1 > listToken.length || listToken[0].equals("VERSION")) { - headerNames = Arrays.asList(listToken); - continue; - } - - // error if the version number does not match the configured value - String strMetVersion = MVUtil.findValue(listToken, headerNames, "VERSION"); - - // if the line type load selector is activated, check that the current line type is on the list - insertData.setLineType(MVUtil.findValue(listToken, headerNames, "LINE_TYPE")); - String lineType = insertData.getLineType(); - - if (!MVUtil.isValidLineType(lineType)) { - logger.warn( - " ** WARNING: unexpected line type: " + lineType - + " the line will be ignored "); - continue; - } - - if (info.lineTypeLoad && !info.tableLineTypeLoad.containsKey(lineType)) { - continue; - } - - // do not load matched pair lines or orank lines - if ((!info.loadMpr && lineType.equals("MPR")) - || (!info.loadOrank && lineType.equals("ORANK"))) { - continue; - } - - insertData.setFileLine(strFilename + ":" + intLine); - - // parse the valid times - LocalDateTime fcstValidBeg = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "FCST_VALID_BEG"), - DB_DATE_STAT_FORMAT); - - - LocalDateTime fcstValidEnd = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "FCST_VALID_END"), - DB_DATE_STAT_FORMAT); - - - LocalDateTime obsValidBeg = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "OBS_VALID_BEG"), - DB_DATE_STAT_FORMAT); - - - LocalDateTime obsValidEnd = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "OBS_VALID_END"), - DB_DATE_STAT_FORMAT); - - // format the valid times for the database insert - String fcstValidBegStr = DATE_FORMAT_1.format(fcstValidBeg); - String fcstValidEndStr = DATE_FORMAT_1.format(fcstValidEnd); - String obsValidBegStr = DATE_FORMAT_1.format(obsValidBeg); - String obsValidEndStr = DATE_FORMAT_1.format(obsValidEnd); - - // calculate the number of seconds corresponding to fcst_lead - String fcstLeadStr = MVUtil.findValue(listToken, headerNames, "FCST_LEAD"); - int fcstLeadLen = fcstLeadStr.length(); - int fcstLeadSec = Integer.parseInt(fcstLeadStr.substring(fcstLeadLen - 2, fcstLeadLen)); - fcstLeadSec += Integer.parseInt( - fcstLeadStr.substring(fcstLeadLen - 4, fcstLeadLen - 2)) * 60; - fcstLeadSec += Integer.parseInt(fcstLeadStr.substring(0, fcstLeadLen - 4)) * 3600; - - // Make obs lead look like an integer (i.e. 0 not 000000) - String obsLeadStr = MVUtil.findValue(listToken, headerNames, "OBS_LEAD"); - obsLeadStr = String.valueOf(Integer.parseInt(obsLeadStr)); - - // determine the init time by combining fcst_valid_beg and fcst_lead - LocalDateTime fcstInitBeg = LocalDateTime.from(fcstValidBeg); - fcstInitBeg = fcstInitBeg.minusSeconds(fcstLeadSec); - - String fcstInitBegStr = DATE_FORMAT_1.format(fcstInitBeg); - - // ensure that the interp_pnts field value is a reasonable integer - String strInterpPnts = MVUtil.findValue(listToken, headerNames, "INTERP_PNTS"); - if (strInterpPnts.equals("NA")) { - strInterpPnts = "0"; - } - - - /* - * * * * stat_header insert * * * * - */ - headerRecords++; - - modelName = MVUtil.findValue(listToken, headerNames, "MODEL"); - - String strFileLine = strFilename + ":" + intLine; - - // build the value list for the stat_header search - String statHeaderValue = - modelName + - MVUtil.findValue(listToken, headerNames, "FCST_VAR") + - MVUtil.findValue(listToken, headerNames, "FCST_LEV") + - MVUtil.findValue(listToken, headerNames, "OBS_VAR") + - MVUtil.findValue(listToken, headerNames, "OBS_LEV") + - MVUtil.findValue(listToken, headerNames, "OBTYPE") + - MVUtil.findValue(listToken, headerNames, "VX_MASK") + - MVUtil.findValue(listToken, headerNames, "INTERP_MTHD") + - strInterpPnts + - MVUtil.findValue(listToken, headerNames, "FCST_THRESH") + - MVUtil.findValue(listToken, headerNames, "OBS_THRESH"); - - // These will eventually be put back into the above code after adding to MySQL - // MVUtil.findValue(listToken, headerNames, "OBS_VAR") + - // MVUtil.findValue(listToken, headerNames, "OBS_LEV") + - - // look for the header key in the table - headerIdString = ""; - if (statHeaders.containsKey(statHeaderValue)) { - headerIdString = statHeaders.get(statHeaderValue); - } - // if the stat_header does not yet exist, create one - else { - // look for an existing stat_header record with the same information - boolean boolFoundStatHeader = false; - long intStatHeaderSearchBegin = System.currentTimeMillis(); - if (info.statHeaderDBCheck) { - // build a Couchbase query to look for duplicate stat_header records - String strDataFileQuery = "SELECT " + - "meta().id as headerFileId, " + - "type, " + - "header_type, " + - "data_type, " + - "data_id " + - "FROM `" + - getBucket().name() + - "` WHERE " + - "type = \'header\' AND " + - "dbname = \'" + getDbName() + "\' AND " + - "header_type = \'stat\' AND " + - "data_type = \'" + info.luTypeName + "\' AND " + - "model = \'" + modelName + "\' AND " + - "fcst_var = \'" + MVUtil.findValue(listToken, headerNames, - "FCST_VAR") + "\' AND " + - "fcst_lev = \'" + MVUtil.findValue(listToken, headerNames, - "FCST_LEV") + "\' AND " + - "obs_var = \'" + MVUtil.findValue(listToken, headerNames, - "OBS_VAR") + "\' AND " + - "obs_lev = \'" + MVUtil.findValue(listToken, headerNames, - "OBS_LEV") + "\' AND " + - "obtype = \'" + MVUtil.findValue(listToken, headerNames, - "OBTYPE") + "\' AND " + - "vx_mask = \'" + MVUtil.findValue(listToken, headerNames, - "VX_MASK") + "\' AND " + - "interp_mthd = \'" + MVUtil.findValue(listToken, - headerNames, - "INTERP_MTHD") + "\' AND " + - "interp_pnts = \'" + strInterpPnts + "\' AND " + - "fcst_thresh = \'" + MVUtil.findValue(listToken, - headerNames, - "FCST_THRESH") + "\' AND " + - "obs_thresh = \'" + MVUtil - .findValue(listToken, headerNames, - "OBS_THRESH") + "\';"; - - try { - queryResult = getBucket().query(N1qlQuery.simple(strDataFileQuery)); - queryList = queryResult.allRows(); - // if the header document is already present in the database, print a warning and return the id - if (queryList.size() > 0) { - firstRow = queryList.get(0); - firstRowObject = firstRow.value(); - // set headerIdString to id of existing header document - headerIdString = firstRowObject.get("headerFileId").toString(); - boolFoundStatHeader = true; - // add header to table - statHeaders.put(statHeaderValue, headerIdString); - logger.warn(" ** WARNING: header document already present in database"); - } - timeStats.put("headerSearchTime", - timeStats.get("headerSearchTime") + System.currentTimeMillis() - - intStatHeaderSearchBegin); - } catch (CouchbaseException e) { - throw new Exception(e.getMessage()); - } - } // end if (info._boolStatHeaderDBCheck) - - // if the stat_header was not found, add it to the table - if (!boolFoundStatHeader) { - // create a unique data_file id from a Couchbase counter, starting at 1 the first time - try { - nextIdNumber = getBucket().counter("HDCounter", 1, 1).content(); - // unique id must be a string - headerIdString = getDbName() + "::header::stat::" + modelName + "::" + String.valueOf( - nextIdNumber); - } catch (CouchbaseException e) { - throw new Exception(e.getMessage()); - } - - try { - headerFile = JsonObject.empty() - .put("type", "header") - .put("header_type", "stat") - .put("data_type", info.luTypeName) - .put("data_id", info.fileIdStr) - .put("dbname", getDbName()) - .put("version", MVUtil.findValue(listToken, headerNames, "VERSION")) - .put("model", modelName) - .put("descr", MVUtil.findValue(listToken, headerNames, "DESC")) - .put("fcst_var", - MVUtil.findValue(listToken, headerNames, "FCST_VAR")) - .put("fcst_lev", - MVUtil.findValue(listToken, headerNames, "FCST_LEV")) - .put("obs_var", MVUtil.findValue(listToken, headerNames, "OBS_VAR")) - .put("obs_lev", MVUtil.findValue(listToken, headerNames, "OBS_LEV")) - .put("obtype", MVUtil.findValue(listToken, headerNames, "OBTYPE")) - .put("vx_mask", MVUtil.findValue(listToken, headerNames, "VX_MASK")) - .put("interp_mthd", - MVUtil.findValue(listToken, headerNames, "INTERP_MTHD")) - .put("interp_pnts", strInterpPnts) - .put("fcst_thresh", - MVUtil.findValue(listToken, headerNames, "FCST_THRESH")) - .put("obs_thresh", - MVUtil.findValue(listToken, headerNames, "OBS_THRESH")); - doc = JsonDocument.create(headerIdString, headerFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from header INSERT"); - } else { - headerInserts++; - } - } catch (Exception e) { - throw new Exception(e.getMessage()); - } - // add header to table - statHeaders.put(statHeaderValue, headerIdString); - } // end if (!boolFoundStatHeader) - } // end else stat_header is not in table - - boolean isMet8 = true; - if (insertData.getLineType().equals("RHIST")) { - int indexOfNrank = headerNames.indexOf("LINE_TYPE") + 2; - boolean isInt = MVUtil.isInteger(listToken[indexOfNrank], 10); - isMet8 = isInt - && (Integer.valueOf(listToken[indexOfNrank]) - + indexOfNrank == listToken.length - 1); - - } - - if (headerIdString != null) { - - int lineDataMax = listToken.length; - dataRecords++; - - // if the line type is of variable length, get the line_data_id - boolean hasVarLengthGroups = MVUtil.lengthGroupIndices - .containsKey(lineType); - - // determine the maximum token index for the data - if (hasVarLengthGroups) { - int[] listVarLengthGroupIndices1 = MVUtil.lengthGroupIndices - .get(lineType); - int[] listVarLengthGroupIndices = Arrays.copyOf(listVarLengthGroupIndices1, - listVarLengthGroupIndices1.length); - if (headerNames.indexOf("DESC") < 0) { - //for old versions - listVarLengthGroupIndices[0] = listVarLengthGroupIndices[0] - 1; - listVarLengthGroupIndices[1] = listVarLengthGroupIndices[1] - 1; - } - - switch (insertData.getLineType()) { - case "RHIST": - - if (!isMet8) { - //old met data - listVarLengthGroupIndices[0] = listVarLengthGroupIndices[0] + 2; - listVarLengthGroupIndices[1] = listVarLengthGroupIndices[1] + 2; - - } - lineDataMax = lineDataMax - Integer.valueOf( - listToken[listVarLengthGroupIndices[0]]) * listVarLengthGroupIndices[2]; - break; - case "PSTD": - lineDataMax = lineDataMax - Integer.valueOf( - listToken[listVarLengthGroupIndices[0]]) * listVarLengthGroupIndices[2]; - break; - default: - lineDataMax = listVarLengthGroupIndices[1]; - break; - } - } - - // build the value list for the insert statement - String strLineDataValues = - getDbName() + ESEP + // database name for ID - lineType.toLowerCase() + ESEP + // line type - modelName + ESEP + // model name for ID - headerIdString + ESEP + // CB header_id - info.fileIdStr + ESEP + // CB data_id for data_file - intLine + ESEP + // line_num - fcstLeadStr + ESEP + // fcst_lead - fcstValidBegStr + ESEP + // fcst_valid_beg - fcstValidEndStr + ESEP + // fcst_valid_end - fcstInitBegStr + ESEP + // fcst_init_beg - obsLeadStr + ESEP + // obs_lead - obsValidBegStr + ESEP + // obs_valid_beg - obsValidEndStr; // obs_valid_end - - // if the line data requires a cov_thresh value, add it - String strCovThresh = MVUtil.findValue(listToken, headerNames, "COV_THRESH"); - if (MVUtil.covThreshLineTypes.containsKey(lineType)) { - strLineDataValues += ESEP + replaceInvalidValues(strCovThresh); - } - - // if the line data requires an alpha value, add it - String strAlpha = MVUtil.findValue(listToken, headerNames, "ALPHA"); - if (MVUtil.alphaLineTypes.containsKey(lineType)) { - if (strAlpha.equals("NA")) { - logger.warn(" ** WARNING: alpha value NA with line type '" + lineType + - "'\n " + insertData.getFileLine()); - } - strLineDataValues += ESEP + replaceInvalidValues(strAlpha); - } else if (!strAlpha.equals("NA")) { - logger.warn( - " ** WARNING: unexpected alpha value '" + strAlpha + "' in line type '" + lineType + - "'\n " + insertData.getFileLine()); - } - - // add total and all of the stats on the rest of the line to the value list - if (lineType.equals("RHIST")) { - int lineTypeIndex = headerNames.indexOf("LINE_TYPE"); - for (int i = lineTypeIndex + 1; i < lineDataMax; i++) { - if (!isMet8) { - //skip crps ,ign,crpss, spread - if (i == lineTypeIndex + 2 || i == lineTypeIndex + 3 - || i == lineTypeIndex + 5 || i == lineTypeIndex + 6) { - continue; - } - } - strLineDataValues += ESEP + replaceInvalidValues(listToken[i]); - } - if (!isMet8) { - //insert crps ,ign,crpss, spread to ECNT table - String ecntLineDataValues = getDbName() + ESEP + "ecnt" + ESEP + - strLineDataValues - .substring(strLineDataValues.indexOf(modelName)); - int indexOfNrankOld = headerNames.indexOf("LINE_TYPE") + 4; - boolean isMetOld = (Double.valueOf(listToken[indexOfNrankOld]) - .intValue() + indexOfNrankOld) == - listToken.length - 1; - - ecntLineDataValues += ESEP + replaceInvalidValues(listToken[lineTypeIndex + 2]); - if (isMetOld) { - ecntLineDataValues += ESEP + "-9999"; - } else { - ecntLineDataValues += ESEP + replaceInvalidValues(listToken[lineTypeIndex + 5]); - } - ecntLineDataValues += ESEP + replaceInvalidValues(listToken[lineTypeIndex + 3]); - ecntLineDataValues += ESEP + "-9999"; - ecntLineDataValues += ESEP + "-9999"; - if (isMetOld) { - ecntLineDataValues += ESEP + "-9999"; - } else { - ecntLineDataValues += ESEP + replaceInvalidValues(listToken[lineTypeIndex + 6]); - } - ecntLineDataValues += ESEP + "-9999"; - ecntLineDataValues += ESEP + "-9999"; - ecntLineDataValues += ESEP + "-9999"; - ecntLineDataValues += ESEP + "-9999"; - - if (!insertData.getTableLineDataValues().containsKey("ECNT")) { - insertData.getTableLineDataValues().put("ECNT", new ArrayList<>()); - } - insertData.getTableLineDataValues().get("ECNT").add(ecntLineDataValues); - dataInserts++; - } - } else { - for (int i = headerNames.indexOf("LINE_TYPE") + 1; i < lineDataMax; i++) { - // add the stats in order - strLineDataValues += ESEP + replaceInvalidValues(listToken[i]); - - } - } - - - if (lineType.equals("ORANK")) { - //skip ensemble fields and get data for the rest - int[] listVarLengthGroupIndices1 = MVUtil.lengthGroupIndices.get(lineType); - int[] listVarLengthGroupIndices = Arrays.copyOf(listVarLengthGroupIndices1, - listVarLengthGroupIndices1.length); - if (headerNames.indexOf("DESC") < 0) { - //for old versions - listVarLengthGroupIndices[0] = listVarLengthGroupIndices[0] - 1; - listVarLengthGroupIndices[1] = listVarLengthGroupIndices[1] - 1; - } - int extraFieldsInd = lineDataMax + Integer.valueOf( - listToken[listVarLengthGroupIndices[0]]) * listVarLengthGroupIndices[2]; - for (int i = extraFieldsInd; i < listToken.length; i++) { - strLineDataValues += ESEP + replaceInvalidValues(listToken[i]); - } - } - - - String[] insertValuesArr = strLineDataValues.split(ESEP); - List insertValuesList = new LinkedList<>(Arrays.asList(insertValuesArr)); - int size = insertValuesList.size(); - int maxSize = size; - switch (lineType) { - case "CNT": - maxSize = 108; - break; - case "MPR": - maxSize = 27; - break; - case "ORANK": - maxSize = 30; - break; - case "CTS": - maxSize = 107; - break; - case "NBRCTS": - maxSize = 108; - break; - case "NBRCNT": - maxSize = 33; - break; - case "SAL1L2": - maxSize = 20; - break; - case "SL1L2": - maxSize = 20; - break; - case "GRAD": - maxSize = 21; - break; - case "PSTD": - maxSize = 32; - break; - case "SSVAR": - maxSize = 49; - break; - case "VL1L2": - maxSize = 23; - break; - case "ECNT": - maxSize = 22; - break; - - default: - } - while (size < maxSize) { - insertValuesList.add("-9999"); - size++; - } - strLineDataValues = ""; - for (String s : insertValuesList) { - strLineDataValues = strLineDataValues + s + ESEP; - } - strLineDataValues = strLineDataValues - .substring(0, strLineDataValues.length() - 1); - - - /* - * * * * var_length insert * * * * - */ - - if (hasVarLengthGroups) { - - // get the index information about the current line type - int[] varLengthGroupIndices1 = MVUtil.lengthGroupIndices.get(lineType); - int[] varLengthGroupIndices = Arrays.copyOf(varLengthGroupIndices1, - varLengthGroupIndices1.length); - if (lineType.equals("RHIST")) { - if (!isMet8) { - //old met data - varLengthGroupIndices[0] = varLengthGroupIndices[0] + 2; - varLengthGroupIndices[1] = varLengthGroupIndices[1] + 2; - - } - } - if (headerNames.indexOf("DESC") < 0) { - //for old versions - varLengthGroupIndices[0] = varLengthGroupIndices[0] - 1; - varLengthGroupIndices[1] = varLengthGroupIndices[1] - 1; - } - int groupCntIndex = varLengthGroupIndices[0]; - int groupIndex = varLengthGroupIndices[1]; - int groupSize = varLengthGroupIndices[2]; - int numGroups = Integer.parseInt(listToken[groupCntIndex]); - - if (lineType.equals("PCT") - || lineType.equals("PJC") - || lineType.equals("PRC")) { - numGroups -= 1; - } - // create an array of objects - String strThreshValues = "["; - - // build an object for each threshold group - if (lineType.equals("MCTC")) { - for (int i = 0; i < numGroups; i++) { - for (int j = 0; j < numGroups; j++) { - strThreshValues += "{" + (i + 1) + "," + (j + 1) + "," + - replaceInvalidValues(listToken[groupIndex++]) + "}"; - lengthRecords++; - } - } - } else { - if (lineType.equals("RHIST")) { - groupIndex = lineDataMax; - } - - String strVarType = tableVarLengthTable.get(lineType); - String[] listFieldsArr; - listFieldsArr = tableLineDataFieldsTable.get(strVarType).split(","); - for (int i = 0; i < numGroups; i++) { - strThreshValues += "{" + listFieldsArr[0] + ":" + (i + 1); - for (int j = 0; j < groupSize; j++) { - strThreshValues += "," + listFieldsArr[j + 1] + ":" + replaceInvalidValues( - listToken[groupIndex++]); - } - strThreshValues += "}"; - // put a comma after every object except the last one - if (i != (numGroups - 1)) { - strThreshValues += ","; - } - lengthRecords++; - } - } - strThreshValues += "]"; - strLineDataValues += ESEP + strThreshValues; - - } - - // add the values list to the line type values map - List listLineTypeValues = new ArrayList<>(); - if (insertData.getTableLineDataValues() - .containsKey(lineType)) { - listLineTypeValues = insertData.getTableLineDataValues() - .get(lineType); - } - listLineTypeValues.add(strLineDataValues); - insertData.getTableLineDataValues() - .put(lineType, listLineTypeValues); - dataInserts++; - - // if the insert threshhold has been reached, commit the stored data to the database - if (info.insertSize <= insertData.getListInsertValues().size()) { - int[] listInserts = commitStatData(insertData); - dataInserts += listInserts[INDEX_LINE_DATA]; - lengthInserts += listInserts[INDEX_VAR_LENGTH]; - } - } - } // end: while( reader.ready() ) - - } catch (Exception e) { - logger.error(e.getMessage()); - } - - // commit all the remaining stored data - int[] listInserts = commitStatData(insertData); - dataInserts += listInserts[INDEX_LINE_DATA]; - lengthInserts += listInserts[INDEX_VAR_LENGTH]; - - - timeStats.put("linesTotal", (long) (intLine - 1)); - timeStats.put("headerRecords", headerRecords); - timeStats.put("headerInserts", headerInserts); - timeStats.put("dataInserts", dataInserts); - timeStats.put("dataRecords", dataRecords); - timeStats.put("lengthRecords", lengthRecords); - timeStats.put("lengthInserts", lengthInserts); - - - // print a performance report - long intStatHeaderLoadTime = new Date().getTime() - intStatHeaderLoadStart; - double dblLinesPerMSec = (double) (intLine - 1) / (double) (intStatHeaderLoadTime); - - if (info.verbose) { - logger.info(MVUtil.padBegin("file lines: ", 36) + (intLine - 1) + "\n" + - MVUtil.padBegin("stat_header records: ", 36) + headerRecords + "\n" + - MVUtil.padBegin("stat_header inserts: ", 36) + headerInserts + "\n" + - MVUtil.padBegin("line_data records: ", 36) + dataRecords + "\n" + - MVUtil.padBegin("line_data inserts: ", 36) + dataInserts + "\n" + - MVUtil.padBegin("line_data skipped: ", 36) + intLineDataSkipped + "\n" + - MVUtil.padBegin("var length records: ", 36) + lengthRecords + "\n" + - MVUtil.padBegin("var length inserts: ", 36) + lengthInserts + "\n" + - MVUtil.padBegin("total load time: ", 36) + MVUtil.formatTimeSpan( - intStatHeaderLoadTime) + "\n" + - MVUtil.padBegin("stat_header search time: ", 36) + MVUtil.formatTimeSpan( - headerSearchTime) + "\n" + - MVUtil.padBegin("lines / msec: ", 36) + MVUtil.DECIMAL_FORMAT.format( - dblLinesPerMSec) + "\n\n"); - } - return timeStats; - } - - /** - * Loads the insert value lists stored in the data structure MVLoadStatInsertData. This method - * was designed to be called from loadStatFile(), which is responsible for building insert value - * lists for the various types of grid_stat and point_stat database tables. - * - * @param statInsertData Data structure loaded with insert value lists - * @return An array of four integers, indexed by the INDEX_* members, representing the number of - * database inserts of each type - * @throws Exception - */ - private int[] commitStatData(MVLoadStatInsertData statInsertData) - throws DatabaseException { - - int[] listInserts = new int[]{0, 0, 0, 0}; - - /* - * * * * stat_header was committed commit * * * * - */ - - statInsertData.getListInsertValues().clear(); - - - /* - * * * * line_data commit * * * * - */ - - // for each line type, build an insert statement with the appropriate list of values - for (Map.Entry> entry : statInsertData.getTableLineDataValues() - .entrySet()) { - statInsertData.setLineType(entry.getKey()); - ArrayList listValues = (ArrayList) entry.getValue(); - String tableName = statInsertData.getLineType().toLowerCase(Locale.US); - - int intResLineDataInsert = executeBatch(listValues); - if (listValues.size() != intResLineDataInsert) { - logger.warn(" ** WARNING: unexpected result from line_data INSERT: " + - intResLineDataInsert + "\n " + statInsertData.getFileLine()); - } - listInserts[INDEX_LINE_DATA]++; - } - statInsertData.getTableLineDataValues().clear(); - - statInsertData.getListStatGroupInsertValues().clear(); - - /* - * * * * variable length data commit * * * * - */ - - // insert probabilistic data into the thresh tables - Set strings = statInsertData.getTableVarLengthValues().keySet(); - String[] varLengthTypes = strings.toArray(new String[strings.size()]); - - - for (String listVarLengthType : varLengthTypes) { - List> listVarLengthValues = - statInsertData.getTableVarLengthValues().get(listVarLengthType); - if (1 > listVarLengthValues.size()) { - continue; - } - String strVarLengthTable = tableVarLengthTable.get(listVarLengthType); - String strThreshInsert = "INSERT INTO " + strVarLengthTable + " VALUES "; - // for (int j = 0; j < listVarLengthValues.size(); j++) { - // strThreshInsert += (0 < j ? ", " : "") + listVarLengthValues[j]; - // listInserts[INDEX_VAR_LENGTH]++; // lengthInserts++; - // } -// int intThreshInsert = executeUpdate(strThreshInsert); -// if (listVarLengthValues.length != intThreshInsert) { -// logger.warn( -// " ** WARNING: unexpected result from thresh INSERT: " + intThreshInsert + " vs. " + -// listVarLengthValues.length + "\n " + statInsertData.getFileLine()); -// } - statInsertData.getTableVarLengthValues().put(listVarLengthType, new ArrayList<>()); - } - - return listInserts; - } - - @Override - public Map loadStatFileVSDB(DataFileInfo info) throws DatabaseException { - - Map timeStats = new HashMap<>(); - - // initialize the insert data structure - MVLoadStatInsertData mvLoadStatInsertData = new MVLoadStatInsertData(); - - // performance counters - long intStatHeaderLoadStart = System.currentTimeMillis(); - long headerSearchTime = 0; - long headerRecords = 0; - long headerInserts = 0; - long dataRecords = 0; - long dataInserts = 0; - long intLineDataSkipped = 0; - long lengthRecords = 0; - long lengthInserts = 0; - timeStats.put("headerSearchTime", 0L); - long nextIdNumber = 0; - String nextIdString = ""; - String headerIdString = ""; - JsonObject headerFile; - JsonDocument response; - JsonDocument doc; - N1qlQueryResult queryResult = null; - List queryList = null; - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - - // set up the input file for reading - String strFilename = info.path + "/" + info.filename; - String ensValue = ""; - String[] ensValueArr = info.path.split("\\/"); - if (ensValueArr[ensValueArr.length - 1].contains("_")) { - String[] ensValue1 = ensValueArr[ensValueArr.length - 1].split("_"); - ensValue = "_" + ensValue1[ensValue1.length - 1]; - } - - int intLine = 0; - try (FileReader fileReader = new FileReader( - strFilename); BufferedReader reader = new BufferedReader(fileReader)) { - List allMatches; - - DateTimeFormatter formatStatVsdb = DateTimeFormatter.ofPattern("yyyyMMddHH"); - - // read in each line of the input file, remove "=" - while (reader.ready()) { - - String line = reader.readLine(); - try { - line = line.replaceAll("=\\s", " "); // remove "= " - Matcher m = Pattern.compile("\\d-0\\.").matcher( - line); // some records do not have a space between columns if the value in column starts with "-" - - allMatches = new ArrayList<>(); - while (m.find()) { - allMatches.add(m.group()); - } - for (String match : allMatches) { - String newStr = match.replace("-", " -"); - line = line.replace(match, newStr); - } - - String[] listToken = line.split("\\s+"); - intLine++; - String thresh = "NA"; - String modelName = listToken[1]; - - if (listToken[6].equals("BSS") || listToken[6].equals("ECON") - || listToken[6].equals("HIST") || listToken[6].equals("RELI") - || listToken[6].equals("RELP") || listToken[6].equals("RMSE") - || listToken[6].equals("RPS")) { - modelName = modelName.split("\\/")[0] + ensValue; - } - - // if the line type load selector is activated, check that the current line type is on the list - - if (listToken[6].equals("RMSE")) { - mvLoadStatInsertData.setLineType("CNT"); - } else if (listToken[6].equals("BSS")) { - mvLoadStatInsertData.setLineType("PSTD"); - } else if (listToken[6].equals("HIST")) { - mvLoadStatInsertData.setLineType("RHIST"); - } else if (listToken[6].equals("RELP")) { - mvLoadStatInsertData.setLineType("RELP"); - } else if (listToken[6].equals("SL1L2")) { - mvLoadStatInsertData.setLineType("SL1L2"); - } else if (listToken[6].equals("GRAD")) { - mvLoadStatInsertData.setLineType("GRAD"); - } else if (listToken[6].equals("SAL1L2")) { - mvLoadStatInsertData.setLineType("SAL1L2"); - } else if (listToken[6].equals("VL1L2")) { - mvLoadStatInsertData.setLineType("VL1L2"); - } else if (listToken[6].equals("VAL1L2")) { - mvLoadStatInsertData.setLineType("VAL1L2"); - } else if (listToken[6].equals("RPS")) { - mvLoadStatInsertData.setLineType("ENSCNT"); - } else if (listToken[6].equals("ECON")) { - mvLoadStatInsertData.setLineType("ECLV"); - } else if (listToken[6].equals("RELI")) { - mvLoadStatInsertData.setLineType("PCT"); - int intGroupSize = Integer.valueOf(listToken[1].split("\\/")[1]) + 1; - thresh = "==1/" + String.valueOf(intGroupSize); - } else if (listToken[6].startsWith("FHO")) { - mvLoadStatInsertData.setLineType("CTC"); - String[] threshArr = listToken[6].split("FHO"); - if (threshArr.length > 1) { - thresh = threshArr[1]; - } - } else if (listToken[6].startsWith("FSS")) { - mvLoadStatInsertData.setLineType("NBRCNT"); - String[] threshArr = listToken[6].split("FSS"); - if (threshArr.length > 1) { - thresh = threshArr[1]; - } - } else { - continue; - } - if (info.lineTypeLoad && !info.tableLineTypeLoad - .containsKey(mvLoadStatInsertData.getLineType())) { - continue; - } - - mvLoadStatInsertData.setFileLine(strFilename + ":" + intLine); - - // parse the valid times - - LocalDateTime fcstValidBeg = LocalDateTime.parse(listToken[3], formatStatVsdb); - - // format the valid times for the database insert - String fcstValidBegStr = DATE_FORMAT_1.format(fcstValidBeg); - - // calculate the number of seconds corresponding to fcst_lead - String strFcstLead = listToken[2]; - int intFcstLeadSec = Integer.parseInt(strFcstLead) * 3600; - strFcstLead = String.valueOf(Integer.parseInt(strFcstLead)); - - // determine the init time by combining fcst_valid_beg and fcst_lead - - LocalDateTime fcstInitBeg = LocalDateTime.from(fcstValidBeg); - fcstInitBeg = fcstInitBeg.minusSeconds(intFcstLeadSec); - String fcstInitBegStr = DATE_FORMAT_1.format(fcstInitBeg); - String obsValidBegStr = DATE_FORMAT_1.format(fcstValidBeg); - String fcstValidEndStr = DATE_FORMAT_1.format(fcstValidBeg); - String obsValidEndStr = DATE_FORMAT_1.format(fcstValidBeg); - - // ensure that the interp_pnts field value is a reasonable integer - String strInterpPnts = "0"; - - String strLineType = mvLoadStatInsertData.getLineType(); - - /* - * * * * stat_header insert * * * * - */ - headerRecords++; - - // build the value list for the stat_header search - String strStatHeaderValueList = modelName + listToken[7] + listToken[8] + - listToken[4] + listToken[5] + thresh; - headerIdString = ""; - - // look for the header key in the table - if (statHeaders.containsKey(strStatHeaderValueList)) { - headerIdString = statHeaders.get(strStatHeaderValueList); - } - - // if the stat_header is not in the table - else { - - // look for an existing stat_header record with the same information - boolean boolFoundStatHeader = false; - long intStatHeaderSearchBegin = new Date().getTime(); - if (info.statHeaderDBCheck) { - // build a Couchbase query to look for duplicate stat_header records - String strDataFileQuery = "SELECT " + - "meta().id as headerFileId, " + - "type, " + - "header_type, " + - "data_type, " + - "data_id " + - "FROM `" + - getBucket().name() + - "` WHERE " + - "type = \'header\' AND " + - "dbname = \'" + getDbName() + "\' AND " + - "header_type = \'stat\' AND " + - "data_type = \'" + info.luTypeName + "\' AND " + - "model = \'" + modelName + "\' AND " + - "fcst_var = \'" + listToken[7] + "\' AND " + - "fcst_lev = \'" + listToken[8] + "\' AND " + - "obtype = \'" + listToken[4] + "\' AND " + - "vx_mask = \'" + listToken[5] + "\' AND " + - "fcst_thresh = \'" + thresh + "\';"; - - try { - queryResult = getBucket().query(N1qlQuery.simple(strDataFileQuery)); - queryList = queryResult.allRows(); - // if the header document is already present in the database, print a warning and return the id - if (queryList.size() > 0) { - firstRow = queryList.get(0); - firstRowObject = firstRow.value(); - // set headerIdString to id of existing header document - headerIdString = firstRowObject.get("headerFileId").toString(); - boolFoundStatHeader = true; - // add header to table - statHeaders.put(strStatHeaderValueList, headerIdString); - logger.warn(" ** WARNING: header document already present in database"); - } - timeStats.put("headerSearchTime", - timeStats.get("headerSearchTime") + new Date().getTime() - - intStatHeaderSearchBegin); - } catch (CouchbaseException e) { - throw new Exception(e.getMessage()); - } - } // end if (info._boolStatHeaderDBCheck) - - // if the stat_header was not found, add it to the database and table - if (!boolFoundStatHeader) { - // create a unique data_file id from a Couchbase counter, starting at 1 the first time - try { - nextIdNumber = getBucket().counter("HDCounter", 1, 1).content(); - // unique id must be a string - headerIdString = getDbName() + "::header::stat::" + modelName + "::" + String - .valueOf( - nextIdNumber); - } catch (CouchbaseException e) { - throw new Exception(e.getMessage()); - } - - try { - headerFile = JsonObject.empty() - .put("type", "header") - .put("header_type", "stat") - .put("data_type", info.luTypeName) - .put("data_id", info.fileIdStr) - .put("dbname", getDbName()) - .put("version", listToken[0]) - .put("model", modelName) - .put("descr", "NA") - .put("fcst_var", listToken[7]) - .put("fcst_lev", listToken[8]) - .put("obs_var", listToken[7]) - .put("obs_lev", listToken[8]) - .put("obtype", listToken[4]) - .put("vx_mask", listToken[5]) - .put("interp_mthd", "NA") - .put("interp_pnts", strInterpPnts) - .put("fcst_thresh", thresh) - .put("obs_thresh", thresh); - doc = JsonDocument.create(headerIdString, headerFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from header INSERT"); - } else { - headerInserts++; - } - } catch (Exception e) { - throw new Exception(e.getMessage()); - } - // add header to table - statHeaders.put(strStatHeaderValueList, headerIdString); - } // end if (!boolFoundStatHeader) - } // end else stat_header is not in table - - if (headerIdString != null) { - dataRecords++; - // is the line type of variable length - boolean boolHasVarLengthGroups = MVUtil.lengthGroupIndices.containsKey(strLineType); - - // build the value list for the insert statement - String strLineDataValueList = - getDbName() + ESEP + // database name for ID - strLineType.toLowerCase() + ESEP + // line type - modelName + ESEP + // model name for ID - headerIdString + ESEP + // CB header_id - info.fileId + ESEP + // CB data_id for data_file - intLine + ESEP + // line_num - strFcstLead + ESEP + // fcst_lead - fcstValidBegStr + ESEP + // fcst_valid_beg - fcstValidEndStr + ESEP + // fcst_valid_end - fcstInitBegStr + ESEP + // fcst_init_beg - "0" + ESEP + // obs_lead - obsValidBegStr + ESEP + // obs_valid_beg - obsValidEndStr; // obs_valid_end - - // if the line data requires a cov_thresh value, add it - String strCovThresh = "NA"; - if (MVUtil.covThreshLineTypes.containsKey(strLineType)) { - strLineDataValueList += ESEP + replaceInvalidValues(strCovThresh); - } - - // if the line data requires an alpha value, add it - String strAlpha = "-9999"; - if (MVUtil.alphaLineTypes.containsKey(strLineType)) { - if (strAlpha.equals("NA")) { - logger.warn( - " ** WARNING: alpha value NA with line type '" + strLineType + "'\n " + - mvLoadStatInsertData.getFileLine()); - } - strLineDataValueList += ESEP + replaceInvalidValues(strAlpha); - } - - if (listToken[6].equals("RMSE")) {//CNT line type - for (int i = 0; i < 94; i++) { - if (i == 53) { - strLineDataValueList += ESEP + listToken[10]; - } else if (i == 31) { - strLineDataValueList += ESEP + listToken[11]; - } else if (i == 36) { - strLineDataValueList += ESEP + listToken[9]; - } else if (i == 44) { - strLineDataValueList += ESEP + listToken[12]; - } else if (i == 0 || i == 28 || i == 29 || i == 30) {//total,ranks, frank_ties, orank_ties - strLineDataValueList += ESEP + "0"; - } else if (i == 77) { - strLineDataValueList += ESEP + listToken[13]; - } else { - strLineDataValueList += ESEP + "-9999"; - } - } - } - - - if (listToken[6].equals("BSS")) {//PSTD line type - for (int i = 0; i < 17; i++) { - switch (i) { - case 0: - case 1: - strLineDataValueList += ESEP + "0"; - break; - case 2: - case 3: - case 8: - case 10: - case 11: - case 4: - case 13: - case 14: - case 16: - strLineDataValueList += ESEP + "-9999"; - break; - case 5: - strLineDataValueList += ESEP + listToken[12]; - break; - case 6: - strLineDataValueList += ESEP + listToken[13]; - break; - case 7: - strLineDataValueList += ESEP + listToken[14]; - break; - case 9: - strLineDataValueList += ESEP + listToken[9]; - break; - case 12: - strLineDataValueList += ESEP + listToken[10]; - break; - case 15: - strLineDataValueList += ESEP + listToken[11]; - break; - default: - } - - } - } - - if (listToken[6].equals("RPS")) {//ENSCNT line type - for (int i = 0; i < 30; i++) { - switch (i) { - case 0: - strLineDataValueList += ESEP + listToken[9]; - break; - case 1: - case 2: - case 3: - case 4: - strLineDataValueList += ESEP + "-9999"; - break; - case 5: - strLineDataValueList += ESEP + listToken[10]; - break; - case 6: - case 7: - case 8: - case 9: - strLineDataValueList += ESEP + "-9999"; - break; - case 10: - strLineDataValueList += ESEP + listToken[11]; - break; - case 11: - case 12: - case 13: - case 14: - strLineDataValueList += ESEP + "-9999"; - break; - case 15: - strLineDataValueList += ESEP + listToken[12]; - break; - case 16: - case 17: - case 18: - case 19: - strLineDataValueList += ESEP + "-9999"; - break; - case 20: - strLineDataValueList += ESEP + listToken[13]; - break; - case 21: - case 22: - case 23: - case 24: - strLineDataValueList += ESEP + "-9999"; - break; - case 25: - strLineDataValueList += ESEP + listToken[14]; - break; - case 26: - case 27: - case 28: - case 29: - strLineDataValueList += ESEP + "-9999"; - break; - default: - - } - } - - } - - if (listToken[6].equals("HIST")) { //RHIST line type - int intGroupSize = Integer.valueOf(listToken[1].split("\\/")[1]) + 1; - strLineDataValueList += ESEP + "0" + ESEP + intGroupSize; - } - - if (listToken[6].equals("RELP")) { // RELP line type - strLineDataValueList += ESEP + "0"; - int intGroupSize = Integer.valueOf(listToken[1].split("\\/")[1]); - strLineDataValueList += ESEP + intGroupSize; - } - - if (listToken[6].equals("ECON")) { // ECLV line type - strLineDataValueList += ESEP + "0" + ESEP + "-9999" + ESEP + "-9999"; - int intGroupSize = 18; - strLineDataValueList += ESEP + intGroupSize; - } - - if (listToken[6].equals("RELI")) { //PCT line type - int total = 0; - int intGroupSize; - int intGroupIndex = 9; - try { - intGroupSize = Integer.valueOf(listToken[1].split("\\/")[1]) + 1; - } catch (Exception e) { - intGroupSize = 0; - } - for (int i = 0; i < intGroupSize; i++) { - Integer on; - try { - on = Double.valueOf(listToken[intGroupIndex + intGroupSize]).intValue(); - total = total + on; - } catch (Exception e) { - logger.error(e.getMessage()); - } - intGroupIndex++; - } - - - strLineDataValueList += ESEP + total + ESEP + intGroupSize; - } - - if (listToken[6].equals("SL1L2") - || listToken[6].equals("SAL1L2")) {//SL1L2,SAL1L2 line types - for (int i = 0; i < 7; i++) { - if (i + 9 < listToken.length) { - if (i == 0) { - strLineDataValueList += ESEP + (Double.valueOf(listToken[i + 9])).intValue(); - } else { - strLineDataValueList += ESEP + Double.valueOf(listToken[i + 9]); - } - - } else { - strLineDataValueList += ESEP + "-9999"; - } - } - } - if (listToken[6].equals("VAL1L2") - || listToken[6].equals("GRAD")) {//VAL1L2,GRAD line type - for (int i = 0; i < 8; i++) { - if (i + 9 < listToken.length) { - if (i == 0) { - strLineDataValueList += ESEP + (Double.valueOf(listToken[i + 9])).intValue(); - } else { - strLineDataValueList += ESEP + Double.valueOf(listToken[i + 9]); - } - } else { - strLineDataValueList += ESEP + "-9999"; - } - - } - } - if (listToken[6].equals("VL1L2")) {//VL1L2 - for (int i = 0; i < 10; i++) { - if (i + 9 < listToken.length) { - if (i == 0) { - strLineDataValueList += ESEP + (Double.valueOf(listToken[i + 9])).intValue(); - } else { - strLineDataValueList += ESEP + Double.valueOf(listToken[i + 9]); - } - } else { - strLineDataValueList += ESEP + "-9999"; - } - - } - } - if (listToken[6].startsWith("FHO")) {//CTC line type - - double total = Double.parseDouble(listToken[9]); - double f_rate = Double.parseDouble(listToken[10]); - double h_rate = Double.parseDouble(listToken[11]); - double o_rate; - if (listToken.length > 12) { - o_rate = Double.valueOf(listToken[12]); - } else { - o_rate = 0; - logger.error("o_rate os 0"); - } - - double fy = total * f_rate; - double fy_oy = total * h_rate; - double oy = total * o_rate; - double fy_on = fy - fy_oy; - double fn_oy = oy - fy_oy; - double fn_on = total - fy - oy + fy_oy; - - - for (int i = 0; i < 5; i++) { - if (i == 4) { - strLineDataValueList += ESEP + Math.max(0, fn_on); - } else if (i == 3) { - strLineDataValueList += ESEP + Math.max(0, fn_oy); - } else if (i == 2) { - strLineDataValueList += ESEP + Math.max(0, fy_on); - } else if (i == 1) { - strLineDataValueList += ESEP + Math.max(0, fy_oy); - } else if (i == 0) {//total, - strLineDataValueList += ESEP + listToken[9]; - } - - } - } - if (listToken[6].startsWith("FSS")) {//NBRCNT line type - double fss = -9999; - if (listToken.length > 11) { - fss = 1 - Double.valueOf(listToken[10]) - / (Double.valueOf(listToken[11]) + Double.valueOf(listToken[12])); - } - for (int i = 0; i < 19; i++) { - if (i == 0) {//total, - strLineDataValueList += ESEP + listToken[9]; - } else if (i == 1) {//fbs - strLineDataValueList += ESEP + listToken[10]; - } else if (i == 4) {//fss - strLineDataValueList += ESEP + fss; - } else { - strLineDataValueList += ESEP + "-9999"; - } - } - } - - /* - * * * * var_length insert * * * * - */ - - if (boolHasVarLengthGroups) { - // get the index information about the current line type - int intGroupIndex = 0; - int intGroupSize = 0; - int intNumGroups = 0; - - if (listToken[6].equals("HIST")) {//RHIST line type - intGroupIndex = 9; - try { - intNumGroups = Integer.valueOf(listToken[1].split("\\/")[1]) + 1; - } catch (Exception e) { - intNumGroups = 0; - } - intGroupSize = 1; - } else if (listToken[6].equals("RELP")) {//RELP line type) - intGroupIndex = 9; - try { - intNumGroups = Integer.valueOf(listToken[1].split("\\/")[1]); - } catch (Exception e) { - intNumGroups = 0; - } - intGroupSize = 1; - } else if (listToken[6].equals("ECON")) {//ECLV line type) - intGroupIndex = 9; - intNumGroups = 18; - intGroupSize = 1; - } else if (listToken[6].equals("RELI")) {//PCT line type) - intGroupIndex = 9; - try { - intGroupSize = Integer.valueOf(listToken[1].split("\\/")[1]) + 1; - } catch (Exception e) { - intGroupSize = 0; - } - intNumGroups = 2; - } - - - String strVarType = tableVarLengthTable.get(strLineType); - String[] listFieldsArr; - listFieldsArr = tableLineDataFieldsTable.get(strVarType).split(","); - String strThreshValues; - - // create an array of objects - strThreshValues = "["; - - // build a insert value statement for each threshold group - if (listToken[6].equals("HIST")) { - for (int i = 0; i < intNumGroups; i++) { - strThreshValues += "{" + listFieldsArr[0] + ":" + (i + 1); - for (int j = 0; j < intGroupSize; j++) { - double res = Double.parseDouble(listToken[intGroupIndex++]); - if (res != -9999) { - strThreshValues += "," + listFieldsArr[j + 1] + ":" + (Math.round(res * 100)); - } - - } - strThreshValues += '}'; - // put a comma after every object except the last one - if (i != (intNumGroups - 1)) { - strThreshValues += ","; - } - lengthRecords++; - } - } else if (listToken[6].equals("RELI")) { - for (int i = 0; i < intGroupSize; i++) { - double thresh_i; - if (intGroupSize > 1) { - thresh_i = (double) i / (double) (intGroupSize - 1); - } else { - thresh_i = 0; - } - strThreshValues += "{" + listFieldsArr[0] + ":" + (i + 1); - strThreshValues += "," + listFieldsArr[1] + ":" + thresh_i; - Integer oy; - Integer on; - try { - oy = Double.valueOf(listToken[intGroupIndex]).intValue(); - on = Double.valueOf(listToken[intGroupIndex + intGroupSize]).intValue() - oy; - strThreshValues += "," + listFieldsArr[2] + ":" + oy; - strThreshValues += "," + listFieldsArr[3] + ":" + on; - } catch (Exception e) { - strThreshValues += ",-9999,-9999"; - } - - intGroupIndex++; - strThreshValues += "}"; - lengthRecords++; - } - } else if (listToken[6].equals("RELP")) { - for (int i = 0; i < intNumGroups; i++) { - strThreshValues += "{" + listFieldsArr[0] + ":" + (i + 1); - for (int j = 0; j < intGroupSize; j++) { - double res = Double.parseDouble(listToken[intGroupIndex++]); - if (res != -9999) { - strThreshValues += "," + listFieldsArr[j + 1] + ":" + (res); - } - } - strThreshValues += '}'; - // put a comma after every object except the last one - if (i != (intNumGroups - 1)) { - strThreshValues += ","; - } - lengthRecords++; - } - } else if (listToken[6].equals("ECON")) { - - for (int i = 0; i < intNumGroups; i++) { - strThreshValues += "{" + listFieldsArr[0] + ":" + (i + 1); - for (int j = 0; j < intGroupSize; j++) { - double res = Double.parseDouble(listToken[intGroupIndex++]); - if (res != -9999) { - strThreshValues += "," + listFieldsArr[j + 1] + ":" + (X_POINTS_FOR_ECON[i]); - } - } - strThreshValues += '}'; - // put a comma after every object except the last one - if (i != (intNumGroups - 1)) { - strThreshValues += ","; - } - lengthRecords++; - } - } - - strThreshValues += "]"; - strLineDataValueList += ESEP + strThreshValues; - } - - // add the values list to the line type values map - List listLineTypeValues = new ArrayList<>(); - if (mvLoadStatInsertData.getTableLineDataValues() - .containsKey(strLineType)) { - listLineTypeValues = mvLoadStatInsertData.getTableLineDataValues() - .get(strLineType); - } - listLineTypeValues.add(strLineDataValueList); - mvLoadStatInsertData.getTableLineDataValues() - .put(strLineType, listLineTypeValues); - dataInserts++; - - // if the insert threshhold has been reached, commit the stored data to the database - if (info.insertSize <= mvLoadStatInsertData.getListInsertValues().size()) { - int[] listInserts = commitStatData(mvLoadStatInsertData); - dataInserts += listInserts[INDEX_LINE_DATA]; - lengthInserts += listInserts[INDEX_VAR_LENGTH]; - } - } - } catch (Exception e) { - logger.error("ERROR: line:" + line + " has errors and would be ignored."); - logger.error(e.getMessage()); - } - - } // end: while( reader.ready() ) - fileReader.close(); - reader.close(); - } catch (Exception e) { - logger.error(e.getMessage()); - } - - // commit all the remaining stored data - int[] listInserts = commitStatData(mvLoadStatInsertData); - dataInserts += listInserts[INDEX_LINE_DATA]; - lengthInserts += listInserts[INDEX_VAR_LENGTH]; - - timeStats.put("linesTotal", (long) (intLine)); - timeStats.put("headerRecords", headerRecords); - timeStats.put("headerInserts", headerInserts); - timeStats.put("dataInserts", dataInserts); - timeStats.put("dataRecords", dataRecords); - timeStats.put("lengthRecords", lengthRecords); - timeStats.put("lengthInserts", lengthInserts); - - // print a performance report - long intStatHeaderLoadTime = new Date().getTime() - intStatHeaderLoadStart; - double dblLinesPerMSec = (double) (intLine) / (double) (intStatHeaderLoadTime); - - if (info.verbose) { - logger.info(MVUtil.padBegin("file lines: ", 6) + (intLine) + "\n" + - MVUtil.padBegin("stat_header records: ", 36) + headerRecords + "\n" + - MVUtil.padBegin("stat_header inserts: ", 36) + headerInserts + "\n" + - MVUtil.padBegin("line_data records: ", 36) + dataRecords + "\n" + - MVUtil.padBegin("line_data inserts: ", 36) + dataInserts + "\n" + - MVUtil.padBegin("line_data skipped: ", 36) + intLineDataSkipped + "\n" + - MVUtil.padBegin("var length records: ", 36) + lengthRecords + "\n" + - MVUtil.padBegin("var length inserts: ", 36) + lengthInserts + "\n" + - MVUtil.padBegin("total load time: ", 36) + MVUtil.formatTimeSpan( - intStatHeaderLoadTime) + "\n" + - MVUtil.padBegin("stat_header search time: ", 36) + MVUtil.formatTimeSpan( - headerSearchTime) + "\n" + - MVUtil.padBegin("lines / msec: ", 36) + MVUtil.DECIMAL_FORMAT.format( - dblLinesPerMSec) + "\n\n"); - } - logger.info("intLine " + intLine); - return timeStats; - } - - /** - * Load the MET output data from the data file underlying the input DataFileInfo object into the - * database underlying the input Connection. The header information can be checked in two - * different ways: using a table for the current file (specified by _boolModeHeaderTableCheck). - * Records in mode_obj_pair tables, mode_obj_single tables and mode_cts tables are created from - * the data in the input file. If necessary, records in the mode_header table are created. - * - * @param info Contains MET output data file information //* @param con Connection to the target - * database - * @throws Exception - */ - @Override - public Map loadModeFile(DataFileInfo info) throws DatabaseException { - Map timeStats = new HashMap<>(); - - // data structure for storing mode object ids - Map tableModeObjectId = new HashMap<>(); - - // performance counters - long intModeHeaderLoadStart = new Date().getTime(); - timeStats.put("headerSearchTime", 0L); - long headerInserts = 0; - long ctsInserts = 0; - long objSingleInserts = 0; - long objPairInserts = 0; - - // get the next mode record ids from the database - int intModeHeaderIdNext = getNextId("mode_header", "mode_header_id"); - int intModeObjIdNext = getNextId("mode_obj_single", "mode_obj_id"); - - // set up the input file for reading - String strFilename = info.path + "/" + info.filename; - int intLine = 1; - List headerNames = new ArrayList<>(); - try ( - FileReader fileReader = new FileReader(strFilename); - BufferedReader reader = new BufferedReader(fileReader)) { - // read each line of the input file - while (reader.ready()) { - String[] listToken = reader.readLine().split("\\s+"); - - // the first line is the header line - if (1 > listToken.length || listToken[0].equals("VERSION")) { - headerNames = Arrays.asList(listToken); - intLine++; - continue; - } - - String strFileLine = strFilename + ":" + intLine; - - // determine the line type - int intLineTypeLuId; - int intDataFileLuId = info.luId; - String strObjectId = MVUtil.findValue(listToken, headerNames, "OBJECT_ID"); - Matcher matModeSingle = MVUtil.patModeSingleObjectId.matcher(strObjectId); - Matcher matModePair = MVUtil.patModePairObjectId.matcher(strObjectId); - int modeCts = 19; - int modeSingle = 17; - int modePair = 18; - if (2 == intDataFileLuId) { - intLineTypeLuId = modeCts; - } else if (matModeSingle.matches()) { - intLineTypeLuId = modeSingle; - } else if (matModePair.matches()) { - intLineTypeLuId = modePair; - } else { - throw new Exception("METViewer load error: loadModeFile() unable to determine line type " - + MVUtil.findValue(listToken, headerNames, "OBJECT_ID") - + "\n " + strFileLine); - } - - - /* - * * * * mode_header insert * * * * - */ - - // parse the valid times - LocalDateTime fcstValidBeg = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "FCST_VALID"), - DB_DATE_STAT_FORMAT); - - - LocalDateTime obsValidBeg = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "OBS_VALID"), - DB_DATE_STAT_FORMAT); - - // format the valid times for the database insert - String fcstValidBegStr = DATE_FORMAT_1.format(fcstValidBeg); - String obsValidBegStr = DATE_FORMAT_1.format(obsValidBeg); - - // calculate the number of seconds corresponding to fcst_lead - String strFcstLead = MVUtil.findValue(listToken, headerNames, "FCST_LEAD"); - int intFcstLeadLen = strFcstLead.length(); - int intFcstLeadSec = Integer.parseInt( - strFcstLead.substring(intFcstLeadLen - 2, intFcstLeadLen)); - intFcstLeadSec += Integer.parseInt( - strFcstLead.substring(intFcstLeadLen - 4, intFcstLeadLen - 2)) * 60; - intFcstLeadSec += Integer.parseInt(strFcstLead.substring(0, intFcstLeadLen - 4)) * 3600; - - // determine the init time by combining fcst_valid_beg and fcst_lead - LocalDateTime fcstInitBeg = LocalDateTime.from(fcstValidBeg); - fcstInitBeg = fcstInitBeg.minusSeconds(intFcstLeadSec); - - String fcstInitStr = DATE_FORMAT_1.format(fcstInitBeg); - - // build a value list from the header information - //replace "NA" for fcst_accum (listToken[4]) and obs_accum (listToken[7]) to NULL - - String strModeHeaderValueList = - "'" + MVUtil.findValue(listToken, headerNames, - "VERSION") + "', " + // version - "'" + MVUtil.findValue(listToken, headerNames, - "MODEL") + "', "; // model - - if ("NA".equals(MVUtil.findValue(listToken, headerNames, "N_VALID"))) { - strModeHeaderValueList = strModeHeaderValueList + "NULL" + ", "; // N_VALID - } else { - strModeHeaderValueList = strModeHeaderValueList - + MVUtil.findValue(listToken, headerNames, "N_VALID") - + ", "; // N_VALID - } - if ("NA".equals(MVUtil.findValue(listToken, headerNames, "GRID_RES"))) { - strModeHeaderValueList = strModeHeaderValueList + "NULL" + ", "; // GRID_RES - } else { - strModeHeaderValueList = strModeHeaderValueList - + MVUtil.findValue(listToken, headerNames, "GRID_RES") - + ", "; // GRID_RES - } - - strModeHeaderValueList = strModeHeaderValueList - + "'" + MVUtil.findValue(listToken, headerNames, "DESC") - + "', " + // GRID_RES - "'" + MVUtil.findValue(listToken, headerNames, "FCST_LEAD") - + "', " + // fcst_lead - "'" + fcstValidBegStr + "', "; // fcst_valid - if ("NA".equals(MVUtil.findValue(listToken, headerNames, "FCST_ACCUM"))) { - strModeHeaderValueList = strModeHeaderValueList + "NULL" + ", "; // fcst_accum - } else { - strModeHeaderValueList = strModeHeaderValueList - + "'" - + MVUtil.findValue(listToken, headerNames, "FCST_ACCUM") - + "', "; // fcst_accum - } - strModeHeaderValueList = strModeHeaderValueList + "'" + fcstInitStr + "', " + // fcst_init - "'" - + MVUtil.findValue(listToken, headerNames, "OBS_LEAD") - + "', " + // obs_lead - "'" + obsValidBegStr + "', "; // obs_valid - if ("NA".equals(MVUtil.findValue(listToken, headerNames, "OBS_ACCUM"))) { - strModeHeaderValueList = strModeHeaderValueList + "NULL" + ", "; // obs_accum - } else { - strModeHeaderValueList = strModeHeaderValueList - + "'" - + MVUtil.findValue(listToken, headerNames, "OBS_ACCUM") - + "', "; // obs_accum - } - strModeHeaderValueList = strModeHeaderValueList - + "'" - + MVUtil.findValue(listToken, headerNames, "FCST_RAD") - + "', " + // fcst_rad - "'" + MVUtil.findValue(listToken, headerNames, - "FCST_THR") + "', " + // fcst_thr - "'" + MVUtil.findValue(listToken, headerNames, - "OBS_RAD") + "', " + // obs_rad - "'" + MVUtil.findValue(listToken, headerNames, - "OBS_THR") + "', " + // obs_thr - "'" + MVUtil.findValue(listToken, headerNames, - "FCST_VAR") + "', " + // fcst_var - "'" + MVUtil.findValue(listToken, headerNames, - "FCST_LEV") + "', " + // fcst_lev - "'" + MVUtil.findValue(listToken, headerNames, - "OBS_VAR") + "', " + // obs_var - "'" + MVUtil.findValue(listToken, headerNames, - "OBS_LEV") + "'"; // obs_lev - - String strModeHeaderWhereClause = - " version = '" + MVUtil.findValue(listToken, headerNames, "VERSION") + "'\n" + - " AND model = '" + MVUtil - .findValue(listToken, headerNames, "MODEL") + "'\n" + - " AND n_valid = '" + MVUtil.findValue(listToken, headerNames, - "N_VALID") + "'\n" + - " AND grid_res = '" + MVUtil.findValue(listToken, headerNames, - "GRID_RES") + "'\n" + - " AND descr = '" + MVUtil - .findValue(listToken, headerNames, "DESC") + "'\n" + - " AND fcst_lead = '" + MVUtil.findValue(listToken, headerNames, - "FCST_LEAD") + "'\n" + - " AND fcst_valid = '" + strFcstLead + "'\n" + - " AND fcst_accum = '" + MVUtil.findValue(listToken, headerNames, - "FCST_ACCUM") + "'\n" + - " AND fcst_init = '" + fcstInitStr + "'\n" + - " AND obs_lead = '" + MVUtil.findValue(listToken, headerNames, - "OBS_LEAD") + "'\n" + - " AND obs_valid = '" + obsValidBegStr + "'\n" + - " AND obs_accum = '" + MVUtil.findValue(listToken, headerNames, - "OBS_ACCUM") + "'\n" + - " AND fcst_rad = '" + MVUtil.findValue(listToken, headerNames, - "FCST_RAD") + "'\n" + - " AND fcst_thr = '" + MVUtil.findValue(listToken, headerNames, - "FCST_THR") + "'\n" + - " AND obs_rad = '" + MVUtil.findValue(listToken, headerNames, - "OBS_RAD") + "'\n" + - " AND obs_thr = '" + MVUtil.findValue(listToken, headerNames, - "OBS_THR") + "'\n" + - " AND fcst_var = '" + MVUtil.findValue(listToken, headerNames, - "FCST_VAR") + "'\n" + - " AND fcst_lev = '" + MVUtil.findValue(listToken, headerNames, - "FCST_LEV") + "'\n" + - " AND obs_var = '" + MVUtil.findValue(listToken, headerNames, - "OBS_VAR") + "'\n" + - " AND obs_lev = '" + MVUtil.findValue(listToken, headerNames, - "OBS_LEV") + "';"; - - // look for the header key in the table - int intModeHeaderId = -1; - if (modeHeaders.containsKey(strModeHeaderValueList)) { - intModeHeaderId = modeHeaders.get(strModeHeaderValueList); - } - - // if the mode_header does not yet exist, create one - else { - - // look for an existing mode_header record with the same information - boolean boolFoundModeHeader = false; - long intModeHeaderSearchBegin = new Date().getTime(); - if (info.modeHeaderDBCheck) { - String strModeHeaderSelect = "SELECT\n mode_header_id\nFROM\n mode_header\nWHERE\n" - + strModeHeaderWhereClause; - } - timeStats.put("headerSearchTime", timeStats.get("headerSearchTime") - + new Date().getTime() - intModeHeaderSearchBegin); - - - // if the mode_header was not found, add it to the table - if (!boolFoundModeHeader) { - - intModeHeaderId = intModeHeaderIdNext++; - modeHeaders.put(strModeHeaderValueList, intModeHeaderId); - - // build an insert statement for the mode header - strModeHeaderValueList = - intModeHeaderId + ", " + // mode_header_id - intLineTypeLuId + ", " + // line_type_lu_id - info.fileIdStr + ", " + // data_file_id - intLine + ", " + // linenumber - strModeHeaderValueList; - - // insert the record into the mode_header database table - String strModeHeaderInsert = "INSERT INTO mode_header VALUES (" - + strModeHeaderValueList + ");"; -// int intModeHeaderInsert = executeUpdate(strModeHeaderInsert); -// if (1 != intModeHeaderInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mode_header INSERT: " -// + intModeHeaderInsert + "\n " + strFileLine); -// } - headerInserts++; - } - } - - - /* - * * * * mode_cts insert * * * * - */ - - if (modeCts == intLineTypeLuId) { - - // build the value list for the mode_cts insert - String strCTSValueList = intModeHeaderId + ", '" - + MVUtil.findValue(listToken, headerNames, "FIELD") - + "'"; - int totalIndex = headerNames.indexOf("TOTAL"); - for (int i = 0; i < 18; i++) { - strCTSValueList += ", " + replaceInvalidValues(listToken[totalIndex + i]); - } - - // insert the record into the mode_cts database table -// String strModeCtsInsert = "INSERT INTO mode_cts VALUES (" + strCTSValueList + ");"; -// int intModeCtsInsert = executeUpdate(strModeCtsInsert); -// if (1 != intModeCtsInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mode_cts INSERT: " -// + intModeCtsInsert + "\n " + strFileLine); -// } - ctsInserts++; - - } - - /* - * * * * mode_obj_single insert * * * * - */ - - else if (modeSingle == intLineTypeLuId) { - - // build the value list for the mode_cts insert - int intModeObjId = intModeObjIdNext++; - String strSingleValueList = intModeObjId + ", " + intModeHeaderId + ", '" - + strObjectId + "'"; - for (String header : modeObjSingleColumns) { - strSingleValueList += ", '" + replaceInvalidValues( - MVUtil.findValue(listToken, headerNames, header)) + "'"; - } - - //set flags - Integer simpleFlag = 1; - Integer fcstFlag = 0; - if (strObjectId.startsWith("C")) { - simpleFlag = 0; - } - if (strObjectId.startsWith("CF") || strObjectId.startsWith("F")) { - fcstFlag = 1; - } - Integer matchedFlag = 0; - String[] objCatArr = MVUtil.findValue(listToken, headerNames, "OBJECT_CAT") - .split("_"); - if (objCatArr.length == 1 && !objCatArr[0].substring(2).equals("000")) { - matchedFlag = 1; - } - strSingleValueList = strSingleValueList + "," + fcstFlag + "," + simpleFlag + "," - + matchedFlag; - - // insert the record into the mode_obj_single database table - String strModeObjSingleInsert = "INSERT INTO mode_obj_single VALUES (" - + strSingleValueList + ");"; -// int intModeObjSingleInsert = executeUpdate(strModeObjSingleInsert); -// if (1 != intModeObjSingleInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mode_obj_single INSERT: " -// + intModeObjSingleInsert + "\n " + strFileLine); -// } - objSingleInserts++; - - // add the mode_obj_id to the table, using the object_id as the key - tableModeObjectId.put(strObjectId, intModeObjId); - - } - - /* - * * * * mode_obj_pair insert * * * * - */ - - else if (modePair == intLineTypeLuId) { - - // determine the mode_obj_id values for the pair - int intModeObjectIdFcst = tableModeObjectId.get(matModePair.group(1)); - int intModeObjectIdObs = tableModeObjectId.get(matModePair.group(2)); - - // build the value list for the mode_cts insert - String strPairValueList = intModeObjectIdObs + ", " + intModeObjectIdFcst - + ", " + intModeHeaderId + ", " + - "'" + strObjectId + "', '" - + MVUtil.findValue(listToken, headerNames, "OBJECT_CAT") - + "'"; - int centroiddistIndex = headerNames.indexOf("CENTROID_DIST"); - for (int i = 0; i < 12; i++) { - strPairValueList += ", " + replaceInvalidValues(listToken[centroiddistIndex + i]); - } - - //set flags - Integer simpleFlag = 1; - String[] objIdArr = strObjectId.split("_"); - if (objIdArr.length == 2 && objIdArr[0].startsWith("C") && objIdArr[1].startsWith("C")) { - simpleFlag = 0; - } - - Integer matchedFlag = 0; - String[] objCatArr = MVUtil.findValue(listToken, headerNames, "OBJECT_CAT") - .split("_"); - if (objCatArr.length == 2 && objCatArr[0].substring(2).equals(objCatArr[1].substring(2)) - && !objCatArr[0].substring(2).equals("000")) { - matchedFlag = 1; - } - strPairValueList = strPairValueList + "," + simpleFlag + "," + matchedFlag; - - // insert the record into the mode_obj_pair database table - String strModeObjPairInsert = "INSERT INTO mode_obj_pair VALUES (" - + strPairValueList + ");"; -// int intModeObjPairInsert = executeUpdate(strModeObjPairInsert); -// if (1 != intModeObjPairInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mode_obj_pair INSERT: " -// + intModeObjPairInsert + "\n " + strFileLine); -// } - objPairInserts++; - - } - - intLine++; - } - fileReader.close(); - reader.close(); - } catch (Exception e) { - logger.error(e.getMessage()); - } - - // increment the global mode counters - timeStats.put("linesTotal", (long) (intLine - 1)); - timeStats.put("headerInserts", headerInserts); - timeStats.put("ctsInserts", ctsInserts); - timeStats.put("objSingleInserts", objSingleInserts); - timeStats.put("objPairInserts", objPairInserts); - - - // print a performance report - if (info.verbose) { - long intModeHeaderLoadTime = new Date().getTime() - intModeHeaderLoadStart; - logger.info(MVUtil.padBegin("mode_header inserts: ", 36) + headerInserts + "\n" + - MVUtil.padBegin("mode_cts inserts: ", 36) + ctsInserts + "\n" + - MVUtil.padBegin("mode_obj_single inserts: ", 36) + objSingleInserts + "\n" + - MVUtil.padBegin("mode_obj_pair inserts: ", 36) + objPairInserts + "\n" + - (info.modeHeaderDBCheck ? MVUtil.padBegin("mode_header search time: ", - 36) + MVUtil.formatTimeSpan( - timeStats.get("headerSearchTime")) + "\n" : "") + - MVUtil.padBegin("total load time: ", 36) + MVUtil.formatTimeSpan( - intModeHeaderLoadTime) + "\n\n"); - } - return timeStats; - } - - /** - * Load the MET output data from the data file underlying the input DataFileInfo object into the - * database underlying the input Connection. The header information can be checked in two - * different ways: using a table for the current file (specified by _boolModeHeaderTableCheck). - * Records in mode_obj_pair tables, mode_obj_single tables and mode_cts tables are created from - * the data in the input file. If necessary, records in the mode_header table are created. - * - * @param info Contains MET output data file information //* @param con Connection to the target - * database - * @throws Exception - */ - @Override - public Map loadMtdFile(DataFileInfo info) throws DatabaseException { - Map timeStats = new HashMap<>(); - - // performance counters - long intMtdHeaderLoadStart = new Date().getTime(); - timeStats.put("headerSearchTime", 0L); - long headerInserts = 0; - long obj3dSingleInserts = 0; - long obj3dPairInserts = 0; - long obj2dInserts = 0; - - // get the next mode record ids from the database - int intMtdHeaderIdNext = getNextId("mtd_header", "mtd_header_id"); - - // set up the input file for reading - String strFilename = info.path + "/" + info.filename; - int intLine = 1; - List headerNames = new ArrayList<>(); - try ( - FileReader fileReader = new FileReader(strFilename); - BufferedReader reader = new BufferedReader(fileReader)) { - // read each line of the input file - while (reader.ready()) { - String lineStr = reader.readLine().trim(); - String[] listToken = lineStr.split("\\s+"); - - // the first line is the header line - if (1 > listToken.length || listToken[0].equals("VERSION")) { - headerNames = Arrays.asList(listToken); - intLine++; - continue; - } - - String strFileLine = strFilename + ":" + intLine; - - // determine the line type - int intLineTypeLuId; - int intDataFileLuId = info.luId; - String strObjectId = MVUtil.findValue(listToken, headerNames, "OBJECT_ID"); - int mtd3dSingle = 17; - int mtd3dPair = 18; - int mtd2d = 19; - if (11 == intDataFileLuId || 12 == intDataFileLuId) { - intLineTypeLuId = mtd3dSingle; - } else if (9 == intDataFileLuId || 10 == intDataFileLuId) { - intLineTypeLuId = mtd3dPair; - } else if (8 == intDataFileLuId) { - intLineTypeLuId = mtd2d; - } else { - throw new Exception("METViewer load error: loadModeFile() unable to determine line type" - + " " + strFileLine); - } - // parse the valid times - LocalDateTime fcstValidBeg = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "FCST_VALID"), - DB_DATE_STAT_FORMAT); - - - LocalDateTime obsValidBeg = LocalDateTime.parse( - MVUtil.findValue(listToken, headerNames, "OBS_VALID"), - DB_DATE_STAT_FORMAT); - - // format the valid times for the database insert - String fcstValidBegStr = DATE_FORMAT_1.format(fcstValidBeg); - String obsValidBegStr = DATE_FORMAT_1.format(obsValidBeg); - - // calculate the number of seconds corresponding to fcst_lead - String strFcstLead = MVUtil.findValue(listToken, headerNames, "FCST_LEAD"); - int intFcstLeadLen = strFcstLead.length(); - int intFcstLeadSec = 0; - try { - intFcstLeadSec = Integer.parseInt( - strFcstLead.substring(intFcstLeadLen - 2, intFcstLeadLen)); - intFcstLeadSec += Integer.parseInt( - strFcstLead.substring(intFcstLeadLen - 4, intFcstLeadLen - 2)) * 60; - intFcstLeadSec += Integer.parseInt(strFcstLead.substring(intFcstLeadLen - 6, - intFcstLeadLen - 4)) * 3600; - } catch (Exception e) { - } - String fcstLeadInsert = MVUtil.findValue(listToken, headerNames, "FCST_LEAD"); - if (fcstLeadInsert.equals("NA")) { - fcstLeadInsert = "0"; - } else { - if (fcstLeadInsert.contains("_")) { - fcstLeadInsert = fcstLeadInsert.split("_")[1]; - } - } - - String obsLeadInsert = MVUtil.findValue(listToken, headerNames, "OBS_LEAD"); - if (obsLeadInsert.equals("NA")) { - obsLeadInsert = "0"; - } else { - if (obsLeadInsert.contains("_")) { - obsLeadInsert = obsLeadInsert.split("_")[1]; - } - } - - // determine the init time by combining fcst_valid_beg and fcst_lead - LocalDateTime fcstInitBeg = LocalDateTime.from(fcstValidBeg); - fcstInitBeg = fcstInitBeg.minusSeconds(intFcstLeadSec); - - String fcstInitStr = DATE_FORMAT_1.format(fcstInitBeg); - - String mtdHeaderValueList = "'" + MVUtil.findValue(listToken, headerNames, "VERSION") - + "', " + "'" - + MVUtil.findValue(listToken, headerNames, "MODEL") - + "', " + "'" - + MVUtil.findValue(listToken, headerNames, "DESC") - + "', "; - - - mtdHeaderValueList = mtdHeaderValueList - + "'" - + fcstLeadInsert - + "', " + "'" + fcstValidBegStr + "', " - + "'" + fcstInitStr + "', " - + "'" + obsLeadInsert - + "', " + "'" + obsValidBegStr + "', "; - - if ("NA".equals(MVUtil.findValue(listToken, headerNames, "T_DELTA"))) { - mtdHeaderValueList = mtdHeaderValueList + "NULL" + ", "; - } else { - mtdHeaderValueList = mtdHeaderValueList + "'" - + MVUtil.findValue(listToken, headerNames, "T_DELTA") - + "', "; - } - mtdHeaderValueList = mtdHeaderValueList - + "'" + MVUtil.findValue(listToken, headerNames, "FCST_RAD") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "FCST_THR") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "OBS_RAD") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "OBS_THR") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "FCST_VAR") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "FCST_LEV") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "OBS_VAR") - + "', " + - "'" + MVUtil.findValue(listToken, headerNames, "OBS_LEV") - + "'"; - - String mtdHeaderWhereClause = - " version = '" + MVUtil.findValue(listToken, headerNames, "VERSION") - + "'\n" - + " AND model = '" + MVUtil.findValue(listToken, headerNames, "MODEL") - + "'\n" - + " AND descr = '" + MVUtil.findValue(listToken, headerNames, "DESC") - + "'\n" - + " AND fcst_lead = '" + fcstLeadInsert + "'\n" + - " AND fcst_valid = '" + fcstValidBegStr + "'\n" + - " AND t_delta = '" + MVUtil.findValue(listToken, headerNames, - "T_DELTA") + "'\n" + - " AND fcst_init = '" + fcstInitStr + "'\n" + - " AND obs_lead = '" + obsLeadInsert + "'\n" + - " AND obs_valid = '" + obsValidBegStr + "'\n" + - - " AND fcst_rad = '" + MVUtil.findValue(listToken, headerNames, - "FCST_RAD") + "'\n" + - " AND fcst_thr = '" + MVUtil.findValue(listToken, headerNames, - "FCST_THR") + "'\n" + - " AND obs_rad = '" + MVUtil.findValue(listToken, headerNames, - "OBS_RAD") + "'\n" + - " AND obs_thr = '" + MVUtil.findValue(listToken, headerNames, - "OBS_THR") + "'\n" + - " AND fcst_var = '" + MVUtil.findValue(listToken, headerNames, - "FCST_VAR") + "'\n" + - " AND fcst_lev = '" + MVUtil.findValue(listToken, headerNames, - "FCST_LEV") + "'\n" + - " AND obs_var = '" + MVUtil.findValue(listToken, headerNames, - "OBS_VAR") + "'\n" + - " AND obs_lev = '" + MVUtil.findValue(listToken, headerNames, - "OBS_LEV") + "';"; - - // look for the header key in the table - int mtdHeaderId = -1; - if (mtdHeaders.containsKey(mtdHeaderValueList)) { - mtdHeaderId = mtdHeaders.get(mtdHeaderValueList); - } - - // if the mtd_header does not yet exist, create one - else { - - // look for an existing mode_header record with the same information - boolean foundMtdHeader = false; - long mtdHeaderSearchBegin = new Date().getTime(); - if (info.mtdHeaderDBCheck) { - String strMtdHeaderSelect = "SELECT\n mtd_header_id\nFROM\n mtd_header\nWHERE\n" + - mtdHeaderWhereClause; - } - timeStats.put("headerSearchTime", timeStats.get("headerSearchTime") - + new Date().getTime() - mtdHeaderSearchBegin); - - - // if the mtd_header was not found, add it to the table - if (!foundMtdHeader) { - - mtdHeaderId = intMtdHeaderIdNext++; - mtdHeaders.put(mtdHeaderValueList, mtdHeaderId); - - // build an insert statement for the mtd header - mtdHeaderValueList = - mtdHeaderId + ", " + - intLineTypeLuId + ", " + - info.fileIdStr + ", " + - intLine + ", " + - mtdHeaderValueList; - - // insert the record into the mtd_header database table - String strMtdHeaderInsert = "INSERT INTO mtd_header VALUES (" + mtdHeaderValueList + - ");"; -// int intMtdHeaderInsert = executeUpdate(strMtdHeaderInsert); -// if (1 != intMtdHeaderInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mtd_header INSERT: " + intMtdHeaderInsert -// + "\n " + strFileLine); -// } - headerInserts++; - } - } - - - if (mtd3dSingle == intLineTypeLuId) { - String str3dSingleValueList = mtdHeaderId + ", '" + strObjectId + "'"; - for (String header : mtdObj3dSingleColumns) { - str3dSingleValueList += ", '" + replaceInvalidValues( - MVUtil.findValue(listToken, headerNames, header)) + "'"; - } - - //set flags - Integer simpleFlag = 1; - Integer fcstFlag = 0; - if (strObjectId.startsWith("C")) { - simpleFlag = 0; - } - if (strObjectId.startsWith("CF") || strObjectId.startsWith("F")) { - fcstFlag = 1; - } - Integer matchedFlag = 0; - String objCat = MVUtil.findValue(listToken, headerNames, "OBJECT_CAT"); - Integer num = null; - try { - num = Integer.valueOf(objCat.substring(objCat.length() - 3)); - } catch (Exception e) { - } - if (num != null && num != 0) { - matchedFlag = 1; - } - str3dSingleValueList = str3dSingleValueList + "," + fcstFlag + "," + simpleFlag + "," - + matchedFlag; - - // insert the record into the mtd_obj_single database table -// int mtd3dObjSingleInsert = executeUpdate("INSERT INTO mtd_3d_obj_single VALUES (" -// + str3dSingleValueList + ");"); -// if (1 != mtd3dObjSingleInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mtd_3d_obj_single INSERT: " -// + mtd3dObjSingleInsert + "\n " + strFileLine); -// } - obj3dSingleInserts++; - } else if (mtd2d == intLineTypeLuId) { - String str2dValueList = mtdHeaderId + ", '" + strObjectId + "'"; - for (String header : mtdObj2dColumns) { - str2dValueList += ", '" + replaceInvalidValues( - MVUtil.findValue(listToken, headerNames, header)) + "'"; - } - - //set flags - Integer simpleFlag = 1; - Integer fcstFlag = 0; - if (strObjectId.startsWith("C")) { - simpleFlag = 0; - } - if (strObjectId.startsWith("CF") || strObjectId.startsWith("F")) { - fcstFlag = 1; - } - Integer matchedFlag = 0; - String objCat = MVUtil.findValue(listToken, headerNames, "OBJECT_CAT"); - - Integer num = null; - try { - num = Integer.valueOf(objCat.substring(objCat.length() - 3)); - } catch (Exception e) { - } - if (num != null && num != 0) { - matchedFlag = 1; - } - str2dValueList = str2dValueList + "," + fcstFlag + "," + simpleFlag + "," + matchedFlag; - - // insert the record into the mtd_obj_single database table -// int mtd2dObjInsert = executeUpdate("INSERT INTO mtd_2d_obj VALUES (" -// + str2dValueList + ");"); -// if (1 != mtd2dObjInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mtd_2d_obj INSERT: " -// + mtd2dObjInsert + "\n " + strFileLine); -// } - obj2dInserts++; - } else if (mtd3dPair == intLineTypeLuId) { - - // build the value list for the mode_cts insert - String str3dPairValueList = mtdHeaderId + ", " - + "'" - + strObjectId - + "', '" - + MVUtil.findValue(listToken, headerNames, - "OBJECT_CAT") - + "'"; - int spaceCentroidDistIndex = headerNames.indexOf("SPACE_CENTROID_DIST"); - for (int i = 0; i < 11; i++) { - str3dPairValueList += ", " + replaceInvalidValues( - listToken[spaceCentroidDistIndex + i]); - } - - //set flags - Integer simpleFlag = 1; - String[] objIdArr = strObjectId.split("_"); - if (objIdArr.length == 2 && objIdArr[0].startsWith("C") && objIdArr[1].startsWith("C")) { - simpleFlag = 0; - } - - Integer matchedFlag = 0; - String[] objCatArr = MVUtil.findValue(listToken, headerNames, "OBJECT_CAT") - .split("_"); - Integer num1 = null; - Integer num2 = null; - try { - num1 = Integer.valueOf(objCatArr[0].substring(objCatArr[0].length() - 3)); - num2 = Integer.valueOf(objCatArr[1].substring(objCatArr[1].length() - 3)); - if (num1.equals(num2) && num1 != 0) { - matchedFlag = 1; - } - } catch (Exception e) { - } - - str3dPairValueList = str3dPairValueList + "," + simpleFlag + "," + matchedFlag; - -// int mtd3dObjPairInsert = executeUpdate("INSERT INTO mtd_3d_obj_pair VALUES (" -// + str3dPairValueList + ");"); -// if (1 != mtd3dObjPairInsert) { -// logger.warn( -// " ** WARNING: unexpected result from mtd_3d_obj_pair INSERT: " + -// mtd3dObjPairInsert + "\n " + strFileLine); -// } - obj3dPairInserts++; - - } - - intLine++; - } - fileReader.close(); - reader.close(); - } catch (Exception e) { - logger.error(e.getMessage()); - } - - // increment the global mode counters - timeStats.put("linesTotal", (long) (intLine - 1)); - timeStats.put("headerInserts", headerInserts); - timeStats.put("obj3dSingleInserts", obj3dSingleInserts); - timeStats.put("obj3dPairInserts", obj3dPairInserts); - timeStats.put("obj2dInserts", obj2dInserts); - - - // print a performance report - if (info.verbose) { - long intMtdHeaderLoadTime = new Date().getTime() - intMtdHeaderLoadStart; - logger.info(MVUtil.padBegin("mtd_header inserts: ", 36) + headerInserts + "\n" + - MVUtil - .padBegin("mtd_3d_obj_single inserts: ", 36) + obj3dSingleInserts + "\n" + - MVUtil.padBegin("mtd_3d_obj_pair inserts: ", 36) + obj3dPairInserts + "\n" + - MVUtil.padBegin("mtd_2d_obj inserts: ", 36) + obj2dInserts + "\n" + - (info.mtdHeaderDBCheck ? MVUtil.padBegin("mtd_header search time: ", - 36) + MVUtil.formatTimeSpan( - timeStats.get("headerSearchTime")) + "\n" : "") + - MVUtil.padBegin("total load time: ", 36) + MVUtil.formatTimeSpan( - intMtdHeaderLoadTime) + "\n\n"); - } - return timeStats; - } - - @Override - public void updateGroup(String group) throws DatabaseException { - String currentGroup = ""; - String currentDescription = ""; - String currentID = ""; - long nextIdNumber = 0; - int nrows = 0; - String nextIdString = ""; - N1qlQueryResult queryResult = null; - List queryList = null; - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - JsonDocument doc = null; - JsonDocument response = null; - JsonObject groupFile = null; - - String strDataFileQuery = "SELECT " + - "meta().id as groupId, " + - "type, " + - "`group`, " + - "description " + - "FROM `" + - getBucket().name() + - "` WHERE " + - "type = \'category\' AND " + - "dbname = \'" + getDbName() + "\';"; - - try { - queryResult = getBucket().query(N1qlQuery.simple(strDataFileQuery)); - queryList = queryResult.allRows(); - - // if a category document is present for the database, get the database group and the ID - if (queryList.size() > 0) { - firstRow = queryList.get(0); - firstRowObject = firstRow.value(); - currentGroup = firstRowObject.get("group").toString(); - currentDescription = firstRowObject.get("description").toString(); - currentID = firstRowObject.get("groupId").toString(); - nrows = nrows + 1; - } - - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - // if no category document exists, or the group is not the same as in the XML - if (!currentGroup.equals(group)) { - if (nrows == 0) { - try { - nextIdNumber = getBucket().counter("DFCounter", 1, 1).content(); - if (0 > nextIdNumber) { - throw new Exception("METViewer load error: processDataFile() unable to get counter"); - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - nextIdString = getDbName() + "::category::" + String.valueOf(nextIdNumber); - } else { - nextIdString = currentID; - } - try { - groupFile = JsonObject.empty() - .put("type", "category") - .put("dbname", getDbName()) - .put("group", group) - .put("description", currentDescription); - - doc = JsonDocument.create(nextIdString, groupFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from data_file INSERT"); - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - } - - } - - @Override - public void updateDescription(String description) throws DatabaseException { - String currentDescription = ""; - String currentGroup = ""; - String currentID = ""; - String newGroup = ""; - long nextIdNumber = 0; - int nrows = 0; - String nextIdString = ""; - N1qlQueryResult queryResult = null; - List queryList = null; - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - JsonDocument doc = null; - JsonDocument response = null; - JsonObject groupFile = null; - - String strDataFileQuery = "SELECT " + - "meta().id as groupId, " + - "type, " + - "`group`, " + - "description " + - "FROM `" + - getBucket().name() + - "` WHERE " + - "type = \'category\' AND " + - "dbname = \'" + getDbName() + "\';"; - - try { - queryResult = getBucket().query(N1qlQuery.simple(strDataFileQuery)); - queryList = queryResult.allRows(); - - // if a category document is present for the database, get the database group and description and the ID - if (queryList.size() > 0) { - firstRow = queryList.get(0); - firstRowObject = firstRow.value(); - currentGroup = firstRowObject.get("group").toString(); - currentDescription = firstRowObject.get("description").toString(); - currentID = firstRowObject.get("groupId").toString(); - nrows = nrows + 1; - } - - } catch (Exception e) { - System.out.println(e.getMessage()); - } - // if no category document exists, or the group is not the same as in the XML - if (!currentDescription.equals(description)) { - if (nrows == 0) { - try { - nextIdNumber = getBucket().counter("DFCounter", 1, 1).content(); - if (0 > nextIdNumber) { - throw new Exception("METViewer load error: processDataFile() unable to get counter"); - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - nextIdString = getDatabaseInfo().getDbName() + "::category::" + String - .valueOf(nextIdNumber); - } else { - nextIdString = currentID; - newGroup = currentGroup; - } - try { - groupFile = JsonObject.empty() - .put("type", "category") - .put("dbname", getDbName()) - .put("group", newGroup) - .put("description", description); - - doc = JsonDocument.create(nextIdString, groupFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from data_file INSERT"); - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - } - } - - private int executeBatch(List listValues) throws DatabaseException { - long nextIdNumber; - String lineDataIdString; - JsonObject lineDataFile; - JsonDocument response; - JsonDocument doc; - String[] listFieldsArr; - String[] listValuesArr; - String[] listObjectArr; - - int intResLineDataInsert = 0; - - // reserve a block of counters to add to ids for all the documents - try { - nextIdNumber = getBucket().counter("LDCounter", listValues.size(), 1).content() - listValues - .size(); - } catch (CouchbaseException e) { - throw new DatabaseException(e.getMessage()); - } - - for (int i = 0; i < listValues.size(); i++) { - - listValuesArr = listValues.get(i).split(ESEP); - // unique id must be a string - lineDataIdString = listValuesArr[0] + "::line::" + listValuesArr[1] + "::" + - listValuesArr[2] + "::" + String.valueOf(nextIdNumber++); - - try { - lineDataFile = JsonObject.empty() - .put("type", "line") - .put("line_type", listValuesArr[1]) - .put("header_id", listValuesArr[3]) - .put("data_id", listValuesArr[4]) - .put("dbname", getDbName()) - .put("line_num", listValuesArr[5]) - .put("fcst_lead", listValuesArr[6]) - .put("fcst_valid_beg", listValuesArr[7]) - .put("fcst_valid_end", listValuesArr[8]) - .put("fcst_init_beg", listValuesArr[9]) - .put("obs_lead", listValuesArr[10]) - .put("obs_valid_beg", listValuesArr[11]) - .put("obs_valid_end", listValuesArr[12]); - - listFieldsArr = tableLineDataFieldsTable.get(listValuesArr[1]).split(","); - String strArrayFields; - int intOpen, intClose, intColon; - JsonArray variableValues = null; - JsonObject variablePairs = null; - for (int j = 0; j < listFieldsArr.length; j++) { - if (listValuesArr[j + 13].contains("[")) { - variableValues = JsonArray.empty(); - while (listValuesArr[j + 13].contains("{")) { - intOpen = listValuesArr[j + 13].indexOf("{"); - intClose = listValuesArr[j + 13].indexOf("}"); - strArrayFields = listValuesArr[j + 13].substring(intOpen + 1, intClose); - listValuesArr[j + 13] = listValuesArr[j + 13].substring(intClose + 1); - listObjectArr = strArrayFields.split(","); - variablePairs = JsonObject.empty(); - for (String element : listObjectArr) { - intColon = element.indexOf(":"); - variablePairs.put(element.substring(0, intColon), element.substring(intColon + 1)); - } - variableValues.add(variablePairs); - } - lineDataFile.put(listFieldsArr[j], variableValues); - } else { - lineDataFile.put(listFieldsArr[j], listValuesArr[j + 13]); - } - } - - doc = JsonDocument.create(lineDataIdString, lineDataFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from line data INSERT"); - } else { - intResLineDataInsert++; - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - - } - - return intResLineDataInsert; - } - - /** - * Analyze the input file object to determine what type of MET output file it is. Create an entry - * in the data_file table for the file and build a DataFileInfo data structure with information - * about the file and return it. - * - * @param file points to a MET output file to process // * @param con database connection to use - * @return data structure containing information about the input file - */ - @Override - public DataFileInfo processDataFile(File file, boolean forceDupFile) throws DatabaseException { - String strPath = file.getParent().replace("\\", "/"); - String strFile = file.getName(); - int strDataFileLuId = -1; - String strDataFileLuTypeName; - Integer dataFileId; - JsonDocument doc; - N1qlQueryResult queryResult = null; - List queryList = null; - N1qlQueryRow firstRow = null; - JsonObject firstRowObject = null; - Long nextIdNumber = 0L; - String nextIdString = ""; - JsonObject dataFile; - JsonDocument response; - - //check file size and return if it is 0 - if (file.length() == 0) { - return null; - } - // set default values for the loaded time (now) and the modified time (that of input file) - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - String strLoadDate = DATE_FORMAT.format(cal.getTime()); - cal.setTimeInMillis(file.lastModified()); - String strModDate = DATE_FORMAT.format(cal.getTime()); - - // determine the type of the input data file by parsing the filename - if (strFile.matches("\\S+\\.stat$")) { - strDataFileLuTypeName = "stat"; - } else if (strFile.matches("\\S+_obj\\.txt$")) { - strDataFileLuTypeName = "mode_obj"; - } else if (strFile.matches("\\S+_cts\\.txt$")) { - strDataFileLuTypeName = "mode_cts"; - } else if (strFile.matches("\\S+\\.vsdb$")) { - strDataFileLuTypeName = "vsdb_point_stat"; - } else if (strFile.matches("\\S+2d.txt$")) { - strDataFileLuTypeName = "mtd_2d"; - } else if (strFile.matches("\\S+3d_pair_cluster.txt$")) { - strDataFileLuTypeName = "mtd_3d_pc"; - } else if (strFile.matches("\\S+3d_pair_simple.txt$")) { - strDataFileLuTypeName = "mtd_3d_ps"; - } else if (strFile.matches("\\S+3d_single_cluster.txt$")) { - strDataFileLuTypeName = "mtd_3d_sc"; - } else if (strFile.matches("\\S+3d_single_simple.txt$")) { - strDataFileLuTypeName = "mtd_3d_ss"; - } else { - return null; - } - - if (!"".equals(strDataFileLuTypeName)) { - strDataFileLuId = tableDataFileLU.get(strDataFileLuTypeName); - } - // for compile. remove when CB fully in - dataFileId = 0; - - // build a Couchbase query to look for the file and path in the data_file table - String strDataFileQuery = "SELECT " + - "meta().id as dataFileId, " + - "type, " + - "data_type, " + - "load_date, " + - "mod_date " + - "FROM `" + - getBucket().name() + - "` WHERE " + - "type = \'file\' AND " + - "dbname = \'" + getDbName() + "\' AND " + - "data_type = \'" + strDataFileLuTypeName + "\' AND " + - "filename = \'" + strFile + "\' AND " + - "`path` = \'" + strPath + "\';"; - - try { - queryResult = getBucket().query(N1qlQuery.simple(strDataFileQuery)); - queryList = queryResult.allRows(); - - // if the data file is already present in the database, print a warning and return the id - if (queryList.size() > 0) { - if (forceDupFile) { - firstRow = queryList.get(0); - firstRowObject = firstRow.value(); - // set dupIdString to id of existing data file document - strLoadDate = firstRowObject.get("load_date").toString(); - strModDate = firstRowObject.get("mod_date").toString(); - // for couchbase, pass string id dupidstring, already in database, as new id - DataFileInfo info = - new DataFileInfo(firstRowObject.get("dataFileId").toString(), - strFile, strPath, strLoadDate, - strModDate, strDataFileLuId, strDataFileLuTypeName); - logger.warn(" ** WARNING: file already present in table data_file"); - return info; - } else { - throw new DatabaseException( - "file already present in table data_file, use force_dup_file setting to override"); - } - } - - } catch (CouchbaseException e) { - throw new DatabaseException(e.getMessage()); - } - // create a unique string data_file id from a Couchbase counter, starting at 1 the first time - try { - nextIdNumber = getBucket().counter("DFCounter", 1, 1).content(); - if (0 > nextIdNumber) { - throw new DatabaseException( - "METViewer load error: processDataFile() unable to get counter"); - } - - } catch (CouchbaseException e) { - throw new DatabaseException(e.getMessage()); - } - nextIdString = getDatabaseInfo() - .getDbName() + "::file::" + strDataFileLuTypeName + "::" + nextIdNumber; - try { - dataFile = JsonObject.empty() - .put("type", "file") - .put("data_type", strDataFileLuTypeName) - .put("dbname", getDbName()) - .put("filename", strFile) - .put("path", strPath) - .put("load_date", strLoadDate) - .put("mod_date", strModDate); - - doc = JsonDocument.create(nextIdString, dataFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from data_file INSERT"); - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - return new DataFileInfo(nextIdString, strFile, strPath, strLoadDate, strModDate, - strDataFileLuId, - strDataFileLuTypeName); - } - - @Override - public void updateInfoTable(String strXML, MVLoadJob job) throws DatabaseException { - Long nextIdNumber = 0L; - String nextIdString = ""; - String headerIdString = ""; - JsonObject instanceFile; - JsonDocument response; - JsonDocument doc; - JsonObject firstRowObject = null; - - // get the instance_info information to insert - String updater = System.getProperty("user.name"); - if(updater == null || updater.isEmpty()){ - updater="mvuser"; - } - String strUpdateDate = DATE_FORMAT.format(new Date()); - String strUpdateDetail = job.getLoadNote(); - - // read the load xml into a string, if requested - String strLoadXML = ""; - if (job.getLoadXML()) { - strXML = MVUtil.cleanString(strXML); - try (BufferedReader reader = new BufferedReader(new FileReader(strXML))) { - while (reader.ready()) { - strLoadXML += reader.readLine().trim(); - } - } catch (IOException e) { - logger.error(e.getMessage()); - } - } - - // create a unique string data_file id from a Couchbase counter, starting at 1 the first time - try { - nextIdNumber = getBucket().counter("DFCounter", 1, 1).content(); - if (0 > nextIdNumber) { - throw new DatabaseException("METViewer load error: updateInfoTable() unable to get " - + "counter"); - } - } catch (CouchbaseException e) { - throw new DatabaseException(e.getMessage()); - } - // Create new id for data file job document - nextIdString = getDbName() + "::job::" + nextIdNumber; - - // execute the CB insert - logger.info("Inserting instance_info record... "); - - try { - instanceFile = JsonObject.empty() - .put("type", "job") - .put("dbname", getDbName()) - .put("updater", updater) - .put("update_date", strUpdateDate) - .put("update_note", strUpdateDetail) - .put("xml_test", strLoadXML); - - doc = JsonDocument.create(nextIdString, instanceFile); - response = getBucket().upsert(doc); - if (response.content().isEmpty()) { - logger.warn(" ** WARNING: unexpected result from instance_info INSERT"); - } - } catch (Exception e) { - throw new DatabaseException(e.getMessage()); - } - - logger.info("Done\n"); - } - - private String replaceInvalidValues(String strData) { - return strData.replace("NA", "-9999").replace("-nan", "-9999").replace("nan", "-9999"); - } - -} diff --git a/lib/couchbase-core-io-1.5.8.jar b/lib/couchbase-core-io-1.5.8.jar deleted file mode 100644 index 89f17f85..00000000 Binary files a/lib/couchbase-core-io-1.5.8.jar and /dev/null differ diff --git a/lib/couchbase-java-client-2.5.8.jar b/lib/couchbase-java-client-2.5.8.jar deleted file mode 100644 index 71b9e6ba..00000000 Binary files a/lib/couchbase-java-client-2.5.8.jar and /dev/null differ diff --git a/lib/rxjava-1.3.4.jar b/lib/rxjava-1.3.4.jar deleted file mode 100644 index 7bdbd63e..00000000 Binary files a/lib/rxjava-1.3.4.jar and /dev/null differ diff --git a/webapp/metviewer/WEB-INF/classes/build.properties b/webapp/metviewer/WEB-INF/classes/build.properties index 1b9ddbe2..edd27c5c 100644 --- a/webapp/metviewer/WEB-INF/classes/build.properties +++ b/webapp/metviewer/WEB-INF/classes/build.properties @@ -4,7 +4,6 @@ db.host= db.user= db.password= -# change to db.managementSystem=cb to build for couchbase db.management.system=mariadb # should redirect to the metviewer path (where metviewer is deployed) diff --git a/webapp/metviewer/WEB-INF/classes/buildCB.properties b/webapp/metviewer/WEB-INF/classes/buildCB.properties deleted file mode 100644 index b64090c6..00000000 --- a/webapp/metviewer/WEB-INF/classes/buildCB.properties +++ /dev/null @@ -1,32 +0,0 @@ -# build properties for ucar installation - default -# customize these paths for a custom installation -db.host= -db.user= -db.password= - -# change to db.managementSystem=cb to build for couchbase -db.management.system=cb - -# should redirect to the metviewer path (where metviewer is deployed) -# the metviewer war file will be named after this i.e. ${redirect}.war -# so that the war file can be automatically deployed -redirect=metviewer - -# metviewer xml output directory -output.dir= -# metviewer webapps directory -webapps.dir= -# metviewer xml directory - will be placed under the output directory -xml.dir=xml -#metviewer R template directory - will be placed under the webapps directory -rtmpl.dir=R_tmpl -# metviewer R work directory - will be placed under the webapps directory -rwork.dir=R_work -# metviewer plots output diectory - - will be placed under the output directory -plots.dir=plots -# metviewer data output diectory - will be placed under the output directory -data.dir=data -# metviewer scripts output diectory - will be placed under the output directory -scripts.dir=scripts -# metviewer url for getting to the output directory -url.output=http://www.dtcenter.org/met/metviewer_output/ diff --git a/webapp/metviewer/WEB-INF/classes/log4j2.xml b/webapp/metviewer/WEB-INF/classes/log4j2.xml index 358f7b70..2734a67d 100644 --- a/webapp/metviewer/WEB-INF/classes/log4j2.xml +++ b/webapp/metviewer/WEB-INF/classes/log4j2.xml @@ -2,28 +2,18 @@ - - - - - - - + - - - - - - - + - + diff --git a/webapp/metviewer/WEB-INF/classes/log4j2.xml.orig b/webapp/metviewer/WEB-INF/classes/log4j2.xml.orig index 358f7b70..2734a67d 100644 --- a/webapp/metviewer/WEB-INF/classes/log4j2.xml.orig +++ b/webapp/metviewer/WEB-INF/classes/log4j2.xml.orig @@ -2,28 +2,18 @@ - - - - - - - + - - - - - - - + - +