From be86750429854784a721ed8848b0050b38a5eb69 Mon Sep 17 00:00:00 2001 From: Sanket Sarang Date: Sat, 3 Aug 2019 16:05:59 +0530 Subject: [PATCH] Resolved issue with boolean values in SQL WHERE clause --- Dockerfile | 2 +- bean-manager/pom.xml | 4 +- console-end-point/pom.xml | 2 +- distribution/pom.xml | 2 +- engine/pom.xml | 14 ++++++- .../db/bquery/BQueryExecutorBean.java | 1 + .../db/bquery/ConsoleExecutorBean.java | 37 +++++++++++++++++++ .../com/blobcity/db/bsql/BSqlDataManager.java | 3 ++ .../db/bsql/filefilters/EQFilenameFilter.java | 4 ++ .../com/blobcity/db/exceptions/ErrorCode.java | 3 +- .../blobcity/db/importer/ExcelImporter.java | 33 +++++++++++++++++ .../com/blobcity/db/sql/lang/Aggregate.java | 28 +++++++------- .../sql/processing/OnDiskWhereHandling.java | 4 ++ .../db/sql/statements/SelectExecutor.java | 5 +++ launcher/pom.xml | 2 +- pom.xml | 2 +- scripts/blobcity.sh | 2 +- tableau/pom.xml | 4 +- tcp-end-point/pom.xml | 2 +- web-end-point/pom.xml | 2 +- 20 files changed, 128 insertions(+), 28 deletions(-) create mode 100644 engine/src/main/java/com/blobcity/db/importer/ExcelImporter.java diff --git a/Dockerfile b/Dockerfile index 7918559..1176371 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ FROM blobcity/java8-ubuntu -ENV DB_VER=1.7.8-alpha +ENV DB_VER=1.7.9-alpha RUN cd / && mkdir data diff --git a/bean-manager/pom.xml b/bean-manager/pom.xml index 78e2543..1146699 100644 --- a/bean-manager/pom.xml +++ b/bean-manager/pom.xml @@ -22,7 +22,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha com.blobcity.lib @@ -63,7 +63,7 @@ along with this program. If not, see . com.google.guava guava - 22.0 + 28.0-jre diff --git a/console-end-point/pom.xml b/console-end-point/pom.xml index 689784b..937022c 100644 --- a/console-end-point/pom.xml +++ b/console-end-point/pom.xml @@ -22,7 +22,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha com.blobcity.lib diff --git a/distribution/pom.xml b/distribution/pom.xml index 6011deb..560dfcc 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -21,7 +21,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha blobcity-db BlobCityDB Distribution diff --git a/engine/pom.xml b/engine/pom.xml index f570ff8..96cdf09 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -22,7 +22,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha com.blobcity.lib.database @@ -540,6 +540,10 @@ along with this program. If not, see . com.fasterxml.jackson.core jackson-core + + com.google.code.findbugs + jsr305 + @@ -585,6 +589,10 @@ along with this program. If not, see . commons-codec commons-codec + + com.google.code.findbugs + jsr305 + @@ -617,6 +625,10 @@ along with this program. If not, see . com.google.collections google-collections + + com.google.code.findbugs + jsr305 + diff --git a/engine/src/main/java/com/blobcity/db/bquery/BQueryExecutorBean.java b/engine/src/main/java/com/blobcity/db/bquery/BQueryExecutorBean.java index 8e3e990..74e05b0 100644 --- a/engine/src/main/java/com/blobcity/db/bquery/BQueryExecutorBean.java +++ b/engine/src/main/java/com/blobcity/db/bquery/BQueryExecutorBean.java @@ -1401,6 +1401,7 @@ private JSONObject search(final String datastore, final JSONObject payloadJson) case GTEQ: case LT: case LTEQ: + System.out.println("The comparator: " + jsonObject.toString()); iterator = dataManager.selectWithPattern(datastore, selectTables.toArray()[0].toString(), jsonObject.getString("c"), jsonObject.get("v"), operator); break; case IN: diff --git a/engine/src/main/java/com/blobcity/db/bquery/ConsoleExecutorBean.java b/engine/src/main/java/com/blobcity/db/bquery/ConsoleExecutorBean.java index 94c16dc..d9780e5 100644 --- a/engine/src/main/java/com/blobcity/db/bquery/ConsoleExecutorBean.java +++ b/engine/src/main/java/com/blobcity/db/bquery/ConsoleExecutorBean.java @@ -455,6 +455,9 @@ public String runCommand(final String user, String query) { case "import-csv": response = importCSV(elements); break; + case "import-excel": + response = importExcel(elements); + break; case "insert-custom": response = insertCustom(elements); break; @@ -1466,6 +1469,40 @@ private String importCSV(String[] elements) throws OperationException { operationsManager.registerOperation(datastore, collection, OperationTypes.IMPORT, opSpecs); +// mapReduceOutputImporter.importCSVFile(datastore, collection, filePath); + + long elapsedTime = System.currentTimeMillis() - startTime; + return "Done in " + elapsedTime + " (ms)"; + } + + /** + * Used to import an Excel document in a specified table. Columns of the table correspond to columns of the Excel + * and the entry will have one record per row. Sheet name is also stored as a field per record. + * @param elements query elements from CLI request + * @return import success/fail status response in text form + * @throws OperationException if an error occurs while executing the oepration + */ + private String importExcel(String[] elements) throws OperationException { + long startTime = System.currentTimeMillis(); + final String databaseAndTable = elements[1]; + if (!databaseAndTable.contains(".")) { + throw new OperationException(ErrorCode.INVALID_QUERY_FORMAT, "Database and table should be specified in format: databaseName.tableName"); + } + final String datastore = databaseAndTable.substring(0, databaseAndTable.indexOf(".", 1)); //start searching dot from index 1 to handle case of .systemdb as datastore name + final String collection = databaseAndTable.substring(databaseAndTable.indexOf(".", 1) + 1, databaseAndTable.length()); + + verifyDCInfo(datastore, collection); + + /* Maybe a public URL or a local file system path. NFS paths are currently not supported */ + String filePath = elements[2]; + + JSONObject opSpecs = new JSONObject(); + opSpecs.put("type", "IMP"); + opSpecs.put("import-type", "CSV"); + opSpecs.put("file", filePath); + + operationsManager.registerOperation(datastore, collection, OperationTypes.IMPORT, opSpecs); + // mapReduceOutputImporter.importCSVFile(datastore, collection, filePath); long elapsedTime = System.currentTimeMillis() - startTime; diff --git a/engine/src/main/java/com/blobcity/db/bsql/BSqlDataManager.java b/engine/src/main/java/com/blobcity/db/bsql/BSqlDataManager.java index 144abdb..c8c6de3 100644 --- a/engine/src/main/java/com/blobcity/db/bsql/BSqlDataManager.java +++ b/engine/src/main/java/com/blobcity/db/bsql/BSqlDataManager.java @@ -523,6 +523,9 @@ public Iterator selectKeysWithPattern(final String datastore, final Stri case DOUBLE: comparableReferenceValue = new Double(referenceValue.toString()); break; + case BOOLEAN: + comparableReferenceValue = (boolean) referenceValue; + break; case STRING: case VARCHAR: case CHARACTER_LARGE_OBJECT: diff --git a/engine/src/main/java/com/blobcity/db/bsql/filefilters/EQFilenameFilter.java b/engine/src/main/java/com/blobcity/db/bsql/filefilters/EQFilenameFilter.java index abff25d..d3883e8 100644 --- a/engine/src/main/java/com/blobcity/db/bsql/filefilters/EQFilenameFilter.java +++ b/engine/src/main/java/com/blobcity/db/bsql/filefilters/EQFilenameFilter.java @@ -66,6 +66,10 @@ public TypeConverter getTypeConverter() { @Override public Object getTypeConvertedReferenceValue() throws OperationException { + if(referenceValue instanceof Boolean) { + if(((Boolean) referenceValue).booleanValue()) return "true"; + else return "false"; + } return typeConverter.getValue(referenceValue.toString()); } } diff --git a/engine/src/main/java/com/blobcity/db/exceptions/ErrorCode.java b/engine/src/main/java/com/blobcity/db/exceptions/ErrorCode.java index 2b528d2..7fc9b65 100644 --- a/engine/src/main/java/com/blobcity/db/exceptions/ErrorCode.java +++ b/engine/src/main/java/com/blobcity/db/exceptions/ErrorCode.java @@ -128,7 +128,8 @@ public enum ErrorCode { INVALID_WEBSERVICE_ENDPOINT("INVALID_WEBSERVICE_ENDPOINT", "No customer web-service registered at the specified endpoint"), CODE_LOAD_ERROR("CODE_LOAD_ERROR", "Unable to load code"), TABLEAU_EXCEPTION("TABLEAU_EXCEPTION", "Internal error with Tableau integration"), - GROUP_BY("GROUP_BY", "Error executing GROUP BY clause"); + GROUP_BY("GROUP_BY", "Error executing GROUP BY clause"), + EXCEL_DATA_READ_ERR("EXCEL_DATA_READ_ERR", "Error reading data in Excel format"); private final String errorCode; private final String errorMessage; diff --git a/engine/src/main/java/com/blobcity/db/importer/ExcelImporter.java b/engine/src/main/java/com/blobcity/db/importer/ExcelImporter.java new file mode 100644 index 0000000..b9fbe2e --- /dev/null +++ b/engine/src/main/java/com/blobcity/db/importer/ExcelImporter.java @@ -0,0 +1,33 @@ +package com.blobcity.db.importer; + +import com.blobcity.db.exceptions.ErrorCode; +import com.blobcity.db.exceptions.OperationException; +import org.apache.poi.openxml4j.exceptions.InvalidFormatException; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.springframework.stereotype.Component; +import org.tensorflow.Operation; + +import java.io.File; +import java.io.IOException; + +@Component +public class ExcelImporter { + + public void importExcel(final String ds, final String collection, final String pathOrUrl) throws OperationException { + final String filepath = pathOrUrl.startsWith("http:") ? downloadFile(pathOrUrl) : pathOrUrl; + + try { + Workbook workbook = WorkbookFactory.create(new File(filepath)); + + } catch (IOException | InvalidFormatException e) { + e.printStackTrace(); + throw new OperationException(ErrorCode.EXCEL_DATA_READ_ERR, "Unable to read file at: " + filepath); + } + + } + + private String downloadFile(final String filepath) throws OperationException { + throw new OperationException(ErrorCode.OPERATION_NOT_SUPPORTED); + } +} diff --git a/engine/src/main/java/com/blobcity/db/sql/lang/Aggregate.java b/engine/src/main/java/com/blobcity/db/sql/lang/Aggregate.java index 1ed9d37..1c82587 100644 --- a/engine/src/main/java/com/blobcity/db/sql/lang/Aggregate.java +++ b/engine/src/main/java/com/blobcity/db/sql/lang/Aggregate.java @@ -27,62 +27,62 @@ public class Aggregate { T total_count = (T) new Double(0.0); T avg = (T) new Double(0.0); - public void add(T number) { + public synchronized void add(T number) { aggregate = (T) new Double(aggregate.doubleValue() + number.doubleValue()); } - public void addCount(T number) { + public synchronized void addCount(T number) { total_count = (T) new Double(total_count.doubleValue() + number.doubleValue()); } - public T getCount() { + public synchronized T getCount() { System.out.println(total_count); return total_count; } - public T getAggregate() { + public synchronized T getAggregate() { return aggregate; } - public void setAverage() { + public synchronized void setAverage() { avg = (T) new Double(aggregate.doubleValue()/total_count.doubleValue()); } - public T getAverage() { + public synchronized T getAverage() { return avg; } - public void setError() { + public synchronized void setError() { this.error = true; } - public boolean isError() { + public synchronized boolean isError() { return error; } - public void setInitialMin(T number) { + public synchronized void setInitialMin(T number) { min = number; } - public void findMin(T number) { + public synchronized void findMin(T number) { if (number.doubleValue()max.doubleValue()) max = number; } - public T getMax() { + public synchronized T getMax() { return max; } } \ No newline at end of file diff --git a/engine/src/main/java/com/blobcity/db/sql/processing/OnDiskWhereHandling.java b/engine/src/main/java/com/blobcity/db/sql/processing/OnDiskWhereHandling.java index ade6c1b..59d05f0 100644 --- a/engine/src/main/java/com/blobcity/db/sql/processing/OnDiskWhereHandling.java +++ b/engine/src/main/java/com/blobcity/db/sql/processing/OnDiskWhereHandling.java @@ -98,6 +98,10 @@ public Set executeWhere(final String ds, final String collection, final refValue = ((NumericConstantNode) rightOperand).getValue(); } else if (rightOperand instanceof CharConstantNode) { refValue = ((CharConstantNode) rightOperand).getValue(); + } else if (rightOperand instanceof BooleanConstantNode) { + refValue = ((BooleanConstantNode) rightOperand).getBooleanValue(); + } else if (rightOperand instanceof BitConstantNode) { + refValue = ((BitConstantNode) rightOperand).getValue(); } else { // if unknown make it CharConstantNode if (tableManager.isInMemory(ds, collection)) { diff --git a/engine/src/main/java/com/blobcity/db/sql/statements/SelectExecutor.java b/engine/src/main/java/com/blobcity/db/sql/statements/SelectExecutor.java index 83bcad2..8c633c6 100644 --- a/engine/src/main/java/com/blobcity/db/sql/statements/SelectExecutor.java +++ b/engine/src/main/java/com/blobcity/db/sql/statements/SelectExecutor.java @@ -39,6 +39,7 @@ import com.blobcity.json.JSON; import com.foundationdb.sql.StandardException; import com.foundationdb.sql.parser.*; +import com.foundationdb.sql.types.TypeId; import com.foundationdb.sql.unparser.NodeToString; import java.util.*; @@ -653,6 +654,10 @@ private Set filter(final String appId, final String tableName, final Res refValue = ((NumericConstantNode) rightOperand).getValue(); } else if (rightOperand instanceof CharConstantNode) { refValue = ((CharConstantNode) rightOperand).getValue(); + } else if (rightOperand instanceof BooleanConstantNode) { + refValue = ((BooleanConstantNode) rightOperand).getBooleanValue(); + } else if (rightOperand instanceof BitConstantNode) { + refValue = ((BitConstantNode) rightOperand).getValue(); } else { // if unknown make it CharConstantNode if (tableManager.isInMemory(appId, tableName)) { diff --git a/launcher/pom.xml b/launcher/pom.xml index 52cc7c1..e140064 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,7 +21,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha com.blobcity.lib.database diff --git a/pom.xml b/pom.xml index 022b973..c2b8928 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha pom Kraken [POM] diff --git a/scripts/blobcity.sh b/scripts/blobcity.sh index 81fcfd2..761bfca 100755 --- a/scripts/blobcity.sh +++ b/scripts/blobcity.sh @@ -22,7 +22,7 @@ # The Java implementation to use. This is required. #export JAVA_HOME= -VERSION="1.7.8-alpha" +VERSION="1.7.9-alpha" JAVA="" if [ "$JAVA_HOME" != "" ]; then JAVA=$JAVA_HOME/bin/java diff --git a/tableau/pom.xml b/tableau/pom.xml index 0d64bec..814b7c3 100644 --- a/tableau/pom.xml +++ b/tableau/pom.xml @@ -22,7 +22,7 @@ along with this program. If not, see . database com.blobcity.pom - 1.7.8-alpha + 1.7.9-alpha 4.0.0 @@ -40,7 +40,7 @@ along with this program. If not, see . com.google.guava guava - 22.0 + 28.0-jre diff --git a/tcp-end-point/pom.xml b/tcp-end-point/pom.xml index 72981de..a7835b6 100644 --- a/tcp-end-point/pom.xml +++ b/tcp-end-point/pom.xml @@ -22,7 +22,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha com.blobcity.lib.database diff --git a/web-end-point/pom.xml b/web-end-point/pom.xml index 69711c0..1a8a8bb 100644 --- a/web-end-point/pom.xml +++ b/web-end-point/pom.xml @@ -22,7 +22,7 @@ along with this program. If not, see . com.blobcity.pom database - 1.7.8-alpha + 1.7.9-alpha com.blobcity.lib.database