Merge pull request #50 from blobcity/develop

1. Resolved issue with boolean values in SQL WHERE clause. 2. Improvement in thread safety for Aggregate operations on tables
blobcity · Aug 3, 2019 · 46ff8ac · 46ff8ac
2 parents 9e94d46 + be86750
commit 46ff8ac
Show file tree

Hide file tree

Showing 20 changed files with 128 additions and 28 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -17,7 +17,7 @@
 
 FROM blobcity/java8-ubuntu
 
-ENV DB_VER=1.7.8-alpha
+ENV DB_VER=1.7.9-alpha
 
 RUN cd / && mkdir data
 

diff --git a/bean-manager/pom.xml b/bean-manager/pom.xml
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
 
     <groupId>com.blobcity.lib</groupId>
@@ -63,7 +63,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
             <!--<version>r05</version>-->
-            <version>22.0</version>
+            <version>28.0-jre</version>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/com.google.guava/guava -->

diff --git a/console-end-point/pom.xml b/console-end-point/pom.xml
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
 
     <groupId>com.blobcity.lib</groupId>

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -21,7 +21,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
     <artifactId>blobcity-db</artifactId>
     <name>BlobCityDB Distribution</name>

diff --git a/engine/pom.xml b/engine/pom.xml
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
 
     <groupId>com.blobcity.lib.database</groupId>
@@ -540,6 +540,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-core</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -585,6 +589,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                     <groupId>commons-codec</groupId>
                     <artifactId>commons-codec</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -617,6 +625,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
                     <groupId>com.google.collections</groupId>
                     <artifactId>google-collections</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.code.findbugs</groupId>
+                    <artifactId>jsr305</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

diff --git a/engine/src/main/java/com/blobcity/db/bquery/BQueryExecutorBean.java b/engine/src/main/java/com/blobcity/db/bquery/BQueryExecutorBean.java
@@ -1401,6 +1401,7 @@ private JSONObject search(final String datastore, final JSONObject payloadJson)
                 case GTEQ:
                 case LT:
                 case LTEQ:
+                    System.out.println("The comparator: " + jsonObject.toString());
                     iterator = dataManager.selectWithPattern(datastore, selectTables.toArray()[0].toString(), jsonObject.getString("c"), jsonObject.get("v"), operator);
                     break;
                 case IN:

diff --git a/engine/src/main/java/com/blobcity/db/bquery/ConsoleExecutorBean.java b/engine/src/main/java/com/blobcity/db/bquery/ConsoleExecutorBean.java
@@ -455,6 +455,9 @@ public String runCommand(final String user, String query) {
                 case "import-csv":
                     response = importCSV(elements);
                     break;
+                case "import-excel":
+                    response = importExcel(elements);
+                    break;
                 case "insert-custom":
                     response = insertCustom(elements);
                     break;
@@ -1466,6 +1469,40 @@ private String importCSV(String[] elements) throws OperationException {
 
         operationsManager.registerOperation(datastore, collection, OperationTypes.IMPORT, opSpecs);
 
+//        mapReduceOutputImporter.importCSVFile(datastore, collection, filePath);
+
+        long elapsedTime = System.currentTimeMillis() - startTime;
+        return "Done in " + elapsedTime + " (ms)";
+    }
+
+    /**
+     * Used to import an Excel document in a specified table. Columns of the table correspond to columns of the Excel
+     * and the entry will have one record per row. Sheet name is also stored as a field per record.
+     * @param elements query elements from CLI request
+     * @return import success/fail status response in text form
+     * @throws OperationException if an error occurs while executing the oepration
+     */
+    private String importExcel(String[] elements) throws OperationException {
+        long startTime = System.currentTimeMillis();
+        final String databaseAndTable = elements[1];
+        if (!databaseAndTable.contains(".")) {
+            throw new OperationException(ErrorCode.INVALID_QUERY_FORMAT, "Database and table should be specified in format: databaseName.tableName");
+        }
+        final String datastore = databaseAndTable.substring(0, databaseAndTable.indexOf(".", 1)); //start searching dot from index 1 to handle case of .systemdb as datastore name
+        final String collection = databaseAndTable.substring(databaseAndTable.indexOf(".", 1) + 1, databaseAndTable.length());
+
+        verifyDCInfo(datastore, collection);
+
+        /* Maybe a public URL or a local file system path. NFS paths are currently not supported */
+        String filePath = elements[2];
+
+        JSONObject opSpecs = new JSONObject();
+        opSpecs.put("type", "IMP");
+        opSpecs.put("import-type", "CSV");
+        opSpecs.put("file", filePath);
+
+        operationsManager.registerOperation(datastore, collection, OperationTypes.IMPORT, opSpecs);
+
 //        mapReduceOutputImporter.importCSVFile(datastore, collection, filePath);
 
         long elapsedTime = System.currentTimeMillis() - startTime;

diff --git a/engine/src/main/java/com/blobcity/db/bsql/BSqlDataManager.java b/engine/src/main/java/com/blobcity/db/bsql/BSqlDataManager.java
@@ -523,6 +523,9 @@ public Iterator<String> selectKeysWithPattern(final String datastore, final Stri
                     case DOUBLE:
                         comparableReferenceValue = new Double(referenceValue.toString());
                         break;
+                    case BOOLEAN:
+                        comparableReferenceValue = (boolean) referenceValue;
+                        break;
                     case STRING:
                     case VARCHAR:
                     case CHARACTER_LARGE_OBJECT:

diff --git a/engine/src/main/java/com/blobcity/db/bsql/filefilters/EQFilenameFilter.java b/engine/src/main/java/com/blobcity/db/bsql/filefilters/EQFilenameFilter.java
@@ -66,6 +66,10 @@ public TypeConverter getTypeConverter() {
 
     @Override
     public Object getTypeConvertedReferenceValue() throws OperationException {
+        if(referenceValue instanceof Boolean) {
+            if(((Boolean) referenceValue).booleanValue()) return "true";
+            else return "false";
+        }
         return typeConverter.getValue(referenceValue.toString());
     }
 }
diff --git a/engine/src/main/java/com/blobcity/db/exceptions/ErrorCode.java b/engine/src/main/java/com/blobcity/db/exceptions/ErrorCode.java
@@ -128,7 +128,8 @@ public enum ErrorCode {
     INVALID_WEBSERVICE_ENDPOINT("INVALID_WEBSERVICE_ENDPOINT", "No customer web-service registered at the specified endpoint"),
     CODE_LOAD_ERROR("CODE_LOAD_ERROR", "Unable to load code"),
     TABLEAU_EXCEPTION("TABLEAU_EXCEPTION", "Internal error with Tableau integration"),
-    GROUP_BY("GROUP_BY", "Error executing GROUP BY clause");
+    GROUP_BY("GROUP_BY", "Error executing GROUP BY clause"),
+    EXCEL_DATA_READ_ERR("EXCEL_DATA_READ_ERR", "Error reading data in Excel format");
 
     private final String errorCode;
     private final String errorMessage;

diff --git a/engine/src/main/java/com/blobcity/db/importer/ExcelImporter.java b/engine/src/main/java/com/blobcity/db/importer/ExcelImporter.java
@@ -0,0 +1,33 @@
+package com.blobcity.db.importer;
+
+import com.blobcity.db.exceptions.ErrorCode;
+import com.blobcity.db.exceptions.OperationException;
+import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.springframework.stereotype.Component;
+import org.tensorflow.Operation;
+
+import java.io.File;
+import java.io.IOException;
+
+@Component
+public class ExcelImporter {
+
+    public void importExcel(final String ds, final String collection, final String pathOrUrl) throws OperationException {
+        final String filepath = pathOrUrl.startsWith("http:") ? downloadFile(pathOrUrl) : pathOrUrl;
+
+        try {
+            Workbook workbook = WorkbookFactory.create(new File(filepath));
+
+        } catch (IOException | InvalidFormatException e) {
+            e.printStackTrace();
+            throw new OperationException(ErrorCode.EXCEL_DATA_READ_ERR, "Unable to read file at: " + filepath);
+        }
+
+    }
+
+    private String downloadFile(final String filepath) throws OperationException {
+        throw new OperationException(ErrorCode.OPERATION_NOT_SUPPORTED);
+    }
+}
diff --git a/engine/src/main/java/com/blobcity/db/sql/lang/Aggregate.java b/engine/src/main/java/com/blobcity/db/sql/lang/Aggregate.java
@@ -27,62 +27,62 @@ public class Aggregate<T extends Number> {
     T total_count = (T) new Double(0.0);
     T avg = (T) new Double(0.0);
 
-    public void add(T number) {
+    public synchronized void add(T number) {
         aggregate = (T) new Double(aggregate.doubleValue() + number.doubleValue());
     }
 
-    public void addCount(T number) {
+    public synchronized void addCount(T number) {
         total_count = (T) new Double(total_count.doubleValue() + number.doubleValue());
     }
 
-    public T getCount() {
+    public synchronized T getCount() {
         System.out.println(total_count);
         return total_count;
     }
 
-    public T getAggregate() {
+    public synchronized T getAggregate() {
         return aggregate;
     }
 
-    public void setAverage() {
+    public synchronized void setAverage() {
         avg = (T) new Double(aggregate.doubleValue()/total_count.doubleValue());
     }
 
-    public T getAverage() {
+    public synchronized T getAverage() {
         return avg;
     }
 
-    public void setError() {
+    public synchronized void setError() {
         this.error = true;
     }
 
-    public boolean isError() {
+    public synchronized boolean isError() {
         return error;
     }
 
-    public void setInitialMin(T number) {
+    public synchronized void setInitialMin(T number) {
         min = number;
     }
 
-    public void findMin(T number) {
+    public synchronized void findMin(T number) {
         if (number.doubleValue()<min.doubleValue())
             min = number;
     }
 
-    public T getMin() {
+    public synchronized T getMin() {
         return min;
     }
 
-    public void setInitialMax(T number) {
+    public synchronized void setInitialMax(T number) {
         max = number;
     }
 
-    public void findMax(T number) {
+    public synchronized void findMax(T number) {
         if (number.doubleValue()>max.doubleValue())
             max = number;
     }
 
-    public T getMax() {
+    public synchronized T getMax() {
         return max;
     }
 }
diff --git a/engine/src/main/java/com/blobcity/db/sql/processing/OnDiskWhereHandling.java b/engine/src/main/java/com/blobcity/db/sql/processing/OnDiskWhereHandling.java
@@ -98,6 +98,10 @@ public Set<String> executeWhere(final String ds, final String collection, final
                     refValue = ((NumericConstantNode) rightOperand).getValue();
                 } else if (rightOperand instanceof CharConstantNode) {
                     refValue = ((CharConstantNode) rightOperand).getValue();
+                } else if (rightOperand instanceof BooleanConstantNode) {
+                    refValue = ((BooleanConstantNode) rightOperand).getBooleanValue();
+                } else if (rightOperand instanceof BitConstantNode) {
+                    refValue = ((BitConstantNode) rightOperand).getValue();
                 } else {
                     // if unknown make it CharConstantNode
                     if (tableManager.isInMemory(ds, collection)) {

diff --git a/engine/src/main/java/com/blobcity/db/sql/statements/SelectExecutor.java b/engine/src/main/java/com/blobcity/db/sql/statements/SelectExecutor.java
@@ -39,6 +39,7 @@
 import com.blobcity.json.JSON;
 import com.foundationdb.sql.StandardException;
 import com.foundationdb.sql.parser.*;
+import com.foundationdb.sql.types.TypeId;
 import com.foundationdb.sql.unparser.NodeToString;
 
 import java.util.*;
@@ -653,6 +654,10 @@ private Set<String> filter(final String appId, final String tableName, final Res
                     refValue = ((NumericConstantNode) rightOperand).getValue();
                 } else if (rightOperand instanceof CharConstantNode) {
                     refValue = ((CharConstantNode) rightOperand).getValue();
+                } else if (rightOperand instanceof BooleanConstantNode) {
+                    refValue = ((BooleanConstantNode) rightOperand).getBooleanValue();
+                } else if (rightOperand instanceof BitConstantNode) {
+                    refValue = ((BitConstantNode) rightOperand).getValue();
                 } else {
                     // if unknown make it CharConstantNode
                     if (tableManager.isInMemory(appId, tableName)) {

diff --git a/launcher/pom.xml b/launcher/pom.xml
@@ -21,7 +21,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
 
     <groupId>com.blobcity.lib.database</groupId>

diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>com.blobcity.pom</groupId>
     <artifactId>database</artifactId>
-    <version>1.7.8-alpha</version>
+    <version>1.7.9-alpha</version>
     <packaging>pom</packaging>
 
     <name>Kraken [POM]</name>

diff --git a/scripts/blobcity.sh b/scripts/blobcity.sh
@@ -22,7 +22,7 @@
 # The Java implementation to use. This is required.
 #export JAVA_HOME=
 
-VERSION="1.7.8-alpha"
+VERSION="1.7.9-alpha"
 JAVA=""
 if [ "$JAVA_HOME" != "" ]; then
   JAVA=$JAVA_HOME/bin/java

diff --git a/tableau/pom.xml b/tableau/pom.xml
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <artifactId>database</artifactId>
         <groupId>com.blobcity.pom</groupId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -40,7 +40,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
-            <version>22.0</version>
+            <version>28.0-jre</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/javax.ws.rs/javax.ws.rs-api -->
         <dependency>

diff --git a/tcp-end-point/pom.xml b/tcp-end-point/pom.xml
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
 
     <groupId>com.blobcity.lib.database</groupId>

diff --git a/web-end-point/pom.xml b/web-end-point/pom.xml
@@ -22,7 +22,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <parent>
         <groupId>com.blobcity.pom</groupId>
         <artifactId>database</artifactId>
-        <version>1.7.8-alpha</version>
+        <version>1.7.9-alpha</version>
     </parent>
 
     <groupId>com.blobcity.lib.database</groupId>