Merge branch 'comparison-datecast' into nestedLimit-comparison-datecast

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java

@@ -17,6 +17,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters;
 import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import com.google.common.base.Objects;
 import java.time.Instant;
@@ -26,37 +27,21 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
-import java.time.temporal.ChronoField;
 import java.time.temporal.ChronoUnit;
 import lombok.RequiredArgsConstructor;
 
 @RequiredArgsConstructor
 public class ExprDatetimeValue extends AbstractExprValue {
   private final LocalDateTime datetime;
 
-  private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
-  private static final int MIN_FRACTION_SECONDS = 0;
-  private static final int MAX_FRACTION_SECONDS = 6;
-
-  static {
-    FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
-        .appendPattern("yyyy-MM-dd HH:mm:ss")
-        .appendFraction(
-            ChronoField.MICRO_OF_SECOND,
-            MIN_FRACTION_SECONDS,
-            MAX_FRACTION_SECONDS,
-            true)
-        .toFormatter();
-  }
-
   /**
    * Constructor with datetime string as input.
    */
   public ExprDatetimeValue(String datetime) {
     try {
-      this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS);
+      this.datetime = LocalDateTime.parse(datetime,
+          ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER);
     } catch (DateTimeParseException e) {
       throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please "
           + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime));

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java

@@ -18,6 +18,7 @@
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
@@ -80,6 +81,11 @@ public LocalTime timeValue() {
     }
   }
 
+  @Override
+  public Instant timestampValue() {
+    return new ExprTimestampValue(value).timestampValue();
+  }
+
   @Override
   public String toString() {
     return String.format("\"%s\"", value);

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java

@@ -19,6 +19,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters;
 import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -69,7 +70,8 @@ public class ExprTimestampValue extends AbstractExprValue {
    */
   public ExprTimestampValue(String timestamp) {
     try {
-      this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS)
+      this.timestamp = LocalDateTime.parse(timestamp,
+          ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER)
           .atZone(ZONE)
           .toInstant();
     } catch (DateTimeParseException e) {

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java

@@ -131,4 +131,8 @@ public static List<ExprCoreType> coreTypes() {
   public static List<ExprType> numberTypes() {
     return ImmutableList.of(INTEGER, LONG, FLOAT, DOUBLE);
   }
+
+  public static List<ExprType> dateTypes() {
+    return ImmutableList.of(DATE, DATETIME, TIMESTAMP, TIME);
+  }
 }

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java → core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java

@@ -15,7 +15,7 @@
  *
  */
 
-package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value;
+package com.amazon.opendistroforelasticsearch.sql.data.utils;
 
 import static java.time.temporal.ChronoField.DAY_OF_MONTH;
 import static java.time.temporal.ChronoField.HOUR_OF_DAY;
@@ -37,7 +37,7 @@
  * Reference org.elasticsearch.common.time.DateFormatters.
  */
 @UtilityClass
-public class ElasticsearchDateFormatters {
+public class ExprDateFormatters {
 
   public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON =
       new DateTimeFormatterBuilder()
@@ -73,7 +73,16 @@ public class ElasticsearchDateFormatters {
       new DateTimeFormatterBuilder()
           .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
           .optionalStart()
+
+          // otherwise use space
+          .optionalStart()
+          .appendLiteral(' ')
+          .optionalEnd()
+          // optional T
+          .optionalStart()
           .appendLiteral('T')
+          .optionalEnd()
+
           .optionalStart()
           .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
           .optionalStart()
@@ -82,13 +91,18 @@ public class ElasticsearchDateFormatters {
           .optionalStart()
           .appendLiteral(':')
           .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
+
+          // optional millis with dot
           .optionalStart()
           .appendFraction(NANO_OF_SECOND, 1, 9, true)
           .optionalEnd()
+
+          // otherwise optional millis use with comma
           .optionalStart()
           .appendLiteral(',')
           .appendFraction(NANO_OF_SECOND, 1, 9, false)
           .optionalEnd()
+
           .optionalEnd()
           .optionalEnd()
           .optionalStart()
@@ -104,4 +118,11 @@ public class ElasticsearchDateFormatters {
 
   public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter
           .ofPattern("yyyy-MM-dd HH:mm:ss");
+
+  public static final DateTimeFormatter TOLERANT_PARSER_DATE_TIME_FORMATTER =
+      new DateTimeFormatterBuilder()
+          .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER)
+          .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT)
+          .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
+          .toFormatter();
 }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java

@@ -25,16 +25,26 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.SerializableFunction;
 import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils;
 import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Table;
+import java.io.Serializable;
+import java.util.List;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import lombok.experimental.UtilityClass;
+import org.apache.commons.lang3.tuple.Pair;
+
 
 /**
  * The definition of binary predicate function
@@ -192,53 +202,44 @@ private static FunctionResolver notEqual() {
   }
 
   private static FunctionResolver less() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.LESS.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(FunctionDSL
-                        .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) < 0)),
-                    BOOLEAN,
-                    type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.LESS.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value < 0));
   }
 
   private static FunctionResolver lte() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.LTE.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(
-                    FunctionDSL
-                        .nullMissingHandling(
-                            (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) <= 0)),
-                    BOOLEAN,
-                    type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.LTE.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value <= 0));
   }
 
   private static FunctionResolver greater() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.GREATER.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(FunctionDSL
-                        .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) > 0)),
-                    BOOLEAN, type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.GREATER.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value > 0));
   }
 
   private static FunctionResolver gte() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.GTE.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(
-                    FunctionDSL.nullMissingHandling(
-                        (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) >= 0)),
-                    BOOLEAN,
-                    type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.GTE.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value >= 0));
+  }
+
+  private static List<SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>>
+      getComparisonFunctions(Function<Integer, Boolean> f) {
+    List<SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>>
+        comparisonFunctions = ExprCoreType.coreTypes()
+        .stream()
+        .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling(
+            (v1, v2) -> ExprBooleanValue.of(f.apply(v1.compareTo(v2)))), BOOLEAN, type, type))
+        .collect(Collectors.toList());
+
+    // all string to date conversions
+    comparisonFunctions.addAll(ExprCoreType.dateTypes()
+        .stream()
+        .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling((v1, v2) -> {
+          // casting V2 from string to whatever type v1 is
+          ExprValue v2Casted = ExprValueUtils.fromObjectValue(v2.value(), (ExprCoreType) v1.type());
+          return ExprBooleanValue.of(f.apply(v1.compareTo(v2Casted)));
+        }), BOOLEAN, type, STRING))
+        .collect(Collectors.toList()));
+    return comparisonFunctions;
   }
 
   private static FunctionResolver like() {

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java

@@ -113,20 +113,9 @@ public void timeInUnsupportedFormat() {
   public void timestampInUnsupportedFormat() {
     SemanticCheckException exception =
         assertThrows(SemanticCheckException.class,
-            () -> new ExprTimestampValue("2020-07-07T01:01:01Z"));
+            () -> new ExprTimestampValue("2020-07-07T1:01:01Z"));
     assertEquals(
-        "timestamp:2020-07-07T01:01:01Z in unsupported format, "
-            + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
-        exception.getMessage());
-  }
-
-  @Test
-  public void datetimeInUnsupportedFormat() {
-    SemanticCheckException exception =
-        assertThrows(SemanticCheckException.class,
-            () -> new ExprDatetimeValue("2020-07-07T01:01:01Z"));
-    assertEquals(
-        "datetime:2020-07-07T01:01:01Z in unsupported format, "
+        "timestamp:2020-07-07T1:01:01Z in unsupported format, "
             + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
         exception.getMessage());
   }
@@ -142,9 +131,9 @@ public void stringDateTimeValue() {
 
     SemanticCheckException exception =
         assertThrows(SemanticCheckException.class,
-            () -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue());
+            () -> new ExprStringValue("2020-07-07T01:01:01Z12345678").datetimeValue());
     assertEquals(
-        "datetime:2020-07-07T01:01:01Z in unsupported format, "
+        "datetime:2020-07-07T01:01:01Z12345678 in unsupported format, "
             + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
         exception.getMessage());
   }
@@ -239,9 +228,9 @@ public void datetimeWithVariableMicroPrecision() {
   public void timestampOverMaxMicroPrecision() {
     SemanticCheckException exception =
         assertThrows(SemanticCheckException.class,
-            () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567"));
+            () -> new ExprTimestampValue("2020-07-07 01:01:01.12345678910"));
     assertEquals(
-        "timestamp:2020-07-07 01:01:01.1234567 in unsupported format, "
+        "timestamp:2020-07-07 01:01:01.12345678910 in unsupported format, "
                 + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
         exception.getMessage());
   }
@@ -250,9 +239,9 @@ public void timestampOverMaxMicroPrecision() {
   public void datetimeOverMaxMicroPrecision() {
     SemanticCheckException exception =
         assertThrows(SemanticCheckException.class,
-            () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567"));
+            () -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567890"));
     assertEquals(
-        "datetime:2020-07-07 01:01:01.1234567 in unsupported format, "
+        "datetime:2020-07-07 01:01:01.1234567890 in unsupported format, "
                 + "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
         exception.getMessage());
   }
@@ -261,9 +250,9 @@ public void datetimeOverMaxMicroPrecision() {
   public void timeOverMaxMicroPrecision() {
     SemanticCheckException exception =
         assertThrows(SemanticCheckException.class,
-            () -> new ExprTimeValue("01:01:01.1234567"));
+            () -> new ExprTimeValue("01:01:01.1234567891"));
     assertEquals(
-        "time:01:01:01.1234567 in unsupported format, please use HH:mm:ss[.SSSSSS]",
+        "time:01:01:01.1234567891 in unsupported format, please use HH:mm:ss[.SSSSSS]",
         exception.getMessage());
   }
 }

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java

@@ -40,12 +40,16 @@
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDateValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDatetimeValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimeValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils;
@@ -148,6 +152,34 @@ private static Stream<Arguments> testCompareValueArguments() {
     builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(1)));
     builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(2)));
     builder.add(Arguments.of(new ExprShortValue(2), new ExprShortValue(1)));
+
+    builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:01"),
+        new ExprStringValue("2006-01-01 00:00:00")));
+    builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:00"),
+        new ExprStringValue("2006-01-01 00:00:01")));
+    builder.add(Arguments.of(new ExprDatetimeValue("2006-01-01 00:00:00"),
+        new ExprStringValue("2006-01-01 00:00:00")));
+
+    builder.add(Arguments.of(new ExprDateValue("2006-01-01"),
+        new ExprStringValue("2006-01-01")));
+    builder.add(Arguments.of(new ExprDateValue("2006-01-01"),
+        new ExprStringValue("2006-01-02")));
+    builder.add(Arguments.of(new ExprDateValue("2006-01-02"),
+        new ExprStringValue("2006-01-01")));
+
+    builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:01"),
+        new ExprStringValue("2006-01-01 00:00:00")));
+    builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:00"),
+        new ExprStringValue("2006-01-01 00:00:01")));
+    builder.add(Arguments.of(new ExprTimestampValue("2006-01-01 00:00:00"),
+        new ExprStringValue("2006-01-01 00:00:00")));
+
+    builder.add(Arguments.of(new ExprTimeValue("00:00:01"),
+        new ExprStringValue("00:00:00")));
+    builder.add(Arguments.of(new ExprTimeValue("00:00:00"),
+        new ExprStringValue("00:00:01")));
+    builder.add(Arguments.of(new ExprTimeValue("00:00:00"),
+        new ExprStringValue("00:00:00")));
     return builder.build();
   }