[feature][dingo-executor] Support document index scan filter

dingo-calcite/src/main/java/io/dingodb/calcite/DingoParserContext.java

@@ -82,12 +82,22 @@ public final class DingoParserContext implements Context {
     @Getter
     private List<SQLWarning> warningList;
 
+    @Getter
+    private final Properties sessionVariables;
+
+    @Getter
+    @Setter
+    private boolean autoCommit;
+
     public DingoParserContext(@NonNull String defaultSchemaName) {
-        this(defaultSchemaName, null);
+        this(defaultSchemaName, null, null);
     }
 
-    public DingoParserContext(@NonNull String defaultSchemaName, @Nullable Properties options) {
+    public DingoParserContext(@NonNull String defaultSchemaName, @Nullable Properties options,
+                              @Nullable Properties sessionVariables) {
         this.defaultSchemaName = defaultSchemaName;
+        this.sessionVariables = sessionVariables;
+        this.autoCommit = false;
 
         String timeZoneId = (options != null ? options.getProperty("timeZone") : null);
         timeZone = (timeZoneId != null ? TimeZone.getTimeZone(timeZoneId) : TimeZone.getDefault());

dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java

@@ -272,7 +272,8 @@ protected void convertCollectionTable(Blackboard bb, SqlCall call) {
                     namespace.getIndex(),
                     null,
                     null,
-                    new ArrayList<>()
+                    new ArrayList<>(),
+                    false
                 );
             }
         } else if (validator.getNamespace(call) instanceof TableHybridFunctionNamespace) {

dingo-calcite/src/main/java/io/dingodb/calcite/rel/DingoDocument.java

@@ -52,9 +52,10 @@ public DingoDocument(
         @NonNull Table indexTable,
         TupleMapping selection,
         RexNode filter,
-        List<RelHint> hints
+        List<RelHint> hints,
+        boolean isDocumentScanFilter
     ) {
-        super(cluster, traitSet, call, table, operands, indexTableId, indexTable, selection, filter, hints);
+        super(cluster, traitSet, call, table, operands, indexTableId, indexTable, selection, filter, hints, isDocumentScanFilter);
     }
 
     @Override

dingo-calcite/src/main/java/io/dingodb/calcite/rel/LogicalDingoDocument.java

@@ -41,6 +41,7 @@
 import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlCharStringLiteral;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -49,6 +50,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -79,6 +81,13 @@ public class LogicalDingoDocument extends TableFunctionScan {
     @Getter
     public List<RelHint> hints;
 
+    @Getter
+    @Setter
+    protected String queryStr;
+    @Getter
+    @Setter
+    protected boolean isDocumentScanFilter;
+
     public LogicalDingoDocument(RelOptCluster cluster,
                                  RelTraitSet traitSet,
                                  RexCall call,
@@ -88,7 +97,8 @@ public LogicalDingoDocument(RelOptCluster cluster,
                                  @NonNull Table indexTable,
                                  TupleMapping selection,
                                  RexNode filter,
-                                 List<RelHint> hints
+                                 List<RelHint> hints,
+                                 boolean isDocumentScanFilter
                               ) {
         super(cluster, traitSet, Collections.emptyList(), call, null, call.type, null);
         this.call = call;
@@ -100,6 +110,8 @@ public LogicalDingoDocument(RelOptCluster cluster,
         this.rowType = null;
         this.realSelection = selection;
         this.hints = hints;
+        this.queryStr = Objects.requireNonNull((SqlCharStringLiteral) operands.get(2)).getStringValue();
+        this.isDocumentScanFilter = isDocumentScanFilter;
         DingoTable dingoTable = table.unwrap(DingoTable.class);
         if (selection != null) {
             if (forDml) {
@@ -144,7 +156,7 @@ public TableFunctionScan copy(RelTraitSet traitSet,
         return new LogicalDingoDocument(
             getCluster(),
             traitSet,
-            call, table, operands, indexTableId, indexTable, selection, filter, hints);
+            call, table, operands, indexTableId, indexTable, selection, filter, hints, isDocumentScanFilter);
     }
 
     @Override

dingo-calcite/src/main/java/io/dingodb/calcite/rel/dingo/DingoDocumentScanFilter.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2021 DataCanvas
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.dingodb.calcite.rel.dingo;
+
+import io.dingodb.calcite.rel.DingoRel;
+import io.dingodb.calcite.rel.logical.LogicalDocumentScanFilter;
+import io.dingodb.calcite.visitor.DingoRelVisitor;
+import io.dingodb.common.CommonId;
+import io.dingodb.common.type.TupleMapping;
+import io.dingodb.meta.entity.Table;
+import lombok.Getter;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.hint.RelHint;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexNode;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.List;
+
+public class DingoDocumentScanFilter extends LogicalDocumentScanFilter implements DingoRel {
+    @Getter
+    private double rowCount;
+
+    public DingoDocumentScanFilter(RelOptCluster cluster,
+                                   RelTraitSet traitSet,
+                                   List<RelHint> hints,
+                                   RelOptTable table,
+                                   @Nullable RexNode filter,
+                                   @Nullable TupleMapping selection,
+                                   Table indexTable,
+                                   CommonId indexId,
+                                   boolean pushDown,
+                                   boolean lookup,
+                                   int keepSerialOrder,
+                                   String queryString) {
+        super(cluster, traitSet, hints, table, filter, selection, indexTable,
+            indexId, pushDown, lookup, keepSerialOrder, queryString);
+    }
+
+    @Override
+    public <T> T accept(@NonNull DingoRelVisitor<T> visitor) {
+        return visitor.visit(this);
+    }
+
+    @Override
+    public double estimateRowCount(RelMetadataQuery mq) {
+        rowCount = super.estimateRowCount(mq);
+        return rowCount;
+    }
+}

dingo-calcite/src/main/java/io/dingodb/calcite/rel/logical/LogicalDocumentScanFilter.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2021 DataCanvas
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.dingodb.calcite.rel.logical;
+
+import io.dingodb.calcite.DingoTable;
+import io.dingodb.calcite.rel.LogicalDingoTableScan;
+import io.dingodb.common.CommonId;
+import io.dingodb.common.type.TupleMapping;
+import io.dingodb.meta.entity.Table;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.hint.RelHint;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexNode;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.List;
+
+import static io.dingodb.calcite.meta.DingoCostModelV1.getAvgRowSize;
+import static io.dingodb.calcite.meta.DingoCostModelV1.getScanAvgRowSize;
+import static io.dingodb.calcite.meta.DingoCostModelV1.getScanCost;
+import static io.dingodb.calcite.meta.DingoCostModelV1.netFactor;
+import static io.dingodb.calcite.meta.DingoCostModelV1.scanConcurrency;
+import static io.dingodb.calcite.meta.DingoCostModelV1.scanFactor;
+
+public class LogicalDocumentScanFilter extends LogicalDingoTableScan {
+    @Getter
+    private CommonId indexId;
+    @Getter
+    private Table indexTable;
+    @Getter
+    private boolean lookup;
+    @Getter
+    @Setter
+    private int keepSerialOrder;
+    @Getter
+    private String queryString;
+
+    public LogicalDocumentScanFilter(RelOptCluster cluster,
+                                     RelTraitSet traitSet,
+                                     List<RelHint> hints,
+                                     RelOptTable table,
+                                     @Nullable RexNode filter,
+                                     @Nullable TupleMapping selection,
+                                     Table indexTable,
+                                     CommonId indexId,
+                                     boolean pushDown,
+                                     boolean lookup,
+                                     int keepSerialOrder,
+                                     String queryString) {
+        super(cluster, traitSet, hints, table, filter, selection, null, null, null, pushDown, false);
+        this.indexTable = indexTable;
+        this.indexId = indexId;
+        this.lookup = lookup;
+        this.keepSerialOrder = keepSerialOrder;
+        this.queryString = queryString;
+    }
+
+    @Override
+    public double estimateRowCount(RelMetadataQuery mq) {
+        return super.estimateRowCount(mq);
+    }
+
+    @Override
+    public @Nullable RelOptCost computeSelfCost(@NonNull RelOptPlanner planner, @NonNull RelMetadataQuery mq) {
+        DingoTable dingoTable = table.unwrap(DingoTable.class);
+
+        assert dingoTable != null;
+        String schemaName = "";
+        if (dingoTable.getNames().size() > 2) {
+            schemaName = dingoTable.getNames().get(1);
+        }
+        double indexRowSize = getAvgRowSize(indexTable.getColumns(),
+            dingoTable.getTable(), schemaName);
+
+        double rowCount = estimateRowCount(mq);
+
+        double indexScanCost = rowCount * (Math.log(indexRowSize) / Math.log(2)) * scanFactor;
+        double indexNetCost = rowCount * indexRowSize * netFactor;
+        double cost = (indexNetCost + indexScanCost) / scanConcurrency;
+
+        if (lookup) {
+            double rowSize = getScanAvgRowSize(this);
+            double estimateRowCount = estimateRowCount(mq);
+            double tableScanCost = getScanCost(estimateRowCount, rowSize);
+            double tableNetCost = estimateRowCount * rowSize * netFactor;
+            cost += (tableScanCost + tableNetCost) / scanConcurrency;
+        }
+
+        return planner.getCostFactory().makeCost(cost * 0.8, 0, 0);
+    }
+
+    @Override
+    public @NonNull RelWriter explainTerms(@NonNull RelWriter pw) {
+        // crucial, this is how Calcite distinguish between different node with different props.
+        pw.itemIf("filter", filter, filter != null);
+        pw.itemIf("selection", selection, selection != null);
+        pw.itemIf("groupSet", groupSet, groupSet != null);
+        pw.itemIf("aggCalls", aggCalls, aggCalls != null);
+        pw.itemIf("pushDown", pushDown, pushDown);
+        pw.itemIf("lookup", lookup, true);
+        pw.itemIf("queryString", queryString, !queryString.equals(""));
+        return pw;
+    }
+}

dingo-calcite/src/main/java/io/dingodb/calcite/rule/DingoDocumentFilterRule.java

@@ -46,7 +46,8 @@ public void onMatch(RelOptRuleCall call) {
                 document.getIndexTable(),
                 document.getSelection(),
                 filter.getCondition(),
-                filter.getHints()
+                filter.getHints(),
+                document.isDocumentScanFilter()
             )
         );
     }