Skip to content

Commit

Permalink
[BugFix] Operator's statistics not be estimated during JoinReorder of…
Browse files Browse the repository at this point in the history 
… table-pruning phase (#51304)

Signed-off-by: satanson <[email protected]>
(cherry picked from commit 61ebfc7)

# Conflicts:
#	fe/fe-core/src/test/java/com/starrocks/sql/plan/ReplayFromDumpTest.java
  • Loading branch information
@satanson @mergify
satanson authored and mergify[bot] committed Sep 24, 2024
1 parent bcc1c1a commit b68f597
Show file tree
Hide file tree
Showing 3 changed files with 4,375 additions and 0 deletions.
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/join/ReorderJoinRule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

package com.starrocks.sql.optimizer.rule.join;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.starrocks.common.FeConstants;
Expand Down Expand Up @@ -267,6 +268,14 @@ private OptExpression deriveNewOptExpression(OptExpression optExpression,
LogicalProperty newProperty = new LogicalProperty(optExpression.getLogicalProperty());
newProperty.setOutputColumns(newCols);

if (!Optional.ofNullable(optExpression.getStatistics()).isPresent()) {
ExpressionContext expressionContext = new ExpressionContext(optExpression);
StatisticsCalculator statisticsCalculator = new StatisticsCalculator(
expressionContext, optimizerContext.getColumnRefFactory(), optimizerContext);
statisticsCalculator.estimatorStats();
optExpression.setStatistics(expressionContext.getStatistics());
}
Preconditions.checkState(optExpression.getStatistics() != null);
Statistics newStats = Statistics.buildFrom(optExpression.getStatistics()).build();
Iterator<Map.Entry<ColumnRefOperator, ColumnStatistic>>
iterator = newStats.getColumnStatistics().entrySet().iterator();
Expand Down
160 changes: 160 additions & 0 deletions fe/fe-core/src/test/java/com/starrocks/sql/plan/ReplayFromDumpTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.junit.Ignore;
import org.junit.Test;

<<<<<<< HEAD
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
Expand Down Expand Up @@ -160,6 +161,19 @@ protected Pair<QueryDumpInfo, String> getPlanFragment(String dumpJsonStr, Sessio
return new Pair<>(queryDumpInfo,
UtFrameUtils.getNewPlanAndFragmentFromDump(connectContext, queryDumpInfo).second.
getExplainString(level));
=======
import java.util.stream.Stream;

public class ReplayFromDumpTest extends ReplayFromDumpTestBase {
@Test
public void testForceRuleBasedRewrite() throws Exception {
QueryDumpInfo queryDumpInfo = getDumpInfoFromJson(getDumpInfoFromFile("query_dump/force_rule_based_mv_rewrite"));
SessionVariable sessionVariable = queryDumpInfo.getSessionVariable();
sessionVariable.setEnableForceRuleBasedMvRewrite(true);
Pair<QueryDumpInfo, String> replayPair =
getCostPlanFragment(getDumpInfoFromFile("query_dump/force_rule_based_mv_rewrite"), sessionVariable);
Assert.assertTrue(replayPair.second, replayPair.second.contains("partition_flat_consumptions_partition_drinks_dates"));
>>>>>>> 61ebfc79f8 ([BugFix] Operator's statistics not be estimated during JoinReorder of table-pruning phase (#51304))
}

@Test
Expand Down Expand Up @@ -872,4 +886,150 @@ public void testNormalizeNonTrivialProject() throws Exception {
FeConstants.USE_MOCK_DICT_MANAGER = false;
}
}
<<<<<<< HEAD
=======

@Test
public void testListPartitionPrunerWithNEExpr() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getCostPlanFragment(getDumpInfoFromFile("query_dump/list_partition_prune_dump"));
// partitions should not be pruned
Assert.assertTrue(replayPair.second, !replayPair.second.contains("partitionsRatio=2/3, tabletsRatio=20/20"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("0:OlapScanNode\n" +
" table: partitions2_keys311, rollup: partitions2_keys311\n" +
" preAggregation: on\n" +
" Predicates: [7: undef_signed_not_null, VARCHAR, false] != 'j'\n" +
" partitionsRatio=3/3, tabletsRatio=30/30"));
}

@Test
public void testTopNPushDownBelowUnionAll() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/topn_push_down_union"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);

// Topn should be pushed down below union all and contains no duplicated ording columns
PlanTestBase.assertContains(replayPair.second, " 26:TOP-N\n" +
" | order by: <slot 240> 240: expr ASC, <slot 241> 241: cast DESC, <slot 206> 206: mock_025 DESC\n" +
" | offset: 0\n" +
" | limit: 200");
PlanTestBase.assertContains(replayPair.second, "17:TOP-N\n" +
" | order by: <slot 165> 165: cast ASC, <slot 153> 153: cast DESC, <slot 166> 166: expr ASC, " +
"<slot 167> 167: cast DESC\n" +
" | offset: 0\n" +
" | limit: 200");
}

@Test
public void testNoCTEOperatorPropertyDerived() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/no_cte_operator_test"),
null, TExplainLevel.NORMAL);
Assert.assertTrue(replayPair.second, replayPair.second.contains("23:Project\n" +
" | <slot 193> : 193: mock_081\n" +
" | <slot 194> : 194: mock_089\n" +
" | <slot 391> : 391: case\n" +
" | <slot 396> : 396: rank()"));
Assert.assertTrue(replayPair.second, replayPair.second.contains(" 20:SORT\n" +
" | order by: <slot 194> 194: mock_089 ASC," +
" <slot 395> 395: case ASC, <slot 193> 193: mock_081 ASC, " +
"<slot 233> 233: mock_065 ASC\n" +
" | analytic partition by: 194: mock_089, 395: case, 193: mock_081\n" +
" | offset: 0\n" +
" | \n" +
" 19:EXCHANGE"));
}

@Test
public void testTimeoutDeepJoinCostPrune() throws Exception {
Tracers.register(connectContext);
Tracers.init(connectContext, Tracers.Mode.TIMER, "optimizer");
connectContext.getSessionVariable().setOptimizerExecuteTimeout(-1);

Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/deep_join_cost"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
String ss = Tracers.printScopeTimer();
int start = ss.indexOf("EnforceAndCostTask[") + "EnforceAndCostTask[".length();
int end = ss.indexOf("]", start);
long count = Long.parseLong(ss.substring(start, end));
Assert.assertTrue(ss, count < 10000);
}

@Test
public void testDistinctConstantRewrite() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/distinct_constant"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
Assert.assertTrue(replayPair.second, replayPair.second.contains("4:AGGREGATE (update serialize)\n" +
" | output: multi_distinct_count(1)"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("9:AGGREGATE (update serialize)\n" +
" | output: multi_distinct_count(NULL)"));
}

@Test
public void testSplitOrderBy() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/split_order_by"),
null, TExplainLevel.NORMAL);
Assert.assertTrue(replayPair.second, replayPair.second.contains("21:MERGING-EXCHANGE"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("20:TOP-N"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("15:MERGING-EXCHANGE"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("14:TOP-N"));

}

@Test
public void testQueryCacheSetOperator() throws Exception {

String savedSv = connectContext.getSessionVariable().getJsonString();
try {
connectContext.getSessionVariable().setEnableQueryCache(true);
QueryDumpInfo dumpInfo = getDumpInfoFromJson(getDumpInfoFromFile("query_dump/query_cache_set_operator"));
ExecPlan execPlan = UtFrameUtils.getPlanFragmentFromQueryDump(connectContext, dumpInfo);
Assert.assertTrue(execPlan.getFragments().stream().anyMatch(frag -> frag.getCacheParam() != null));
} finally {
connectContext.getSessionVariable().replayFromJson(savedSv);
}
}

@Test
public void testQueryTimeout() {
Assert.assertThrows(StarRocksPlannerException.class,
() -> getPlanFragment(getDumpInfoFromFile("query_dump/query_timeout"), null, TExplainLevel.NORMAL));
}

@Test
public void testQueryCacheMisuseExogenousRuntimeFilter() throws Exception {
String savedSv = connectContext.getSessionVariable().getJsonString();
try {
connectContext.getSessionVariable().setEnableQueryCache(true);
QueryDumpInfo dumpInfo =
getDumpInfoFromJson(getDumpInfoFromFile("query_dump/query_cache_misuse_exogenous_runtime_filter"));
ExecPlan execPlan = UtFrameUtils.getPlanFragmentFromQueryDump(connectContext, dumpInfo);
Assert.assertTrue(execPlan.getFragments().stream().noneMatch(frag -> frag.getCacheParam() != null));
Assert.assertTrue(
execPlan.getFragments().stream().anyMatch(frag -> !frag.getProbeRuntimeFilters().isEmpty()));
} finally {
connectContext.getSessionVariable().replayFromJson(savedSv);
}
}

@Test
public void testPruneTableNPE() throws Exception {
String savedSv = connectContext.getSessionVariable().getJsonString();
try {
connectContext.getSessionVariable().setEnableCboTablePrune(true);
connectContext.getSessionVariable().setEnableRboTablePrune(true);
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/prune_table_npe"),
null, TExplainLevel.NORMAL);
long numHashJoins = Stream.of(replayPair.second.split("\n"))
.filter(ln -> ln.contains("HASH JOIN")).count();
Assert.assertEquals(numHashJoins, 2);
} finally {
connectContext.getSessionVariable().replayFromJson(savedSv);
}
}
>>>>>>> 61ebfc79f8 ([BugFix] Operator's statistics not be estimated during JoinReorder of table-pruning phase (#51304))
}
Loading

0 comments on commit b68f597

Please sign in to comment.