From e1a7a760733a47080d80244801d8632f478b3129 Mon Sep 17 00:00:00 2001 From: meegoo Date: Wed, 4 Dec 2024 08:20:39 +0800 Subject: [PATCH] [BugFix] Fix delete fail on multi level partition primary key table Signed-off-by: meegoo --- .../java/com/starrocks/sql/DeletePlanner.java | 3 +- .../sql/analyzer/CreateTableAnalyzer.java | 2 +- .../sql/analyzer/DeleteAnalyzer.java | 5 +- .../R/test_multi_expr | 178 ++++++++++++++++-- .../T/test_multi_expr | 50 ++++- 5 files changed, 220 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java index 57e0653f43358..db013ed7c2fb4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java @@ -24,6 +24,7 @@ import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Partition; import com.starrocks.catalog.Type; +import com.starrocks.common.FeConstants; import com.starrocks.common.StarRocksException; import com.starrocks.load.Load; import com.starrocks.planner.DataSink; @@ -86,7 +87,7 @@ public ExecPlan plan(DeleteStmt deleteStatement, ConnectContext session) { OlapTable table = (OlapTable) deleteStatement.getTable(); for (Column column : table.getBaseSchema()) { - if (column.isKey()) { + if (column.isKey() || column.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) { SlotDescriptor slotDescriptor = descriptorTable.addSlotDescriptor(olapTuple); slotDescriptor.setIsMaterialized(true); slotDescriptor.setType(column.getType()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java index bc3e90347d4c6..bc4bb2d7be1da 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java @@ -495,7 +495,7 @@ public static void analyzeMultiExprsPartition(CreateTableStmt stmt, TableName ta FunctionCallExpr expr = (FunctionCallExpr) partitionExpr; ExpressionAnalyzer.analyzeExpression(expr, new AnalyzeState(), new Scope(RelationId.anonymous(), new RelationFields(columnDefs.stream().map(col -> new Field(col.getName(), - col.getType(), tableName, null)).collect(Collectors.toList()))), + col.getType(), null, null)).collect(Collectors.toList()))), new ConnectContext()); String columnName = FeConstants.GENERATED_PARTITION_COLUMN_PREFIX + placeHolderSlotId++; partitionColumnList.add(columnName); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java index 983b2f0d6d52b..f2b3fd8bc215b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DeleteAnalyzer.java @@ -38,6 +38,7 @@ import com.starrocks.catalog.Table; import com.starrocks.catalog.Type; import com.starrocks.common.Config; +import com.starrocks.common.FeConstants; import com.starrocks.load.Load; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; @@ -229,10 +230,10 @@ public static void analyze(DeleteStmt deleteStatement, ConnectContext session) { SelectList selectList = new SelectList(); for (Column col : table.getBaseSchema()) { SelectListItem item; - if (col.isKey()) { + if (col.isKey() || col.isNameWithPrefix(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) { item = new SelectListItem(new SlotRef(tableName, col.getName()), col.getName()); } else { - break; + continue; } selectList.addItem(item); } diff --git a/test/sql/test_automatic_partition/R/test_multi_expr b/test/sql/test_automatic_partition/R/test_multi_expr index 3ab1414124c7f..a8db4daf80c0a 100644 --- a/test/sql/test_automatic_partition/R/test_multi_expr +++ b/test/sql/test_automatic_partition/R/test_multi_expr @@ -30,6 +30,11 @@ select * from t; + + + + + @@ -98,6 +103,11 @@ explain select * from t where k2='2020-01-03'; + + + + + -- name: test_mulit_timestamp_function create table t(k1 int, k2 bigint, v int) partition by from_unixtime(k2),k1; -- result: @@ -163,6 +173,11 @@ explain select * from t1 where k2=1196389819; + + + + + -- name: test_single_column_partition create table t(k1 int, k2 bigint, v int) partition by from_unixtime(k2, '%Y-%m-%d'); -- result: @@ -196,6 +211,11 @@ explain select * from t where k2=UNIX_TIMESTAMP('2007-11-30 10:30:19'); + + + + + -- name: test_primary_key_table create table t(k1 int, k2 bigint, v int) PRIMARY KEY(k1, k2) partition by from_unixtime(k2, '%Y-%m-%d'); -- result: @@ -288,6 +308,11 @@ explain select * from t1 where k2=1196389819; + + + + + -- name: test_create_error create table t(k1 int, k2 bigint, v int sum) AGGREGATE KEY(k1,k2) partition by from_unixtime(k2, '%Y-%m-%d'); -- result: @@ -308,6 +333,11 @@ E: (1064, 'Getting analyzing error. Detail message: The partition expr should ba + + + + + -- name: test_insert_into_select CREATE TABLE multi_level_expr_par_tbl ( `k1` date, @@ -350,13 +380,12 @@ select * from multi_level_expr_par_tbl; + + + + + -- name: test_ctas_from_normal_table -create database test_ctas_from_normal_table; --- result: --- !result -use test_ctas_from_normal_table; --- result: --- !result CREATE TABLE base_tbl_unix_ts (c1 bigint, c2 string, c3 date); -- result: -- !result @@ -376,7 +405,7 @@ multi_level_expr_par_tbl_1 CREATE TABLE `multi_level_expr_par_tbl_1` ( `c1` bigint(20) NOT NULL COMMENT "", `c2` varchar(65533) NULL COMMENT "", `c3` date NULL COMMENT "", - `__generated_partition_column_0` varchar(1048576) NULL AS from_unixtime(`test_ctas_from_normal_table`.`multi_level_expr_par_tbl_1`.`c1`) COMMENT "" + `__generated_partition_column_0` varchar(1048576) NULL AS from_unixtime(`c1`) COMMENT "" ) ENGINE=OLAP PRIMARY KEY(`c1`) PARTITION BY (`__generated_partition_column_0`) @@ -431,8 +460,8 @@ multi_level_expr_par_tbl_2 CREATE TABLE `multi_level_expr_par_tbl_2` ( `k11` decimal(38, 9) NULL COMMENT "", `k12` decimal(38, 9) NULL COMMENT "", `k13` decimal(27, 9) NULL COMMENT "", - `__generated_partition_column_0` varchar(1048576) NULL AS substring(`test_ctas_from_normal_table`.`multi_level_expr_par_tbl_2`.`k4`, 1, 5) COMMENT "", - `__generated_partition_column_1` date NULL AS date_trunc('month', `test_ctas_from_normal_table`.`multi_level_expr_par_tbl_2`.`k1`) COMMENT "" + `__generated_partition_column_0` varchar(1048576) NULL AS substring(`k4`, 1, 5) COMMENT "", + `__generated_partition_column_1` date NULL AS date_trunc('month', `k1`) COMMENT "" ) ENGINE=OLAP PRIMARY KEY(`k1`, `k2`, `k3`, `k4`, `k5`) COMMENT "OLAP" @@ -447,9 +476,6 @@ PROPERTIES ( ); -- !result - - - -- name: test_create_reserved_column create table t(k1 int, k2 bigint, __generated_partition_column_ int); -- result: @@ -492,4 +518,132 @@ PROPERTIES ( "replication_num" = "3" ); -- result: +-- !result + + + + +-- name: test_delete +CREATE TABLE test ( + `k1` date, + `k2` datetime, + `k3` varchar(20), + `k4` varchar(20), + `k5` boolean, + `k6` tinyint +) +primary KEY(k1,k2,k3,k4,k5) +COMMENT "OLAP" +PARTITION BY substring(k4, 1, 5), k3, date_trunc("month", k1) +DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3; +-- result: +-- !result +insert into test values('2020-01-01','2020-01-01',"abc","abc",true,1); +-- result: +-- !result +insert into test values('2020-04-01','2020-01-01',"abc","abc",true,1); +-- result: +-- !result +select * from test; +-- result: +2020-01-01 2020-01-01 00:00:00 abc abc 1 1 +2020-04-01 2020-01-01 00:00:00 abc abc 1 1 +-- !result +delete from test where k1 = '2020-04-01'; +-- result: +-- !result +select * from test; +-- result: +2020-01-01 2020-01-01 00:00:00 abc abc 1 1 +-- !result + + + +-- name: test_create_table_like +CREATE TABLE base_tbl ( + `k1` date, + `k2` datetime, + `k3` varchar(20), + `k4` varchar(20), + `k5` boolean, + `k6` tinyint, + `k7` smallint, + `k8` int, + `k9` bigint, + `k10` largeint, + `k11` float, + `k12` double, + `k13` decimal(27,9) + ) + duplicate KEY(k1,k2,k3,k4,k5) + COMMENT "OLAP" + PARTITION BY substring(k4, 1, 5), k3, date_trunc("month", k1) + DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3 + PROPERTIES ( + "replication_num" = "3" + ); +-- result: +-- !result +show create table base_tbl; +-- result: +base_tbl CREATE TABLE `base_tbl` ( + `k1` date NULL COMMENT "", + `k2` datetime NULL COMMENT "", + `k3` varchar(20) NULL COMMENT "", + `k4` varchar(20) NULL COMMENT "", + `k5` boolean NULL COMMENT "", + `k6` tinyint(4) NULL COMMENT "", + `k7` smallint(6) NULL COMMENT "", + `k8` int(11) NULL COMMENT "", + `k9` bigint(20) NULL COMMENT "", + `k10` largeint(40) NULL COMMENT "", + `k11` float NULL COMMENT "", + `k12` double NULL COMMENT "", + `k13` decimal(27, 9) NULL COMMENT "", + `__generated_partition_column_0` varchar(1048576) NULL AS substring(`k4`, 1, 5) COMMENT "", + `__generated_partition_column_1` date NULL AS date_trunc('month', `k1`) COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`) +COMMENT "OLAP" +PARTITION BY (`__generated_partition_column_0`,`k3`,`__generated_partition_column_1`) +DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3 +PROPERTIES ( +"compression" = "LZ4", +"fast_schema_evolution" = "true", +"replicated_storage" = "true", +"replication_num" = "3" +); +-- !result +create table t1 like base_tbl; +-- result: +-- !result +show create table t1; +-- result: +t1 CREATE TABLE `t1` ( + `k1` date NULL COMMENT "", + `k2` datetime NULL COMMENT "", + `k3` varchar(20) NULL COMMENT "", + `k4` varchar(20) NULL COMMENT "", + `k5` boolean NULL COMMENT "", + `k6` tinyint(4) NULL COMMENT "", + `k7` smallint(6) NULL COMMENT "", + `k8` int(11) NULL COMMENT "", + `k9` bigint(20) NULL COMMENT "", + `k10` largeint(40) NULL COMMENT "", + `k11` float NULL COMMENT "", + `k12` double NULL COMMENT "", + `k13` decimal(27, 9) NULL COMMENT "", + `__generated_partition_column_0` varchar(1048576) NULL AS substring(`k4`, 1, 5) COMMENT "", + `__generated_partition_column_1` date NULL AS date_trunc('month', `k1`) COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`k1`, `k2`, `k3`, `k4`, `k5`) +COMMENT "OLAP" +PARTITION BY (`__generated_partition_column_0`,`k3`,`__generated_partition_column_1`) +DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3 +PROPERTIES ( +"compression" = "LZ4", +"fast_schema_evolution" = "true", +"replicated_storage" = "true", +"replication_num" = "3" +); -- !result \ No newline at end of file diff --git a/test/sql/test_automatic_partition/T/test_multi_expr b/test/sql/test_automatic_partition/T/test_multi_expr index 31b439301a89d..033b444058c55 100644 --- a/test/sql/test_automatic_partition/T/test_multi_expr +++ b/test/sql/test_automatic_partition/T/test_multi_expr @@ -101,8 +101,6 @@ insert into multi_level_expr_par_tbl select * from multi_level_expr_par_tbl; select * from multi_level_expr_par_tbl; -- name: test_ctas_from_normal_table -create database test_ctas_from_normal_table; -use test_ctas_from_normal_table; CREATE TABLE base_tbl_unix_ts (c1 bigint, c2 string, c3 date); insert into base_tbl_unix_ts values(1592841600, 'beijing', '2020-06-23'); CREATE TABLE multi_level_expr_par_tbl_1 primary KEY(c1) PARTITION BY from_unixtime(c1) DISTRIBUTED BY HASH(c1) BUCKETS 3 as select * from base_tbl_unix_ts; @@ -166,3 +164,51 @@ PROPERTIES ( "replicated_storage" = "true", "replication_num" = "3" ); + +-- name: test_delete +CREATE TABLE test ( + `k1` date, + `k2` datetime, + `k3` varchar(20), + `k4` varchar(20), + `k5` boolean, + `k6` tinyint +) +primary KEY(k1,k2,k3,k4,k5) +COMMENT "OLAP" +PARTITION BY substring(k4, 1, 5), k3, date_trunc("month", k1) +DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3; + +insert into test values('2020-01-01','2020-01-01',"abc","abc",true,1); +insert into test values('2020-04-01','2020-01-01',"abc","abc",true,1); + +select * from test; +delete from test where k1 = '2020-04-01'; +select * from test; + +-- name: test_create_table_like +CREATE TABLE base_tbl ( + `k1` date, + `k2` datetime, + `k3` varchar(20), + `k4` varchar(20), + `k5` boolean, + `k6` tinyint, + `k7` smallint, + `k8` int, + `k9` bigint, + `k10` largeint, + `k11` float, + `k12` double, + `k13` decimal(27,9) + ) + duplicate KEY(k1,k2,k3,k4,k5) + COMMENT "OLAP" + PARTITION BY substring(k4, 1, 5), k3, date_trunc("month", k1) + DISTRIBUTED BY HASH(`k1`, `k2`, `k3`) BUCKETS 3 + PROPERTIES ( + "replication_num" = "3" + ); +show create table base_tbl; +create table t1 like base_tbl; +show create table t1;