From 9877f7221364e2ba56fb5510967a3af8f30aa179 Mon Sep 17 00:00:00 2001 From: stdpain <34912776+stdpain@users.noreply.github.com> Date: Mon, 30 Dec 2024 20:27:18 +0800 Subject: [PATCH 1/2] [BugFix] Use a separate thread pool for dictionary collection (#54454) Signed-off-by: stdpain (cherry picked from commit 9a6445a4879a101119645a7e3d76f1b123095fe4) --- .../java/com/starrocks/common/Config.java | 3 ++ .../statistics/CacheDictManager.java | 5 ++- .../catalog/CacheDictManagerTest.java | 35 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java index f73dd5e05a34b..53362dde915b8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/Config.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/Config.java @@ -2040,6 +2040,9 @@ public class Config extends ConfigBase { @ConfField public static long statistic_dict_columns = 100000; + @ConfField + public static int dict_collect_thread_pool_size = 16; + /** * The column statistic cache update interval */ diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java index 62fe42ad6bdbf..6ee54e8eefedc 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/statistics/CacheDictManager.java @@ -25,6 +25,7 @@ import com.starrocks.common.Config; import com.starrocks.common.Pair; import com.starrocks.common.Status; +import com.starrocks.common.ThreadPoolManager; import com.starrocks.memory.MemoryTrackable; import com.starrocks.qe.ConnectContext; import com.starrocks.server.GlobalStateMgr; @@ -54,7 +55,7 @@ public class CacheDictManager implements IDictManager, MemoryTrackable { public static final Integer LOW_CARDINALITY_THRESHOLD = 255; - private CacheDictManager() { + public CacheDictManager() { } private static final CacheDictManager INSTANCE = new CacheDictManager(); @@ -107,6 +108,8 @@ public CompletableFuture> asyncReload( private final AsyncLoadingCache> dictStatistics = Caffeine.newBuilder() .maximumSize(Config.statistic_dict_columns) + .executor(ThreadPoolManager.newDaemonCacheThreadPool(Config.dict_collect_thread_pool_size, "cache-dict", + false)) .buildAsync(dictLoader); private Optional deserializeColumnDict(long tableId, ColumnId columnName, TStatisticData statisticData) { diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java new file mode 100644 index 0000000000000..51fa0d051f7f1 --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java @@ -0,0 +1,35 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.catalog; + +import com.starrocks.sql.optimizer.statistics.CacheDictManager; +import mockit.Expectations; +import org.junit.Test; + +import java.util.Optional; + +public class CacheDictManagerTest { + @Test + public void test() { + CacheDictManager manager = new CacheDictManager(); + new Expectations(manager) { + { + manager.getGlobalDict(anyLong, ColumnId.create("val")); + result = Optional.empty(); + } + }; + manager.getGlobalDict(1, ColumnId.create("val")); + } +} From a8adbb8068c0e3b138d0e2ca5f14619875843ce6 Mon Sep 17 00:00:00 2001 From: stdpain <34912776+stdpain@users.noreply.github.com> Date: Tue, 31 Dec 2024 14:20:16 +0800 Subject: [PATCH 2/2] Delete fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java Signed-off-by: stdpain <34912776+stdpain@users.noreply.github.com> --- .../catalog/CacheDictManagerTest.java | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java deleted file mode 100644 index 51fa0d051f7f1..0000000000000 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/CacheDictManagerTest.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2021-present StarRocks, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.starrocks.catalog; - -import com.starrocks.sql.optimizer.statistics.CacheDictManager; -import mockit.Expectations; -import org.junit.Test; - -import java.util.Optional; - -public class CacheDictManagerTest { - @Test - public void test() { - CacheDictManager manager = new CacheDictManager(); - new Expectations(manager) { - { - manager.getGlobalDict(anyLong, ColumnId.create("val")); - result = Optional.empty(); - } - }; - manager.getGlobalDict(1, ColumnId.create("val")); - } -}