From 9a4222d2a8dd3da5a3dc9242370cf944d64a4dc1 Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 31 Dec 2024 11:20:00 +0800 Subject: [PATCH 1/4] Disable stats collection for generated columns Signed-off-by: shuming.li --- .../src/main/java/com/starrocks/statistic/StatisticUtils.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java index cf7a29bc34647..561821a135672 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java @@ -369,6 +369,10 @@ public static List getCollectibleColumns(Table table) { } List columns = new ArrayList<>(); for (Column column : table.getBaseSchema()) { + // disable stats collection for generated columns + if (column.isGeneratedColumn()) { + continue; + } if (!column.isAggregated()) { columns.add(column.getName()); } else if (isPrimaryEngine && column.getAggregationType().equals(AggregateType.REPLACE)) { From bf81bb5e4f35433d1ce436d609defab3f5362d8a Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 31 Dec 2024 13:18:22 +0800 Subject: [PATCH 2/4] add more tests Signed-off-by: shuming.li --- .../statistic/StatisticsCollectJobTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java index 71d8c23367e1c..a87b53b00e25a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java @@ -1535,4 +1535,20 @@ public long getDataSize() { } } + @Test + public void testGetCollectibleColumns() throws Exception { + starRocksAssert.withTable("CREATE TABLE test.t_gen_col (" + + " c1 datetime NOT NULL," + + " c2 bigint," + + " c3 DATETIME NULL AS date_trunc('month', c1) " + + " ) " + + " DUPLICATE KEY(c1) " + + " PARTITION BY (c2, c3) " + + " PROPERTIES('replication_num'='1')"); + Table table = starRocksAssert.getTable("test", "t_gen_col"); + List cols = StatisticUtils.getCollectibleColumns(table); + Assert.assertTrue(cols.size() == 2); + Assert.assertTrue(!cols.contains("c3")); + starRocksAssert.dropTable("test.t_gen_col"); + } } From 940212a659693d90bbcb7da0fc2273563deded17 Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 31 Dec 2024 17:35:53 +0800 Subject: [PATCH 3/4] fix bugs Signed-off-by: shuming.li --- .../starrocks/statistic/StatisticUtils.java | 5 +++-- .../statistic/StatisticsCollectJobTest.java | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java index 561821a135672..c4a36d98b0d8e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java @@ -368,9 +368,10 @@ public static List getCollectibleColumns(Table table) { isPrimaryEngine = KeysType.PRIMARY_KEYS.equals(((OlapTable) table).getKeysType()); } List columns = new ArrayList<>(); + Set partitionColumns = Sets.newHashSet(table.getPartitionColumnNames()); for (Column column : table.getBaseSchema()) { - // disable stats collection for generated columns - if (column.isGeneratedColumn()) { + // disable stats collection for auto generated columns, see SelectAnalyzer#analyzeSelect + if (column.isGeneratedColumn() && column.getName().startsWith(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) { continue; } if (!column.isAggregated()) { diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java index a87b53b00e25a..4a9b0471d1304 100644 --- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java @@ -1536,7 +1536,7 @@ public long getDataSize() { } @Test - public void testGetCollectibleColumns() throws Exception { + public void testGetCollectibleColumns1() throws Exception { starRocksAssert.withTable("CREATE TABLE test.t_gen_col (" + " c1 datetime NOT NULL," + " c2 bigint," + @@ -1547,8 +1547,23 @@ public void testGetCollectibleColumns() throws Exception { " PROPERTIES('replication_num'='1')"); Table table = starRocksAssert.getTable("test", "t_gen_col"); List cols = StatisticUtils.getCollectibleColumns(table); + Assert.assertTrue(cols.size() == 3); + Assert.assertTrue(cols.contains("c3")); + starRocksAssert.dropTable("test.t_gen_col"); + } + + @Test + public void testGetCollectibleColumns2() throws Exception { + starRocksAssert.withTable("CREATE TABLE test.t_gen_col (" + + " c1 datetime NOT NULL," + + " c2 bigint" + + " ) " + + " DUPLICATE KEY(c1) " + + " PARTITION BY c2, date_trunc('month', c1) " + + " PROPERTIES('replication_num'='1')"); + Table table = starRocksAssert.getTable("test", "t_gen_col"); + List cols = StatisticUtils.getCollectibleColumns(table); Assert.assertTrue(cols.size() == 2); - Assert.assertTrue(!cols.contains("c3")); starRocksAssert.dropTable("test.t_gen_col"); } } From 1f9e9fc1c8cf123090766495f4e4d3298eee2531 Mon Sep 17 00:00:00 2001 From: "shuming.li" Date: Tue, 31 Dec 2024 17:39:34 +0800 Subject: [PATCH 4/4] fix bugs Signed-off-by: shuming.li --- .../src/main/java/com/starrocks/statistic/StatisticUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java index c4a36d98b0d8e..d61096899e851 100644 --- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java +++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticUtils.java @@ -368,7 +368,6 @@ public static List getCollectibleColumns(Table table) { isPrimaryEngine = KeysType.PRIMARY_KEYS.equals(((OlapTable) table).getKeysType()); } List columns = new ArrayList<>(); - Set partitionColumns = Sets.newHashSet(table.getPartitionColumnNames()); for (Column column : table.getBaseSchema()) { // disable stats collection for auto generated columns, see SelectAnalyzer#analyzeSelect if (column.isGeneratedColumn() && column.getName().startsWith(FeConstants.GENERATED_PARTITION_COLUMN_PREFIX)) {