[Enhancement] in null/not having null supports normalized to value ra…

…nge (StarRocks#54646) Signed-off-by: trueeyu <[email protected]> Signed-off-by: maggie-zhu <[email protected]>
maggie-zhu · Jan 7, 2025 · 0ed4ed5 · 0ed4ed5
1 parent af470dc
commit 0ed4ed5
Show file tree

Hide file tree

Showing 13 changed files with 531 additions and 109 deletions.
diff --git a/be/src/exec/olap_common.cpp b/be/src/exec/olap_common.cpp
@@ -177,6 +177,17 @@ void ColumnValueRange<T>::clear() {
     _empty_range = false;
 }
 
+template <class T>
+void ColumnValueRange<T>::clear_to_empty() {
+    _fixed_values.clear();
+    _low_value = _type_max;
+    _high_value = _type_min;
+    _low_op = FILTER_LARGER_OR_EQUAL;
+    _high_op = FILTER_LESS_OR_EQUAL;
+    _fixed_op = FILTER_IN;
+    _empty_range = true;
+}
+
 Status OlapScanKeys::get_key_range(std::vector<std::unique_ptr<OlapScanRange>>* key_range) {
     key_range->clear();
 

diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h
@@ -130,6 +130,7 @@ class ColumnValueRange {
     TCondition to_olap_not_null_filter() const;
 
     void clear();
+    void clear_to_empty();
 
 private:
     std::string _column_name;

diff --git a/be/src/exec/olap_scan_prepare.cpp b/be/src/exec/olap_scan_prepare.cpp
@@ -385,15 +385,29 @@ requires(!lt_is_date<SlotType>) Status ChunkPredicateBuilder<E, Type>::normalize
                     continue;
                 }
 
-                if (is_not_in<Negative>(pred) || pred->null_in_set() ||
-                    pred->hash_set().size() > config::max_pushdown_conditions_per_column) {
+                if (is_not_in<Negative>(pred) || pred->hash_set().size() > config::max_pushdown_conditions_per_column) {
                     continue;
                 }
 
                 std::set<RangeValueType> values;
+                if (pred->null_in_set()) {
+                    if (pred->is_eq_null()) {
+                        // TODO: equal null is also can be normalized, will be optimized later
+                        continue;
+                    }
+                    if constexpr (Negative) {
+                        // or col not in (v1, v2, v3, null)
+                        _normalized_exprs[i] = true;
+                        continue;
+                    } else {
+                        // and col in (v1, v2, v3, null), null can be eliminated
+                    }
+                }
+
                 for (const auto& value : pred->hash_set()) {
                     values.insert(value);
                 }
+
                 if (range->add_fixed_values(FILTER_IN, values).ok()) {
                     _normalized_exprs[i] = true;
                 }
@@ -462,10 +476,19 @@ requires lt_is_date<SlotType> Status ChunkPredicateBuilder<E, Type>::normalize_i
                         continue;
                     }
 
-                    if (is_not_in<Negative>(pred) || pred->null_in_set() ||
+                    if (is_not_in<Negative>(pred) ||
                         pred->hash_set().size() > config::max_pushdown_conditions_per_column) {
                         continue;
                     }
+                    if (pred->null_in_set()) {
+                        if (pred->is_eq_null()) {
+                            continue;
+                        }
+                        if constexpr (Negative) {
+                            _normalized_exprs[i] = true;
+                            continue;
+                        }
+                    }
 
                     for (const TimestampValue& ts : pred->hash_set()) {
                         auto date = implicit_cast<DateValue>(ts);
@@ -476,10 +499,19 @@ requires lt_is_date<SlotType> Status ChunkPredicateBuilder<E, Type>::normalize_i
                 } else if (pred_type == starrocks::TYPE_DATE) {
                     const auto* pred = down_cast<const VectorizedInConstPredicate<starrocks::TYPE_DATE>*>(root_expr);
 
-                    if (is_not_in<Negative>(pred) || pred->null_in_set() ||
+                    if (is_not_in<Negative>(pred) ||
                         pred->hash_set().size() > config::max_pushdown_conditions_per_column) {
                         continue;
                     }
+                    if (pred->null_in_set()) {
+                        if (pred->is_eq_null()) {
+                            continue;
+                        }
+                        if constexpr (Negative) {
+                            _normalized_exprs[i] = true;
+                            continue;
+                        }
+                    }
                     for (const DateValue& date : pred->hash_set()) {
                         values.insert(date);
                     }
@@ -759,12 +791,26 @@ Status ChunkPredicateBuilder<E, Type>::normalize_not_in_or_not_equal_predicate(
                     continue;
                 }
 
-                if (!is_not_in<Negative>(pred) || pred->null_in_set() ||
+                if (!is_not_in<Negative>(pred) ||
                     pred->hash_set().size() > config::max_pushdown_conditions_per_column) {
                     continue;
                 }
 
                 std::set<RangeValueType> values;
+                if (pred->null_in_set()) {
+                    if (pred->is_eq_null()) {
+                        continue;
+                    }
+                    if constexpr (!Negative) {
+                        // and col not in (v1, v2, v3, null)
+                        range->clear_to_empty();
+                        _normalized_exprs[i] = true;
+                        continue;
+                    } else {
+                        // or col in (v1, v2, v3, null)
+                    }
+                }
+
                 for (const auto& value : pred->hash_set()) {
                     values.insert(value);
                 }
@@ -1185,4 +1231,5 @@ const UnarrivedRuntimeFilterList& ScanConjunctsManager::unarrived_runtime_filter
 }
 
 template class ChunkPredicateBuilder<BoxedExprContext, CompoundNodeType::AND>;
+template class ChunkPredicateBuilder<BoxedExprContext, CompoundNodeType::OR>;
 } // namespace starrocks
diff --git a/be/src/exprs/in_const_predicate.hpp b/be/src/exprs/in_const_predicate.hpp
@@ -385,6 +385,8 @@ class VectorizedInConstPredicate final : public Predicate {
 
     bool null_in_set() const { return _null_in_set; }
 
+    bool is_eq_null() const { return _eq_null; }
+
     void set_null_in_set(bool v) { _null_in_set = v; }
 
     bool is_join_runtime_filter() const { return _is_join_runtime_filter; }

diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
@@ -370,6 +370,7 @@ static const TypeDescriptor TYPE_SMALLINT_DESC = TypeDescriptor{LogicalType::TYP
 static const TypeDescriptor TYPE_INT_DESC = TypeDescriptor(LogicalType::TYPE_INT);
 static const TypeDescriptor TYPE_BIGINT_DESC = TypeDescriptor(LogicalType::TYPE_BIGINT);
 static const TypeDescriptor TYPE_TIME_DESC = TypeDescriptor(LogicalType::TYPE_TIME);
+static const TypeDescriptor TYPE_DATE_DESC = TypeDescriptor(LogicalType::TYPE_DATE);
 static const TypeDescriptor TYPE_DATETIME_DESC = TypeDescriptor(LogicalType::TYPE_DATETIME);
 static const TypeDescriptor TYPE_CHAR_DESC = TypeDescriptor::create_char_type(TypeDescriptor::MAX_CHAR_LENGTH);
 static const TypeDescriptor TYPE_VARCHAR_DESC = TypeDescriptor::create_varchar_type(TypeDescriptor::MAX_VARCHAR_LENGTH);

diff --git a/be/src/testutil/CMakeLists.txt b/be/src/testutil/CMakeLists.txt
@@ -25,5 +25,6 @@ add_library(TestUtil
     sync_point_impl.cc
     schema_test_helper.cpp
     tablet_test_helper.cpp
+    exprs_test_helper.cpp
 )
 
diff --git a/be/src/testutil/exprs_test_helper.cpp b/be/src/testutil/exprs_test_helper.cpp
@@ -0,0 +1,27 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "testutil/exprs_test_helper.h"
+
+namespace starrocks {
+const TTypeDesc ExprsTestHelper::SmallIntTTypeDesc = ExprsTestHelper::create_scalar_type_desc(TPrimitiveType::SMALLINT);
+const TTypeDesc ExprsTestHelper::IntTTypeDesc = ExprsTestHelper::create_scalar_type_desc(TPrimitiveType::INT);
+const TTypeDesc ExprsTestHelper::BigIntTTypeDesc = ExprsTestHelper::create_scalar_type_desc(TPrimitiveType::BIGINT);
+const TTypeDesc ExprsTestHelper::DateTTypeDesc = ExprsTestHelper::create_scalar_type_desc(TPrimitiveType::DATE);
+const TTypeDesc ExprsTestHelper::DateTimeTTypeDesc = ExprsTestHelper::create_scalar_type_desc(TPrimitiveType::DATETIME);
+const TTypeDesc ExprsTestHelper::Decimal128TTypeDesc =
+        ExprsTestHelper::create_decimal_type_desc(TPrimitiveType::DECIMAL128, 27, 9);
+const TTypeDesc ExprsTestHelper::VarcharTTypeDesc =
+        ExprsTestHelper::create_varchar_type_desc(TypeDescriptor::MAX_VARCHAR_LENGTH);
+} // namespace starrocks