Make the behavior of the substring function consistent with Spark

.gitignore

@@ -167,3 +167,6 @@ tests/queries/0_stateless/*.expect.history
 /rust/**/target
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
+
+utils/local-engine/tests/testConfig.h
+/utils/extern-local-engine

CMakeLists.txt

@@ -398,6 +398,8 @@ endif ()
 
 include(cmake/dbms_glob_sources.cmake)
 
+set (CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 add_library(global-group INTERFACE)
 if (OS_LINUX OR OS_ANDROID)
     include(cmake/linux/default_libs.cmake)
@@ -434,7 +436,6 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
 if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X))
     # Slightly more efficient code can be generated
     # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
@@ -497,8 +498,10 @@ macro (clickhouse_add_executable target)
     if (${type} STREQUAL EXECUTABLE)
         # disabled for TSAN and gcc since libtsan.a provides overrides too
         if (TARGET clickhouse_new_delete)
-            # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+            if (NOT ${target} STREQUAL Git::Git)
+                # operator::new/delete for executables (MemoryTracker stuff)
+                target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+            endif()
         endif()
 
         # In case of static jemalloc, because zone_register() is located in zone.c and

base/base/IPv4andIPv6.h

@@ -51,3 +51,15 @@ namespace DB
     };
 
 }
+
+namespace std
+{
+    template <>
+    struct hash<DB::IPv6>
+    {
+        size_t operator()(const DB::IPv6 & x) const
+        {
+            return std::hash<DB::IPv6::UnderlyingType>()(x.toUnderType());
+        }
+    };
+}

base/base/strong_typedef.h

@@ -35,7 +35,7 @@ struct StrongTypedef
     Self & operator=(T && rhs) { t = std::move(rhs); return *this;}
 
     // NOLINTBEGIN(google-explicit-constructor)
-    operator const T & () const { return t; }
+    constexpr operator const T & () const { return t; }
     operator T & () { return t; }
     // NOLINTEND(google-explicit-constructor)
 

cmake/autogenerated_versions.txt

@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54472)
 SET(VERSION_MAJOR 23)
 SET(VERSION_MINOR 3)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 52bf836e03a6ba7cf2d654eaaf73231701abc3a2)
-SET(VERSION_DESCRIBE v23.3.1.2537-testing)
-SET(VERSION_STRING 23.3.1.2537)
+SET(VERSION_PATCH 3)
+SET(VERSION_GITHASH 1b144bcd101ddf23466ba67e4fa0fd27afb9c060)
+SET(VERSION_DESCRIBE v23.3.3.1-lts)
+SET(VERSION_STRING 23.3.3.1)
 # end of autochange

docker/test/integration/runner/Dockerfile

@@ -46,10 +46,9 @@ ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 ENV DOCKER_CHANNEL stable
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}"
-
-RUN apt-get update \
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+    && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
+    && apt-get update \
     && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
         docker-ce \
     && rm -rf \
@@ -60,7 +59,7 @@ RUN apt-get update \
 
 RUN dockerd --version; docker --version
 
-RUN python3 -m pip install \
+RUN python3 -m pip install --no-cache-dir \
     PyMySQL \
     aerospike==4.0.0 \
     avro==1.10.2 \

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -16,7 +16,9 @@ echo '{
 # and on hung you can simply press Ctrl-C and it will spawn a python pdb,
 # but on SIGINT dockerd will exit, so ignore it to preserve the daemon.
 trap '' INT
-dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log &
+# Binding to an IP address without --tlsverify is deprecated. Startup is intentionally being slowed
+# unless --tls=false or --tlsverify=false is set
+dockerd --host=unix:///var/run/docker.sock --tls=false --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log &
 
 set +e
 reties=0

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -68,9 +68,9 @@ Result:
 
 ## mapFromArrays
 
-Merges an [Array](../../sql-reference/data-types/array.md) of keys and an [Array](../../sql-reference/data-types/array.md) of values into a [Map(key, value)](../../sql-reference/data-types/map.md).
+Merges an [Array](../../sql-reference/data-types/array.md) of keys and an [Array](../../sql-reference/data-types/array.md) of values into a [Map(key, value)](../../sql-reference/data-types/map.md). Notice that the second argument could also be a [Map](../../sql-reference/data-types/map.md), thus it is casted to an Array when executing.
 
-The function is a more convenient alternative to `CAST((key_array, value_array), 'Map(key_type, value_type)')`. For example, instead of writing `CAST((['aa', 'bb'], [4, 5]), 'Map(String, UInt32)')`, you can write `mapFromArrays(['aa', 'bb'], [4, 5])`.
+The function is a more convenient alternative to `CAST((key_array, value_array_or_map), 'Map(key_type, value_type)')`. For example, instead of writing `CAST((['aa', 'bb'], [4, 5]), 'Map(String, UInt32)')`, you can write `mapFromArrays(['aa', 'bb'], [4, 5])`.
 
 **Syntax**
 
@@ -82,25 +82,29 @@ Alias: `MAP_FROM_ARRAYS(keys, values)`
 
 **Arguments**
 -   `keys` — Given key array to create a map from. The nested type of array must be: [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md)
--   `values`  - Given value array to create a map from.
+-   `values`  - Given value array or map to create a map from.
 
 **Returned value**
 
-- A map whose keys and values are constructed from the key and value arrays
+- A map whose keys and values are constructed from the key array and value array/map.
 
 **Example**
 
 Query:
 
 ```sql
 select mapFromArrays(['a', 'b', 'c'], [1, 2, 3])
-```
-
-```text
+
 ┌─mapFromArrays(['a', 'b', 'c'], [1, 2, 3])─┐
 │ {'a':1,'b':2,'c':3}                       │
 └───────────────────────────────────────────┘
-```  
+
+SELECT mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))
+
+┌─mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))─┐
+│ {1:('a',1),2:('b',2),3:('c',3)}                       │
+└───────────────────────────────────────────────────────┘
+```
 
 ## mapAdd
 

programs/server/MetricsTransmitter.cpp

@@ -87,7 +87,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
 
     if (send_events)
     {
-        for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
+        for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
         {
             const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
             const auto counter_increment = counter - prev_counters[i];
@@ -100,7 +100,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
 
     if (send_events_cumulative)
     {
-        for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
+        for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i)
         {
             const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
             std::string key{ProfileEvents::getName(static_cast<ProfileEvents::Event>(i))};
@@ -110,7 +110,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
 
     if (send_metrics)
     {
-        for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i)
+        for (CurrentMetrics::Metric i = CurrentMetrics::Metric(0), end = CurrentMetrics::end(); i < end; ++i)
         {
             const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed);
 

src/AggregateFunctions/AggregateFunctionGroupArray.cpp

@@ -121,7 +121,7 @@ AggregateFunctionPtr createAggregateFunctionGroupArraySample(
 
 void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
 {
-    AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
+    AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true };
 
     factory.registerFunction("groupArray", { createAggregateFunctionGroupArray<false>, properties });
     factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });

src/AggregateFunctions/Helpers.h

@@ -130,6 +130,8 @@ static IAggregateFunction * createWithNumericBasedType(const IDataType & argumen
     if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<UInt16>(std::forward<TArgs>(args)...);
     if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<UInt32>(std::forward<TArgs>(args)...);
     if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UUID>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::IPv4) return new AggregateFunctionTemplate<IPv4>(std::forward<TArgs>(args)...);
+    if (which.idx == TypeIndex::IPv6) return new AggregateFunctionTemplate<IPv6>(std::forward<TArgs>(args)...);
     return nullptr;
 }
 

src/Columns/ColumnAggregateFunction.cpp

@@ -525,6 +525,35 @@ void ColumnAggregateFunction::insertDefault()
     pushBackAndCreateState(data, arena, func.get());
 }
 
+void ColumnAggregateFunction::insertRangeSelective(
+    const IColumn & from, const IColumn::Selector & selector, size_t selector_start, size_t length)
+{
+    const ColumnAggregateFunction & from_concrete = static_cast<const ColumnAggregateFunction &>(from);
+    const auto & from_data = from_concrete.data;
+    if (!empty() && src.get() != &from_concrete)
+    {
+        ensureOwnership();
+        Arena & arena = createOrGetArena();
+        Arena * arena_ptr = &arena;
+        data.reserve(size() + length);
+        for (size_t i = 0; i < length; ++i)
+        {
+            pushBackAndCreateState(data, arena, func.get());
+            func->merge(data.back(), from_data[selector[selector_start + i]], arena_ptr);
+        }
+        return;
+    }
+    /// Keep shared ownership of aggregation states.
+    src = from_concrete.getPtr();
+
+    size_t old_size = data.size();
+    data.resize(old_size + length);
+    auto * data_start = data.data();
+    size_t element_size = sizeof(data[0]);
+    for (size_t i = 0; i < length; ++i)
+        memcpy(data_start + old_size + i, &from_concrete.data[selector[selector_start + i]], element_size);
+}
+
 StringRef ColumnAggregateFunction::serializeValueIntoArena(size_t n, Arena & arena, const char *& begin) const
 {
     WriteBufferFromArena out(arena, begin);

src/Columns/ColumnAggregateFunction.h

@@ -184,6 +184,8 @@ class ColumnAggregateFunction final : public COWHelper<IColumn, ColumnAggregateF
 
     void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
 
+    void insertRangeSelective(const IColumn & src, const IColumn::Selector & selector, size_t selector_start, size_t length) override;
+
     void popBack(size_t n) override;
 
     ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;

src/Columns/ColumnArray.cpp

@@ -547,6 +547,29 @@ void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t leng
     }
 }
 
+void ColumnArray::insertRangeSelective(const IColumn & src, const Selector & selector, size_t selector_start, size_t length){
+
+    const ColumnArray & src_concrete = static_cast<const ColumnArray &>(src);
+    const Offsets & src_offsets = src_concrete.getOffsets();
+    const IColumn & src_data = src_concrete.getData();
+    IColumn & cur_data = getData();
+    Offsets & cur_offsets = getOffsets();
+
+    size_t old_size = cur_offsets.size();
+    size_t cur_size = old_size + length;
+    cur_data.reserve(cur_size);
+    cur_offsets.resize(cur_size);
+
+    for (size_t i = 0; i < length; ++i)
+    {
+        size_t src_pos = selector[selector_start + i];
+        size_t offset = src_offsets[src_pos - 1];
+        size_t size = src_offsets[src_pos] - offset;
+        cur_data.insertRangeFrom(src_data, offset, size);
+        cur_offsets[old_size + i] = cur_offsets[old_size + i - 1] + size; // PaddedPODArray allows to use -1th element that will have value 0
+    }
+}
+
 
 ColumnPtr ColumnArray::filter(const Filter & filt, ssize_t result_size_hint) const
 {

src/Columns/ColumnArray.h

@@ -84,6 +84,7 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray>
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+    void insertRangeSelective(const IColumn & src, const Selector & selector, size_t selector_start, size_t length) override;
     void insert(const Field & x) override;
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertDefault() override;
@@ -149,6 +150,8 @@ class ColumnArray final : public COWHelper<IColumn, ColumnArray>
 
     void gather(ColumnGathererStream & gatherer_stream) override;
 
+    bool canBeInsideNullable() const override { return true; }
+
     ColumnPtr compress() const override;
 
     void forEachSubcolumn(ColumnCallback callback) const override

src/Columns/ColumnCompressed.h

@@ -85,6 +85,7 @@ class ColumnCompressed : public COWHelper<IColumn, ColumnCompressed>
     bool isDefaultAt(size_t) const override { throwMustBeDecompressed(); }
     void insert(const Field &) override { throwMustBeDecompressed(); }
     void insertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
+    void insertRangeSelective(const IColumn &, const Selector &, size_t, size_t) override { throwMustBeDecompressed(); }
     void insertData(const char *, size_t) override { throwMustBeDecompressed(); }
     void insertDefault() override { throwMustBeDecompressed(); }
     void popBack(size_t) override { throwMustBeDecompressed(); }

src/Columns/ColumnConst.h

@@ -126,6 +126,11 @@ class ColumnConst final : public COWHelper<IColumn, ColumnConst>
         s += length;
     }
 
+    void insertRangeSelective(const IColumn & /*src*/, const Selector & /*selector*/, size_t /*selector_start*/, size_t length) override
+    {
+        s += length;
+    }
+
     void insert(const Field &) override
     {
         ++s;

src/Columns/ColumnDecimal.cpp

@@ -261,6 +261,18 @@ void ColumnDecimal<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
     memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
 }
 
+template <is_decimal T>
+void ColumnDecimal<T>::insertRangeSelective(const IColumn & src, const IColumn::Selector & selector, size_t selector_start, size_t length)
+{
+    size_t old_size = data.size();
+    data.resize(old_size + length);
+    const auto & src_data = (static_cast<const Self &>(src)).getData();
+    for (size_t i = 0; i < length; ++i)
+    {
+        data[old_size + i] = src_data[selector[selector_start + i]];
+    }
+}
+
 template <is_decimal T>
 ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {

src/Columns/ColumnDecimal.h

@@ -62,6 +62,7 @@ class ColumnDecimal final : public COWHelper<ColumnVectorHelper, ColumnDecimal<T
     void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); }
     void insert(const Field & x) override { data.push_back(x.get<T>()); }
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
+    void insertRangeSelective(const IColumn & src, const IColumn::Selector & selector, size_t selector_start, size_t length) override;
 
     void popBack(size_t n) override
     {

src/Columns/ColumnFixedString.cpp

@@ -202,6 +202,22 @@ void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_
     memcpy(chars.data() + old_size, &src_concrete.chars[start * n], length * n);
 }
 
+void ColumnFixedString::insertRangeSelective(const IColumn & src, const IColumn::Selector & selector, size_t selector_start, size_t length)
+{
+    const ColumnFixedString & src_concrete = static_cast<const ColumnFixedString &>(src);
+
+    size_t old_size = chars.size();
+    chars.resize(old_size + length * n);
+    auto * cur_data_end = chars.data() + old_size;
+    auto * src_data_start = src_concrete.chars.data();
+
+    for (size_t i = 0; i < length; ++i)
+    {
+        size_t src_pos = selector[selector_start + i];
+        memcpySmallAllowReadWriteOverflow15(cur_data_end + i * n, src_data_start+ n * src_pos, n);
+    }
+}
+
 ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
 {
     size_t col_size = size();

src/Columns/ColumnFixedString.h

@@ -154,6 +154,8 @@ class ColumnFixedString final : public COWHelper<ColumnVectorHelper, ColumnFixed
 
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 
+    void insertRangeSelective(const IColumn & src, const Selector & selector, size_t selector_start, size_t length) override;
+
     ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;
 
     void expand(const IColumn::Filter & mask, bool inverted) override;

src/Columns/ColumnMap.cpp

@@ -148,6 +148,11 @@ void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length
         start, length);
 }
 
+void ColumnMap::insertRangeSelective(const IColumn & src, const Selector & selector, size_t selector_start, size_t length)
+{
+    nested->insertRangeSelective(assert_cast<const ColumnMap &>(src).getNestedColumn(), selector, selector_start, length);
+}
+
 ColumnPtr ColumnMap::filter(const Filter & filt, ssize_t result_size_hint) const
 {
     auto filtered = nested->filter(filt, result_size_hint);