[CH-247] support async reader when reading parquet from hdfs

.github/workflows/backport_branches.yml

@@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy
     branches:
       - 'backport/**'
 jobs:
+  PythonUnitTests:
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Python unit tests
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:

.github/workflows/master.yml

@@ -911,6 +911,34 @@ jobs:
           docker rm -f "$(docker ps -a -q)" ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
+##################################### Docker images  #######################################
+############################################################################################
+  DockerServerImages:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # otherwise we will have no version info
+      - name: Check docker clickhouse/clickhouse-server building
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_server.py --release-type head
+          python3 docker_server.py --release-type head --no-ubuntu \
+            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:

.github/workflows/pull_request.yml

@@ -4,7 +4,7 @@ env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
 
-on: # yamllint disable-line rule:truthy
+on:  # yamllint disable-line rule:truthy
   pull_request:
     types:
       - synchronize
@@ -956,6 +956,34 @@ jobs:
           docker rm -f "$(docker ps -a -q)" ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
 ############################################################################################
+##################################### Docker images  #######################################
+############################################################################################
+  DockerServerImages:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # otherwise we will have no version info
+      - name: Check docker clickhouse/clickhouse-server building
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_server.py --release-type head --no-push
+          python3 docker_server.py --release-type head --no-push --no-ubuntu \
+            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:
@@ -3016,6 +3044,7 @@ jobs:
     needs:
       - StyleCheck
       - DockerHubPush
+      - DockerServerImages
       - CheckLabels
       - BuilderReport
       - FastTest

.github/workflows/release.yml

@@ -32,7 +32,32 @@ jobs:
       uses: svenstaro/upload-release-action@v2
       with:
         repo_token: ${{ secrets.GITHUB_TOKEN }}
-        file: ${{runner.temp}}/release_packages/*
+        file: ${{runner.temp}}/push_to_artifactory/*
         overwrite: true
         tag: ${{ github.ref }}
         file_glob: true
+  ############################################################################################
+  ##################################### Docker images  #######################################
+  ############################################################################################
+  DockerServerImages:
+    runs-on: [self-hosted, style-checker]
+    steps:
+    - name: Clear repository
+      run: |
+        sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+    - name: Check out repository code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0  # otherwise we will have no version info
+    - name: Check docker clickhouse/clickhouse-server building
+      run: |
+        cd "$GITHUB_WORKSPACE/tests/ci"
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}"
+        python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \
+          --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+    - name: Cleanup
+      if: always()
+      run: |
+        docker kill "$(docker ps -q)" ||:
+        docker rm -f "$(docker ps -a -q)" ||:
+        sudo rm -fr "$TEMP_PATH"

.gitignore

@@ -153,3 +153,5 @@ website/package-lock.json
 /programs/server/metadata
 /programs/server/store
 
+
+utils/local-engine/tests/testConfig.h

CMakeLists.txt

@@ -187,6 +187,9 @@ if (COMPILER_CLANG)
             set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing")
         endif()
     endif()
+
+    # set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
+    # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -fno-optimize-sibling-calls")
 endif ()
 
 # If compiler has support for -Wreserved-identifier. It is difficult to detect by clang version,
@@ -261,13 +264,16 @@ endif ()
 # Add a section with the hash of the compiled machine code for integrity checks.
 # Only for official builds, because adding a section can be time consuming (rewrite of several GB).
 # And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary)
-if (OBJCOPY_PATH AND YANDEX_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE))
-    set (USE_BINARY_HASH 1)
+if (OBJCOPY_PATH AND CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$"))
+    set (USE_BINARY_HASH 1 CACHE STRING "Calculate binary hash and store it in the separate section")
 endif ()
 
 # Allows to build stripped binary in a separate directory
-if (OBJCOPY_PATH AND READELF_PATH)
-    set(BUILD_STRIPPED_BINARIES_PREFIX "" CACHE STRING "Build stripped binaries with debug info in separate directory")
+if (OBJCOPY_PATH AND STRIP_PATH)
+    option(INSTALL_STRIPPED_BINARIES "Build stripped binaries with debug info in separate directory" OFF)
+    if (INSTALL_STRIPPED_BINARIES)
+        set(STRIPPED_BINARIES_OUTPUT "stripped" CACHE STRING "A separate directory for stripped information")
+    endif()
 endif()
 
 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
@@ -517,8 +523,10 @@ macro (add_executable target)
     if (${type} STREQUAL EXECUTABLE)
         # disabled for TSAN and gcc since libtsan.a provides overrides too
         if (TARGET clickhouse_new_delete)
-            # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+            if (NOT ${target} STREQUAL Git::Git)
+                # operator::new/delete for executables (MemoryTracker stuff)
+                target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+            endif()
         endif()
 
         # In case of static jemalloc, because zone_register() is located in zone.c and

base/glibc-compatibility/CMakeLists.txt

@@ -51,6 +51,6 @@ if (GLIBC_COMPATIBILITY)
 
     message (STATUS "Some symbols from glibc will be replaced for compatibility")
 
-elseif (YANDEX_OFFICIAL_BUILD)
+elseif (CLICKHOUSE_OFFICIAL_BUILD)
     message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.")
 endif ()

benchmark/greenplum/result_parser.py

@@ -4,11 +4,12 @@
 import sys
 import json
 
+
 def parse_block(block=[], options=[]):
 
-    #print('block is here', block)
-    #show_query = False
-    #show_query = options.show_query
+    # print('block is here', block)
+    # show_query = False
+    # show_query = options.show_query
     result = []
     query = block[0].strip()
     if len(block) > 4:
@@ -20,9 +21,9 @@ def parse_block(block=[], options=[]):
         timing2 = block[2].strip().split()[1]
         timing3 = block[3].strip().split()[1]
     if options.show_queries:
-        result.append( query )
+        result.append(query)
     if not options.show_first_timings:
-        result += [ timing1 , timing2, timing3 ]
+        result += [timing1, timing2, timing3]
     else:
         result.append(timing1)
     return result
@@ -37,86 +38,113 @@ def read_stats_file(options, fname):
 
         for line in f.readlines():
 
-            if 'SELECT' in line:
+            if "SELECT" in line:
                 if len(block) > 1:
-                    result.append( parse_block(block, options) )
-                block = [ line ]
-            elif 'Time:' in line:
-                block.append( line )
+                    result.append(parse_block(block, options))
+                block = [line]
+            elif "Time:" in line:
+                block.append(line)
 
     return result
 
 
 def compare_stats_files(options, arguments):
     result = []
     file_output = []
-    pyplot_colors = ['y', 'b', 'g', 'r']
+    pyplot_colors = ["y", "b", "g", "r"]
     for fname in arguments[1:]:
         file_output.append((read_stats_file(options, fname)))
     if len(file_output[0]) > 0:
         timings_count = len(file_output[0])
     for idx, data_set in enumerate(file_output):
         int_result = []
         for timing in data_set:
-           int_result.append(float(timing[0])) #y values
-        result.append([[x for x in range(0, len(int_result)) ], int_result,
-pyplot_colors[idx] + '^' ] )
-#        result.append([x for x in range(1, len(int_result)) ]) #x values
-#        result.append( pyplot_colors[idx] + '^' )
+            int_result.append(float(timing[0]))  # y values
+        result.append(
+            [
+                [x for x in range(0, len(int_result))],
+                int_result,
+                pyplot_colors[idx] + "^",
+            ]
+        )
+    #        result.append([x for x in range(1, len(int_result)) ]) #x values
+    #        result.append( pyplot_colors[idx] + '^' )
 
     return result
 
+
 def parse_args():
     from optparse import OptionParser
-    parser = OptionParser(usage='usage: %prog [options] [result_file_path]..')
-    parser.add_option("-q", "--show-queries", help="Show statements along with timings", action="store_true", dest="show_queries")
-    parser.add_option("-f", "--show-first-timings", help="Show only first tries timings", action="store_true", dest="show_first_timings")
-    parser.add_option("-c", "--compare-mode", help="Prepare output for pyplot comparing result files.", action="store", dest="compare_mode")
+
+    parser = OptionParser(usage="usage: %prog [options] [result_file_path]..")
+    parser.add_option(
+        "-q",
+        "--show-queries",
+        help="Show statements along with timings",
+        action="store_true",
+        dest="show_queries",
+    )
+    parser.add_option(
+        "-f",
+        "--show-first-timings",
+        help="Show only first tries timings",
+        action="store_true",
+        dest="show_first_timings",
+    )
+    parser.add_option(
+        "-c",
+        "--compare-mode",
+        help="Prepare output for pyplot comparing result files.",
+        action="store",
+        dest="compare_mode",
+    )
     (options, arguments) = parser.parse_args(sys.argv)
     if len(arguments) < 2:
         parser.print_usage()
         sys.exit(1)
-    return ( options, arguments )
+    return (options, arguments)
+
 
 def gen_pyplot_code(options, arguments):
-    result = ''
+    result = ""
     data_sets = compare_stats_files(options, arguments)
     for idx, data_set in enumerate(data_sets, start=0):
         x_values, y_values, line_style = data_set
-        result += '\nplt.plot('
-        result += '%s, %s, \'%s\'' % ( x_values, y_values, line_style )
-        result += ', label=\'%s try\')' % idx
-    print('import matplotlib.pyplot as plt')
+        result += "\nplt.plot("
+        result += "%s, %s, '%s'" % (x_values, y_values, line_style)
+        result += ", label='%s try')" % idx
+    print("import matplotlib.pyplot as plt")
     print(result)
-    print( 'plt.xlabel(\'Try number\')' )
-    print( 'plt.ylabel(\'Timing\')' )
-    print( 'plt.title(\'Benchmark query timings\')' )
-    print('plt.legend()')
-    print('plt.show()')
+    print("plt.xlabel('Try number')")
+    print("plt.ylabel('Timing')")
+    print("plt.title('Benchmark query timings')")
+    print("plt.legend()")
+    print("plt.show()")
 
 
 def gen_html_json(options, arguments):
     tuples = read_stats_file(options, arguments[1])
-    print('{')
+    print("{")
     print('"system:       GreenPlum(x2),')
-    print(('"version":      "%s",' % '4.3.9.1'))
+    print(('"version":      "%s",' % "4.3.9.1"))
     print('"data_size":    10000000,')
     print('"time":         "",')
     print('"comments":     "",')
     print('"result":')
-    print('[')
+    print("[")
     for s in tuples:
         print(s)
-    print(']')
-    print('}')
+    print("]")
+    print("}")
 
 
 def main():
-    ( options, arguments ) = parse_args()
+    (options, arguments) = parse_args()
     if len(arguments) > 2:
         gen_pyplot_code(options, arguments)
     else:
         gen_html_json(options, arguments)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main()

cmake/autogenerated_versions.txt

@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54460)
 SET(VERSION_MAJOR 22)
 SET(VERSION_MINOR 3)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 75366fc95e510b7ac76759ef670702ae5f488a51)
-SET(VERSION_DESCRIBE v22.3.1.1-testing)
-SET(VERSION_STRING 22.3.1.1)
+SET(VERSION_PATCH 4)
+SET(VERSION_GITHASH abb756d3ca261a4a2c853adb6938d2195b8cbff2)
+SET(VERSION_DESCRIBE v22.3.4.44-lts)
+SET(VERSION_STRING 22.3.4.44)
 # end of autochange