Bugfix/issue 626 native build mac os binary (#801)

.github/workflows/manual_trigger_build.yml

@@ -27,11 +27,19 @@ on: # yamllint disable-line rule:truthy
         required: true
         type: string
         default: 'manual'
+      enable_native_build_for_macOS:
+        description: 'Experimental try to build macos binary native'
+        required: false
+        default: 'false'
+        type: choice
+        options:
+          - true
+          - false
 
 jobs:
   prepare_sanitizer:
     runs-on: ubuntu-latest
-    if: github.event.inputs.build_type != 'release'
+    if: github.event.inputs.enable_native_build_for_macOS == 'false' && github.event.inputs.build_type != 'release'
     outputs:
       command: |
         cd $GITHUB_WORKSPACE
@@ -74,7 +82,7 @@ jobs:
           echo 'command: ${{ steps.set_command.outputs.command }}'
   prepare_release:
     runs-on: ubuntu-latest
-    if: github.event.inputs.build_type == 'release'
+    if:  github.event.inputs.enable_native_build_for_macOS == 'false' && github.event.inputs.build_type == 'release'
     outputs:
       command: |
         cd $GITHUB_WORKSPACE
@@ -197,3 +205,52 @@ jobs:
       GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  build_native_macOS_arm:
+    runs-on: [self-hosted, macOS, ARM64]
+    env:
+      build_directory: ${{ github.workspace }}/build
+      build_type: Release
+    if: github.event.inputs.enable_native_build_for_macOS == 'true' && github.event.inputs.arch == 'arm'
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 1
+          submodules: true
+
+      - name: Create and Configure Build
+        run: |
+          mkdir -p ${{ env.build_directory }}
+          export CC=$(brew --prefix llvm@17)/bin/clang
+          export CXX=$(brew --prefix llvm@17)/bin/clang++
+          export PATH=$(brew --prefix llvm@17)/bin:$PATH
+          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF
+
+      - name: Build with Ninja
+        run: cmake --build ${{ env.build_directory }}
+
+      - name: Strip and Rename binary
+        run: |
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+          ARCH=$(uname -m)
+          ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton
+          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton_${COMMIT_HASH}_${ARCH}
+          ls -lh $ORIGINAL_BINARY
+          /opt/homebrew/opt/llvm@17/bin/llvm-strip $ORIGINAL_BINARY
+          mv $ORIGINAL_BINARY $STRIPPED_BINARY
+          ls -lh $STRIPPED_BINARY
+
+      - name: Configure AWS credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Upload Artifact To S3
+        run: |
+          COMMIT_HASH=$(git rev-parse --short HEAD)
+          ARCH=$(uname -m)
+          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton_${COMMIT_HASH}_${ARCH}
+          aws s3 cp --no-progress $STRIPPED_BINARY s3://tp-internal/proton/native_build_macOS/

.github/workflows/release_build.yml

@@ -130,7 +130,7 @@ jobs:
     with:
       ec2-instance-type: c5.4xlarge
       ec2-image-id: ami-042a37e33a285c22b
-      submodules: 'recursive'
+      submodules: 'false'
       command: |
         cd $GITHUB_WORKSPACE
 
@@ -237,62 +237,71 @@ jobs:
       GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-  Build_Darwin_Arm64:
-    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
-    with:
-      ec2-instance-type: c6g.4xlarge
-      ec2-image-id: ami-0f3dbc4cc9994fdee
-      submodules: 'recursive'
-      run_mode: 'start' # start ec2 on demand instance
-      upload_files: proton-*-Darwin-arm64
-      command: |
-        cd $GITHUB_WORKSPACE
-
-        # git config
-        git config user.name "proton-robot"
-        git config user.email "[email protected]"
-        
-        # update version
-        ./release --version patch
-
-        # download the pre-built binary of v8 (this is only for cross-compile)
-        aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
-        tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
-        rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz
-        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
-        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
-        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
-
-        # prepare build cache
-        mkdir $GITHUB_WORKSPACE/ccache
-        echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
-
-        # compiling
-        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64
-        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
-          echo "Compiling proton Failed"
-          exit 127
-        fi
-
-        # clean ccache and build footprint
-        rm -rf $GITHUB_WORKSPACE/ccache
-        rm -rf $GITHUB_WORKSPACE/build_docker
-
-        # get proton tag
-        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
-        echo "Proton tag: $PROTON_TAG"
-        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
-
-        PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64
-        echo "Proton Binary Name: $PROTON_BINARY"
-        sha256sum $GITHUB_WORKSPACE/output/proton
 
-        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
-    secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      AWS_REGION: ${{ secrets.AWS_REGION }}
-      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  Build_Native_Darwin_arm64:
+    runs-on: [self-hosted, macOS, ARM64]
+    env:
+      build_directory: ${{ github.workspace }}/build
+      build_type: Release
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 1
+          submodules: true
+
+      # Setup build environment and configure CMake
+      - name: Create and Configure Build
+        run: |
+          mkdir -p ${{ env.build_directory }}
+          export CC=$(brew --prefix llvm@17)/bin/clang
+          export CXX=$(brew --prefix llvm@17)/bin/clang++
+          export PATH=$(brew --prefix llvm@17)/bin:$PATH
+          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF
+
+      # Compile the project using Ninja
+      - name: Build with Ninja
+        run: cmake --build ${{ env.build_directory }}
+
+      # Extract the version tag from the cmake configuration
+      - name: Determine Proton Tag
+        id: determine_tag
+        run: |
+          PROTON_TAG=$(grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//')
+          echo "::set-output name=tag_name::$PROTON_TAG"
+
+      # Strip the binary to reduce its size and rename it
+      - name: Strip and Rename binary
+        run: |
+          ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton
+          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ steps.determine_tag.outputs.tag_name }}-Darwin-arm64
+          /opt/homebrew/opt/llvm@17/bin/llvm-strip $ORIGINAL_BINARY -o $STRIPPED_BINARY
+
+      # Set up AWS credentials for S3 upload
+      - name: Configure AWS credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      # Upload the stripped binary to an AWS S3 bucket
+      - name: Upload Artifact To S3
+        run: |
+          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ steps.determine_tag.outputs.tag_name }}-Darwin-arm64
+          aws s3 cp --no-progress $STRIPPED_BINARY s3://tp-internal/proton/native_build_macOS/
+      # Sha256 for homebrew binary
+      - name: Print SHA256 Checksum
+        run: |
+          STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ steps.determine_tag.outputs.tag_name }}-Darwin-arm64
+          sha256sum $STRIPPED_BINARY
+
+      # Release the binary on GitHub Releases with automatic release notes
+      - name: Release binary
+        uses: softprops/[email protected]
+        with:
+          files: ${{ env.build_directory }}/programs/proton-${{ steps.determine_tag.outputs.tag_name }}-Darwin-arm64
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          generate_release_notes: true
+          tag_name: ${{ steps.determine_tag.outputs.tag_name }}

src/Formats/FormatSchemaFactory.h

@@ -3,6 +3,8 @@
 #include <boost/core/noncopyable.hpp>
 #include <Interpreters/Context_fwd.h>
 
+#include <mutex>
+
 namespace DB
 {
 /// Manages format schemas.