Merge remote-tracking branch 'upstream/master' into storage_v2

jiaoew1991 · Oct 23, 2023 · 5d6833b · 5d6833b
2 parents 65d25cd + 8c605ca
commit 5d6833b
Show file tree

Hide file tree

Showing 1,253 changed files with 74,652 additions and 59,574 deletions.
diff --git a/.env b/.env
@@ -1,10 +1,11 @@
 IMAGE_REPO=milvusdb
 IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
-DATE_VERSION=20230830-30ca458
-LATEST_DATE_VERSION=20230830-30ca458
+DATE_VERSION=20231011-11b5213
+LATEST_DATE_VERSION=20231011-11b5213
 GPU_DATE_VERSION=20230822-a64488a
 LATEST_GPU_DATE_VERSION=20230317-a1c7b0c
 MINIO_ADDRESS=minio:9000
 PULSAR_ADDRESS=pulsar://pulsar:6650
 ETCD_ENDPOINTS=etcd:2379
+AZURITE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -30,11 +30,12 @@ pull_request_rules:
         add:
           - dco-passed
 
-  - name: Test passed for code changed-master
+  - name: Test passed for code changed-master 
     conditions:
       - or:
           - base=sql_beta
           - base=master
+          - base~=^2\.3\.\d+$
       - 'status-success=Build and test AMD64 Ubuntu 20.04'
       - 'status-success=Code Checker AMD64 Ubuntu 20.04'
       - 'status-success=Code Checker MacOS 12'
@@ -46,7 +47,7 @@ pull_request_rules:
       label:
         add:
           - ci-passed
-  - name: Test passed for code changed -2.*.*
+  - name: Test passed for code changed -2.2.*
     conditions:
       - base~=^2(\.\d+){2}$
       - 'status-success=Code Checker AMD64 Ubuntu 20.04'
@@ -60,6 +61,7 @@ pull_request_rules:
       label:
         add:
           - ci-passed
+
   - name: Test passed for tests changed
     conditions:
       - or:
@@ -103,21 +105,23 @@ pull_request_rules:
       - or:
           - base=master
           - base=sql_beta
+          - base~=^2\.3\.\d+$
       - 'status-success=Build and test AMD64 Ubuntu 20.04'
       - 'status-success=Code Checker AMD64 Ubuntu 20.04'
       - 'status-success=Code Checker MacOS 12'
       - 'status-success=Code Checker Amazonlinux 2023'
       - 'status-success=UT for Go (20.04)'
       - or:
-          - -files~=^(?!pkg\/..*test\.go).*$
+          - -files~=^(?!pkg\/.*_test\.go).*$
           - -files~=^(?!internal\/.*_test\.go).*$
     actions:
       label:
         add:
           - ci-passed
-  - name: Test passed for go unittest code changed -2.*.*
+
+  - name: Test passed for go unittest code changed -2.2.*
     conditions:
-      - base~=^2(\.\d+){2}$
+      - base~=^2\.2\.\d+$
       - 'status-success=Code Checker AMD64 Ubuntu 20.04'
       - 'status-success=Build and test AMD64 Ubuntu 20.04'
       - 'status-success=Code Checker MacOS 12'
@@ -197,8 +201,7 @@ pull_request_rules:
 
   - name: Blocking PR if missing a related master PR or doesn't have kind/branch-feature label
     conditions:
-      - base=2.2.0
-      # - base~=^2(\.\d+){2}$
+      - base~=^2(\.\d+){2}$
       - and:
           - -body~=pr\:\ \#[0-9]{1,6}(\s+|$)
           - -body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$)
@@ -214,8 +217,7 @@ pull_request_rules:
 
   - name: Dismiss block label if related pr be added into PR
     conditions:
-      - base=2.2.0
-      # - base~=^2(\.\d+){2}$
+      - base~=^2(\.\d+){2}$
       - or:
           - body~=pr\:\ \#[0-9]{1,6}(\s+|$)
           - body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$)
@@ -243,6 +245,7 @@ pull_request_rules:
       - or:
           - base=master
           - base=sql_beta
+          - base~=^2\.3\.\d+$
       - title~=\[skip e2e\]
       - 'status-success=Code Checker AMD64 Ubuntu 20.04'
       - 'status-success=Build and test AMD64 Ubuntu 20.04'
@@ -254,9 +257,9 @@ pull_request_rules:
         add:
           - ci-passed
 
-  - name: Test passed for skip e2e - 2.*.*
+  - name: Test passed for skip e2e - 2.2.*
     conditions:
-      - base~=^2(\.\d+){2}$
+      - base~=^2\.2\.\d+$
       - title~=\[skip e2e\]
       - 'status-success=Code Checker AMD64 Ubuntu 20.04'
       - 'status-success=Build and test AMD64 Ubuntu 20.04'
@@ -268,12 +271,13 @@ pull_request_rules:
         add:
           - ci-passed
 
-  - name: Remove ci-passed label when status for code checker or ut  is not success-master
+  - name: Remove ci-passed label when status for code checker or ut is not success-master
     conditions:
       - label!=manual-pass
       - or:
           - base=master
           - base=sql_beta
+          - base~=^2\.3\.\d+$
       - files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$
       - or:
           - 'status-success!=Code Checker AMD64 Ubuntu 20.04'
@@ -284,10 +288,11 @@ pull_request_rules:
       label:
         remove:
           - ci-passed
-  - name: Remove ci-passed label when status for code checker or ut  is not success-2.*.*
+
+  - name: Remove ci-passed label when status for code checker or ut  is not success-2.2.*
     conditions:
       - label!=manual-pass
-      - base~=^2(\.\d+){2}$
+      - base~=^2\.2\.\d+$
       - files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$
       - or:
           - 'status-success!=Code Checker AMD64 Ubuntu 20.04'
@@ -298,6 +303,7 @@ pull_request_rules:
       label:
         remove:
           - ci-passed
+
   - name: Remove ci-passed label when  status for jenkins job is not success
     conditions:
       - label!=manual-pass
@@ -306,7 +312,7 @@ pull_request_rules:
           - base=sql_beta
           - base~=^2(\.\d+){2}$
       - -title~=\[skip e2e\]
-      - files~=^(?!(internal\/.*_test\.go|.*\.md)).*$
+      - files~=^(?!(.*_test\.go|.*\.md)).*$
       - 'status-success!=cpu-e2e'
     actions:
       label:
@@ -338,9 +344,9 @@ pull_request_rules:
         message: |
           @{{author}} ut workflow job failed, comment `rerun ut` can trigger the job again.
 
-  - name: Add comment when code checker or ut failed -2.*.*
+  - name: Add comment when code checker or ut failed -2.2.*
     conditions:
-      - base~=^2(\.\d+){2}$
+      - base~=^2\.2\.\d+$
       - or:
           - 'check-failure=Code Checker AMD64 Ubuntu 20.04'
           - 'check-failure=Build and test AMD64 Ubuntu 20.04'

diff --git a/.github/workflows/check-issue.yaml b/.github/workflows/check-issue.yaml
@@ -0,0 +1,48 @@
+name: Add Comment for issue
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check_issue_title:
+    name: Check issue
+    runs-on: ubuntu-latest
+    env:
+      TITLE_PASSED: "T"
+    permissions:
+      issues: write
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Check Issue
+        shell: bash
+        run: |
+          echo Issue title: ${{ github.event.issue.title }}
+          cat >> check_title.py << EOF
+          import re
+          import sys
+          check_str = sys.argv[1]
+          pattern = re.compile(r'[\u4e00-\u9fa5]')
+          match = pattern.search(check_str)
+          if match:
+              print("TITLE_PASSED=F")
+          else:
+              print("TITLE_PASSED=T")
+          EOF
+          
+          python3 check_title.py "${{ github.event.issue.title }}" >> "$GITHUB_ENV"
+          cat $GITHUB_ENV
+
+      - name: Check env
+        shell: bash
+        run: |
+          echo ${{ env.TITLE_PASSED }}
+      - name: Add comment
+        if: ${{ env.TITLE_PASSED == 'F'}}
+        uses: peter-evans/create-or-update-comment@5f728c3dae25f329afbe34ee4d08eef25569d79f
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            The title and description of this issue contains Chinese. Please use English to describe your issue.
diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml
@@ -53,7 +53,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: .docker/amd64-ubuntu20.04-go-mod
-          key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }}
+          key: ubuntu20.04-go-mod-${{ hashFiles('go.sum, */go.sum') }}
           restore-keys: ubuntu20.04-go-mod-
       - name: Cache Conan Packages
         uses: pat-s/always-upload-cache@v3
@@ -98,7 +98,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: .docker/amd64-amazonlinux2023-go-mod
-          key: amazonlinux2023-go-mod-${{ hashFiles('**/go.sum') }}
+          key: amazonlinux2023-go-mod-${{ hashFiles('go.sum, */go.sum') }}
           restore-keys: amazonlinux2023-go-mod-
       - name: Cache Conan Packages
         uses: pat-s/always-upload-cache@v3
@@ -107,7 +107,6 @@ jobs:
           key: amazonlinux2023-conan-${{ hashFiles('internal/core/conanfile.*') }}
           restore-keys: amazonlinux2023-conan-
       - name: Code Check
-        env:
-          OS_NAME: 'amazonlinux2023'
         run: |
+          sed -i 's/ubuntu20.04/amazonlinux2023/g' .env
           ./build/builder.sh /bin/bash -c "make install"
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -67,7 +67,7 @@ jobs:
       - name: 'Setup Use USE_ASAN'
         if: steps.changed-files-cpp.outputs.any_changed == 'true'
         run: |
-          echo "useasan=true" >> $GITHUB_ENV
+          echo "useasan=ON" >> $GITHUB_ENV
           echo "Setup USE_ASAN to true since cpp file(s) changed"
       - name: 'Generate CCache Hash'
         env:
@@ -123,6 +123,12 @@ jobs:
           path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan
           key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
           restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
+      - name: Start Service
+        shell: bash
+        run: |
+          docker-compose up -d azurite
+#      - name: 'Setup upterm session'
+#        uses: lhotari/action-upterm@v1
       - name: UT
         run: |
           chmod +x build/builder.sh
@@ -166,7 +172,9 @@ jobs:
       - name: Start Service
         shell: bash
         run: |
-          docker-compose up -d pulsar etcd minio
+          docker-compose up -d pulsar etcd minio azurite
+#      - name: 'Setup upterm session'
+#        uses: lhotari/action-upterm@v1
       - name: UT
         run: |
           chmod +x build/builder.sh

diff --git a/.github/workflows/update-knowhere-commit.yaml b/.github/workflows/update-knowhere-commit.yaml
@@ -39,7 +39,7 @@ jobs:
         continue-on-error: true
         shell: bash
         run: |
-          sed -i "s#( KNOWHERE_VERSION.*#( KNOWHERE_VERSION ${{ steps.get-knowhere-latest-commit.outputs.knowhere-commit }} )#g" internal/core/thirdparty/knowhere/CMakeLists.txt
+          sed -i "0,/(\ KNOWHERE_VERSION/ s#( KNOWHERE_VERSION.*#( KNOWHERE_VERSION ${{ steps.get-knowhere-latest-commit.outputs.knowhere-commit }} )#g" internal/core/thirdparty/knowhere/CMakeLists.txt
           head -n 17 internal/core/thirdparty/knowhere/CMakeLists.txt
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"

diff --git a/.gitignore b/.gitignore
@@ -5,11 +5,13 @@
 #   proxy/cmake-build-debug/
 #   a/b/c/cmake-build-debug/
 **/cmake-build-debug/*
+**/cmake-build-debug-coverage/*
 **/cmake-build-release/*
 **/cmake_build_release/*
 **/cmake_build/*
 **/CmakeFiles/*
 .cache
+coverage_report/
 
 internal/core/output/*
 internal/core/build/*