diff --git a/.env b/.env
index 5f461fdebce86..e066cc2b2b350 100644
--- a/.env
+++ b/.env
@@ -1,10 +1,11 @@
IMAGE_REPO=milvusdb
IMAGE_ARCH=amd64
OS_NAME=ubuntu20.04
-DATE_VERSION=20230830-30ca458
-LATEST_DATE_VERSION=20230830-30ca458
+DATE_VERSION=20231011-11b5213
+LATEST_DATE_VERSION=20231011-11b5213
GPU_DATE_VERSION=20230822-a64488a
LATEST_GPU_DATE_VERSION=20230317-a1c7b0c
MINIO_ADDRESS=minio:9000
PULSAR_ADDRESS=pulsar://pulsar:6650
ETCD_ENDPOINTS=etcd:2379
+AZURITE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
\ No newline at end of file
diff --git a/.github/mergify.yml b/.github/mergify.yml
index ff4a56afef4ac..6b6911ec92e3b 100644
--- a/.github/mergify.yml
+++ b/.github/mergify.yml
@@ -30,11 +30,12 @@ pull_request_rules:
add:
- dco-passed
- - name: Test passed for code changed-master
+ - name: Test passed for code changed-master
conditions:
- or:
- base=sql_beta
- base=master
+ - base~=^2\.3\.\d+$
- 'status-success=Build and test AMD64 Ubuntu 20.04'
- 'status-success=Code Checker AMD64 Ubuntu 20.04'
- 'status-success=Code Checker MacOS 12'
@@ -46,7 +47,7 @@ pull_request_rules:
label:
add:
- ci-passed
- - name: Test passed for code changed -2.*.*
+ - name: Test passed for code changed -2.2.*
conditions:
- base~=^2(\.\d+){2}$
- 'status-success=Code Checker AMD64 Ubuntu 20.04'
@@ -60,6 +61,7 @@ pull_request_rules:
label:
add:
- ci-passed
+
- name: Test passed for tests changed
conditions:
- or:
@@ -103,21 +105,23 @@ pull_request_rules:
- or:
- base=master
- base=sql_beta
+ - base~=^2\.3\.\d+$
- 'status-success=Build and test AMD64 Ubuntu 20.04'
- 'status-success=Code Checker AMD64 Ubuntu 20.04'
- 'status-success=Code Checker MacOS 12'
- 'status-success=Code Checker Amazonlinux 2023'
- 'status-success=UT for Go (20.04)'
- or:
- - -files~=^(?!pkg\/..*test\.go).*$
+ - -files~=^(?!pkg\/.*_test\.go).*$
- -files~=^(?!internal\/.*_test\.go).*$
actions:
label:
add:
- ci-passed
- - name: Test passed for go unittest code changed -2.*.*
+
+ - name: Test passed for go unittest code changed -2.2.*
conditions:
- - base~=^2(\.\d+){2}$
+ - base~=^2\.2\.\d+$
- 'status-success=Code Checker AMD64 Ubuntu 20.04'
- 'status-success=Build and test AMD64 Ubuntu 20.04'
- 'status-success=Code Checker MacOS 12'
@@ -197,8 +201,7 @@ pull_request_rules:
- name: Blocking PR if missing a related master PR or doesn't have kind/branch-feature label
conditions:
- - base=2.2.0
- # - base~=^2(\.\d+){2}$
+ - base~=^2(\.\d+){2}$
- and:
- -body~=pr\:\ \#[0-9]{1,6}(\s+|$)
- -body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$)
@@ -214,8 +217,7 @@ pull_request_rules:
- name: Dismiss block label if related pr be added into PR
conditions:
- - base=2.2.0
- # - base~=^2(\.\d+){2}$
+ - base~=^2(\.\d+){2}$
- or:
- body~=pr\:\ \#[0-9]{1,6}(\s+|$)
- body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$)
@@ -243,6 +245,7 @@ pull_request_rules:
- or:
- base=master
- base=sql_beta
+ - base~=^2\.3\.\d+$
- title~=\[skip e2e\]
- 'status-success=Code Checker AMD64 Ubuntu 20.04'
- 'status-success=Build and test AMD64 Ubuntu 20.04'
@@ -254,9 +257,9 @@ pull_request_rules:
add:
- ci-passed
- - name: Test passed for skip e2e - 2.*.*
+ - name: Test passed for skip e2e - 2.2.*
conditions:
- - base~=^2(\.\d+){2}$
+ - base~=^2\.2\.\d+$
- title~=\[skip e2e\]
- 'status-success=Code Checker AMD64 Ubuntu 20.04'
- 'status-success=Build and test AMD64 Ubuntu 20.04'
@@ -268,12 +271,13 @@ pull_request_rules:
add:
- ci-passed
- - name: Remove ci-passed label when status for code checker or ut is not success-master
+ - name: Remove ci-passed label when status for code checker or ut is not success-master
conditions:
- label!=manual-pass
- or:
- base=master
- base=sql_beta
+ - base~=^2\.3\.\d+$
- files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$
- or:
- 'status-success!=Code Checker AMD64 Ubuntu 20.04'
@@ -284,10 +288,11 @@ pull_request_rules:
label:
remove:
- ci-passed
- - name: Remove ci-passed label when status for code checker or ut is not success-2.*.*
+
+ - name: Remove ci-passed label when status for code checker or ut is not success-2.2.*
conditions:
- label!=manual-pass
- - base~=^2(\.\d+){2}$
+ - base~=^2\.2\.\d+$
- files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$
- or:
- 'status-success!=Code Checker AMD64 Ubuntu 20.04'
@@ -298,6 +303,7 @@ pull_request_rules:
label:
remove:
- ci-passed
+
- name: Remove ci-passed label when status for jenkins job is not success
conditions:
- label!=manual-pass
@@ -306,7 +312,7 @@ pull_request_rules:
- base=sql_beta
- base~=^2(\.\d+){2}$
- -title~=\[skip e2e\]
- - files~=^(?!(internal\/.*_test\.go|.*\.md)).*$
+ - files~=^(?!(.*_test\.go|.*\.md)).*$
- 'status-success!=cpu-e2e'
actions:
label:
@@ -338,9 +344,9 @@ pull_request_rules:
message: |
@{{author}} ut workflow job failed, comment `rerun ut` can trigger the job again.
- - name: Add comment when code checker or ut failed -2.*.*
+ - name: Add comment when code checker or ut failed -2.2.*
conditions:
- - base~=^2(\.\d+){2}$
+ - base~=^2\.2\.\d+$
- or:
- 'check-failure=Code Checker AMD64 Ubuntu 20.04'
- 'check-failure=Build and test AMD64 Ubuntu 20.04'
diff --git a/.github/workflows/check-issue.yaml b/.github/workflows/check-issue.yaml
new file mode 100644
index 0000000000000..952c07641b91c
--- /dev/null
+++ b/.github/workflows/check-issue.yaml
@@ -0,0 +1,48 @@
+name: Add Comment for issue
+
+on:
+ issues:
+ types: [opened]
+
+jobs:
+ check_issue_title:
+ name: Check issue
+ runs-on: ubuntu-latest
+ env:
+ TITLE_PASSED: "T"
+ permissions:
+ issues: write
+ timeout-minutes: 20
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ - name: Check Issue
+ shell: bash
+ run: |
+ echo Issue title: ${{ github.event.issue.title }}
+ cat >> check_title.py << EOF
+ import re
+ import sys
+ check_str = sys.argv[1]
+ pattern = re.compile(r'[\u4e00-\u9fa5]')
+ match = pattern.search(check_str)
+ if match:
+ print("TITLE_PASSED=F")
+ else:
+ print("TITLE_PASSED=T")
+ EOF
+
+ python3 check_title.py "${{ github.event.issue.title }}" >> "$GITHUB_ENV"
+ cat $GITHUB_ENV
+
+ - name: Check env
+ shell: bash
+ run: |
+ echo ${{ env.TITLE_PASSED }}
+ - name: Add comment
+ if: ${{ env.TITLE_PASSED == 'F'}}
+ uses: peter-evans/create-or-update-comment@5f728c3dae25f329afbe34ee4d08eef25569d79f
+ with:
+ issue-number: ${{ github.event.issue.number }}
+ body: |
+ The title and description of this issue contains Chinese. Please use English to describe your issue.
\ No newline at end of file
diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml
index c801ad97bb957..d4f4416745f51 100644
--- a/.github/workflows/code-checker.yaml
+++ b/.github/workflows/code-checker.yaml
@@ -53,7 +53,7 @@ jobs:
uses: actions/cache@v3
with:
path: .docker/amd64-ubuntu20.04-go-mod
- key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }}
+ key: ubuntu20.04-go-mod-${{ hashFiles('go.sum, */go.sum') }}
restore-keys: ubuntu20.04-go-mod-
- name: Cache Conan Packages
uses: pat-s/always-upload-cache@v3
@@ -98,7 +98,7 @@ jobs:
uses: actions/cache@v3
with:
path: .docker/amd64-amazonlinux2023-go-mod
- key: amazonlinux2023-go-mod-${{ hashFiles('**/go.sum') }}
+ key: amazonlinux2023-go-mod-${{ hashFiles('go.sum, */go.sum') }}
restore-keys: amazonlinux2023-go-mod-
- name: Cache Conan Packages
uses: pat-s/always-upload-cache@v3
@@ -107,7 +107,6 @@ jobs:
key: amazonlinux2023-conan-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: amazonlinux2023-conan-
- name: Code Check
- env:
- OS_NAME: 'amazonlinux2023'
run: |
+ sed -i 's/ubuntu20.04/amazonlinux2023/g' .env
./build/builder.sh /bin/bash -c "make install"
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index 79509edd62d0e..590e0b1cf17db 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -67,7 +67,7 @@ jobs:
- name: 'Setup Use USE_ASAN'
if: steps.changed-files-cpp.outputs.any_changed == 'true'
run: |
- echo "useasan=true" >> $GITHUB_ENV
+ echo "useasan=ON" >> $GITHUB_ENV
echo "Setup USE_ASAN to true since cpp file(s) changed"
- name: 'Generate CCache Hash'
env:
@@ -123,6 +123,12 @@ jobs:
path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan
key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
+ - name: Start Service
+ shell: bash
+ run: |
+ docker-compose up -d azurite
+# - name: 'Setup upterm session'
+# uses: lhotari/action-upterm@v1
- name: UT
run: |
chmod +x build/builder.sh
@@ -166,7 +172,9 @@ jobs:
- name: Start Service
shell: bash
run: |
- docker-compose up -d pulsar etcd minio
+ docker-compose up -d pulsar etcd minio azurite
+# - name: 'Setup upterm session'
+# uses: lhotari/action-upterm@v1
- name: UT
run: |
chmod +x build/builder.sh
diff --git a/.github/workflows/update-knowhere-commit.yaml b/.github/workflows/update-knowhere-commit.yaml
index 6739c175bb2de..051f3b8d49290 100644
--- a/.github/workflows/update-knowhere-commit.yaml
+++ b/.github/workflows/update-knowhere-commit.yaml
@@ -39,7 +39,7 @@ jobs:
continue-on-error: true
shell: bash
run: |
- sed -i "s#( KNOWHERE_VERSION.*#( KNOWHERE_VERSION ${{ steps.get-knowhere-latest-commit.outputs.knowhere-commit }} )#g" internal/core/thirdparty/knowhere/CMakeLists.txt
+ sed -i "0,/(\ KNOWHERE_VERSION/ s#( KNOWHERE_VERSION.*#( KNOWHERE_VERSION ${{ steps.get-knowhere-latest-commit.outputs.knowhere-commit }} )#g" internal/core/thirdparty/knowhere/CMakeLists.txt
head -n 17 internal/core/thirdparty/knowhere/CMakeLists.txt
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
diff --git a/.gitignore b/.gitignore
index 99a8e5736aa59..3becfcd2d0d55 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,11 +5,13 @@
# proxy/cmake-build-debug/
# a/b/c/cmake-build-debug/
**/cmake-build-debug/*
+**/cmake-build-debug-coverage/*
**/cmake-build-release/*
**/cmake_build_release/*
**/cmake_build/*
**/CmakeFiles/*
.cache
+coverage_report/
internal/core/output/*
internal/core/build/*
diff --git a/.golangci.yml b/.golangci.yml
index cca8c7a7b4024..3d6dd38ad6296 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -1,5 +1,5 @@
run:
- go: '1.18'
+ go: "1.18"
skip-dirs:
- build
- configs
@@ -7,29 +7,80 @@ run:
- docs
- scripts
- internal/core
+ - cmake_build
linters:
disable-all: true
enable:
- - typecheck
- - goimports
- - misspell
+ - gosimple
- govet
- ineffassign
- - gosimple
+ - staticcheck
+ - decorder
+ - depguard
+ - gofmt
+ - goimports
- gosec
- revive
- - durationcheck
- unconvert
+ - misspell
+ - typecheck
+ - durationcheck
- forbidigo
- - depguard
- # - gocritic
+ - gci
+ - whitespace
+ - gofumpt
+ - gocritic
linters-settings:
+ gci:
+ sections:
+ - standard
+ - default
+ - prefix(github.com/milvus-io)
+ custom-order: true
+ gofumpt:
+ lang-version: "1.18"
+ module-path: github.com/milvus-io
+ goimports:
+ local-prefixes: github.com/milvus-io
revive:
rules:
- name: unused-parameter
disabled: true
+ - name: var-naming
+ severity: warning
+ disabled: false
+ arguments:
+ - ["ID"] # Allow list
+ - name: context-as-argument
+ severity: warning
+ disabled: false
+ arguments:
+ - allowTypesBefore: "*testing.T"
+ - name: datarace
+ severity: warning
+ disabled: false
+ - name: duplicated-imports
+ severity: warning
+ disabled: false
+ - name: waitgroup-by-value
+ severity: warning
+ disabled: false
+ - name: indent-error-flow
+ severity: warning
+ disabled: false
+ arguments:
+ - "preserveScope"
+ - name: range-val-in-closure
+ severity: warning
+ disabled: false
+ - name: range-val-address
+ severity: warning
+ disabled: false
+ - name: string-of-int
+ severity: warning
+ disabled: false
misspell:
locale: US
gocritic:
@@ -38,28 +89,40 @@ linters-settings:
settings:
ruleguard:
failOnError: true
- rules: 'rules.go'
+ rules: "rules.go"
depguard:
rules:
main:
deny:
- - pkg: 'errors'
- desc: not allowd, use github.com/cockroachdb/errors
- - pkg: 'github.com/pkg/errors'
- desc: not allowd, use github.com/cockroachdb/errors
- - pkg: 'github.com/pingcap/errors'
- desc: not allowd, use github.com/cockroachdb/errors
- - pkg: 'golang.org/x/xerrors'
- desc: not allowd, use github.com/cockroachdb/errors
- - pkg: 'github.com/go-errors/errors'
- desc: not allowd, use github.com/cockroachdb/errors
+ - pkg: "errors"
+ desc: not allowed, use github.com/cockroachdb/errors
+ - pkg: "github.com/pkg/errors"
+ desc: not allowed, use github.com/cockroachdb/errors
+ - pkg: "github.com/pingcap/errors"
+ desc: not allowed, use github.com/cockroachdb/errors
+ - pkg: "golang.org/x/xerrors"
+ desc: not allowed, use github.com/cockroachdb/errors
+ - pkg: "github.com/go-errors/errors"
+ desc: not allowed, use github.com/cockroachdb/errors
+ - pkg: "io/ioutil"
+ desc: ioutil is deprecated after 1.16, 1.17, use os and io package instead
forbidigo:
forbid:
- '^time\.Tick$'
+ - 'return merr\.Err[a-zA-Z]+'
+ - 'merr\.Wrap\w+\(\)\.Error\(\)'
+ - '\.(ErrorCode|Reason) = '
+ - 'Reason:\s+\w+\.Error\(\)'
+ - 'errors.New\((.+)\.GetReason\(\)\)'
+ - 'commonpb\.Status\{[\s\n]*ErrorCode:[\s\n]*.+[\s\S\n]*?\}'
#- 'fmt\.Print.*' WIP
issues:
exclude-use-default: false
+ exclude-rules:
+ - path: .+_test\.go
+ linters:
+ - forbidigo
exclude:
- should have a package comment
- should have comment
@@ -81,10 +144,12 @@ issues:
- G402
# Use of weak random number generator math/rand
- G404
+ # Unused parameters
+ - SA1019
+ # defer return errors
+ - SA5001
+
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-issues-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
-
-service:
- golangci-lint-version: 1.27.0 # use the fixed version to not introduce new linters unexpectedly
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c74859be45915..4e6b15096ee19 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -16,4 +16,4 @@ repos:
description: Detect secrets in your data.
entry: bash -c 'trufflehog git file://. --max-depth 1 --since-commit HEAD --only-verified --fail'
language: system
- stages: ["commit", "push"]
+ stages: ["commit"]
diff --git a/Makefile b/Makefile
index 5ded97d8a72d5..f643f02edef59 100644
--- a/Makefile
+++ b/Makefile
@@ -19,11 +19,17 @@ INSTALL_PATH := $(PWD)/bin
LIBRARY_PATH := $(PWD)/lib
OS := $(shell uname -s)
mode = Release
-disk_index = OFF
-useasan = false
-ifeq (${USE_ASAN}, true)
-useasan = true
+
+use_disk_index = OFF
+ifdef disk_index
+ use_disk_index = ${disk_index}
+endif
+
+use_asan = OFF
+ifdef USE_ASAN
+ use_asan =${USE_ASAN}
endif
+
use_dynamic_simd = OFF
ifdef USE_DYNAMIC_SIMD
use_dynamic_simd = ${USE_DYNAMIC_SIMD}
@@ -36,9 +42,23 @@ INSTALL_GOLANGCI_LINT := $(findstring $(GOLANGCI_LINT_VERSION), $(GOLANGCI_LINT_
MOCKERY_VERSION := 2.32.4
MOCKERY_OUTPUT := $(shell $(INSTALL_PATH)/mockery --version 2>/dev/null)
INSTALL_MOCKERY := $(findstring $(MOCKERY_VERSION),$(MOCKERY_OUTPUT))
+# gci
+GCI_VERSION := 0.11.2
+GCI_OUTPUT := $(shell $(INSTALL_PATH)/gci --version 2>/dev/null)
+INSTALL_GCI := $(findstring $(GCI_VERSION),$(GCI_OUTPUT))
+# gofumpt
+GOFUMPT_VERSION := 0.5.0
+GOFUMPT_OUTPUT := $(shell $(INSTALL_PATH)/gofumpt --version 2>/dev/null)
+INSTALL_GOFUMPT := $(findstring $(GOFUMPT_VERSION),$(GOFUMPT_OUTPUT))
+
+index_engine = knowhere
export GIT_BRANCH=master
+ifeq (${ENABLE_AZURE}, false)
+ AZURE_OPTION := -Z
+endif
+
milvus: build-cpp print-build-info
@echo "Building Milvus ..."
@source $(PWD)/scripts/setenv.sh && \
@@ -68,7 +88,7 @@ getdeps:
echo "Installing mockery v$(MOCKERY_VERSION) to ./bin/" && GOBIN=$(INSTALL_PATH) go install github.com/vektra/mockery/v2@v$(MOCKERY_VERSION); \
else \
echo "Mockery v$(MOCKERY_VERSION) already installed"; \
- fi
+ fi
tools/bin/revive: tools/check/go.mod
cd tools/check; \
@@ -88,21 +108,39 @@ else
@GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh internal/
@GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh tests/integration/
@GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh tests/go/
- @GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh pkg/
+ @GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh pkg/
endif
-lint: tools/bin/revive
- @echo "Running $@ check"
- @tools/bin/revive -formatter friendly -config tools/check/revive.toml ./...
+lint-fix: getdeps
+ @mkdir -p $(INSTALL_PATH)
+ @if [ -z "$(INSTALL_GCI)" ]; then \
+ echo "Installing gci v$(GCI_VERSION) to ./bin/" && GOBIN=$(INSTALL_PATH) go install github.com/daixiang0/gci@v$(GCI_VERSION); \
+ else \
+ echo "gci v$(GCI_VERSION) already installed"; \
+ fi
+ @if [ -z "$(INSTALL_GOFUMPT)" ]; then \
+ echo "Installing gofumpt v$(GOFUMPT_VERSION) to ./bin/" && GOBIN=$(INSTALL_PATH) go install mvdan.cc/gofumpt@v$(GOFUMPT_VERSION); \
+ else \
+ echo "gofumpt v$(GOFUMPT_VERSION) already installed"; \
+ fi
+ @echo "Running gofumpt fix"
+ @$(INSTALL_PATH)/gofumpt -l -w internal/
+ @$(INSTALL_PATH)/gofumpt -l -w cmd/
+ @$(INSTALL_PATH)/gofumpt -l -w pkg/
+ @$(INSTALL_PATH)/gofumpt -l -w tests/integration/
+ @echo "Running gci fix"
+ @$(INSTALL_PATH)/gci write cmd/ --skip-generated -s standard -s default -s "prefix(github.com/milvus-io)" --custom-order
+ @$(INSTALL_PATH)/gci write internal/ --skip-generated -s standard -s default -s "prefix(github.com/milvus-io)" --custom-order
+ @$(INSTALL_PATH)/gci write pkg/ --skip-generated -s standard -s default -s "prefix(github.com/milvus-io)" --custom-order
+ @$(INSTALL_PATH)/gci write tests/ --skip-generated -s standard -s default -s "prefix(github.com/milvus-io)" --custom-order
+ @echo "Running golangci-lint auto-fix"
+ @source $(PWD)/scripts/setenv.sh && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --fix --timeout=30m --config $(PWD)/.golangci.yml; cd pkg && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --fix --timeout=30m --config $(PWD)/.golangci.yml
#TODO: Check code specifications by golangci-lint
static-check: getdeps
@echo "Running $@ check"
- @GO111MODULE=on $(INSTALL_PATH)/golangci-lint cache clean
- @source $(PWD)/scripts/setenv.sh && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/...
- @source $(PWD)/scripts/setenv.sh && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config ./.golangci.yml ./cmd/...
- @source $(PWD)/scripts/setenv.sh && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config ./.golangci.yml ./tests/integration/...
- @source $(PWD)/scripts/setenv.sh && cd pkg && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config ../.golangci.yml ./...
+ @source $(PWD)/scripts/setenv.sh && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config $(PWD)/.golangci.yml
+ @source $(PWD)/scripts/setenv.sh && cd pkg && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config $(PWD)/.golangci.yml
verifiers: build-cpp getdeps cppcheck fmt static-check
@@ -169,19 +207,19 @@ generated-proto: download-milvus-proto build-3rdparty
build-cpp: generated-proto
@echo "Building Milvus cpp library ..."
- @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd})
+ @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine})
build-cpp-gpu: generated-proto
- @echo "Building Milvus cpp gpu library ..."
- @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd})
+ @echo "Building Milvus cpp gpu library ... "
+ @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine})
build-cpp-with-unittest: generated-proto
- @echo "Building Milvus cpp library with unittest ..."
- @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd})
+ @echo "Building Milvus cpp library with unittest ... "
+ @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine})
build-cpp-with-coverage: generated-proto
@echo "Building Milvus cpp library with coverage and unittest ..."
- @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd})
+ @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -c -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine})
check-proto-product: generated-proto
@(env bash $(PWD)/scripts/check_proto_product.sh)
@@ -296,13 +334,10 @@ gpu-install: milvus-gpu
clean:
@echo "Cleaning up all the generated files"
- @find . -name '*.test' | xargs rm -fv
- @find . -name '*~' | xargs rm -fv
@rm -rf bin/
@rm -rf lib/
@rm -rf $(GOPATH)/bin/milvus
@rm -rf cmake_build
- @rm -rf cwrapper_build
@rm -rf internal/core/output
milvus-tools: print-build-info
@@ -336,9 +371,9 @@ generate-mockery-types: getdeps
# Proxy
$(INSTALL_PATH)/mockery --name=ProxyComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_proxy.go --with-expecter --structname=MockProxy
# QueryCoord
- $(INSTALL_PATH)/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord
+ $(INSTALL_PATH)/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord
# QueryNode
- $(INSTALL_PATH)/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNode
+ $(INSTALL_PATH)/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNode
# DataCoord
$(INSTALL_PATH)/mockery --name=DataCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_datacoord.go --with-expecter --structname=MockDataCoord
# DataNode
@@ -346,6 +381,15 @@ generate-mockery-types: getdeps
# IndexNode
$(INSTALL_PATH)/mockery --name=IndexNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_indexnode.go --with-expecter --structname=MockIndexNode
+ # Clients
+ $(INSTALL_PATH)/mockery --name=RootCoordClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_rootcoord_client.go --with-expecter --structname=MockRootCoordClient
+ $(INSTALL_PATH)/mockery --name=QueryCoordClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord_client.go --with-expecter --structname=MockQueryCoordClient
+ $(INSTALL_PATH)/mockery --name=DataCoordClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_datacoord_client.go --with-expecter --structname=MockDataCoordClient
+ $(INSTALL_PATH)/mockery --name=QueryNodeClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode_client.go --with-expecter --structname=MockQueryNodeClient
+ $(INSTALL_PATH)/mockery --name=DataNodeClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_datanode_client.go --with-expecter --structname=MockDataNodeClient
+ $(INSTALL_PATH)/mockery --name=IndexNodeClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_indexnode_client.go --with-expecter --structname=MockIndexNodeClient
+ $(INSTALL_PATH)/mockery --name=ProxyClient --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_proxy_client.go --with-expecter --structname=MockProxyClient
+
generate-mockery-rootcoord: getdeps
$(INSTALL_PATH)/mockery --name=IMetaTable --dir=$(PWD)/internal/rootcoord --output=$(PWD)/internal/rootcoord/mocks --filename=meta_table.go --with-expecter --outpkg=mockrootcoord
$(INSTALL_PATH)/mockery --name=GarbageCollector --dir=$(PWD)/internal/rootcoord --output=$(PWD)/internal/rootcoord/mocks --filename=garbage_collector.go --with-expecter --outpkg=mockrootcoord
@@ -356,6 +400,7 @@ generate-mockery-proxy: getdeps
$(INSTALL_PATH)/mockery --name=LBPolicy --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_policy.go --structname=MockLBPolicy --with-expecter --outpkg=proxy --inpackage
$(INSTALL_PATH)/mockery --name=LBBalancer --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_balancer.go --structname=MockLBBalancer --with-expecter --outpkg=proxy --inpackage
$(INSTALL_PATH)/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy --inpackage
+ $(INSTALL_PATH)/mockery --name=channelsMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_channels_manager.go --structname=MockChannelsMgr --with-expecter --outpkg=proxy --inpackage
generate-mockery-querycoord: getdeps
$(INSTALL_PATH)/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer
@@ -367,6 +412,7 @@ generate-mockery-querycoord: getdeps
generate-mockery-querynode: getdeps build-cpp
@source $(PWD)/scripts/setenv.sh # setup PKG_CONFIG_PATH
+ $(INSTALL_PATH)/mockery --name=QueryHook --dir=$(PWD)/internal/querynodev2/optimizers --output=$(PWD)/internal/querynodev2/optimizers --filename=mock_query_hook.go --with-expecter --outpkg=optimizers --structname=MockQueryHook --inpackage
$(INSTALL_PATH)/mockery --name=Manager --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_manager.go --with-expecter --outpkg=cluster --structname=MockManager --inpackage
$(INSTALL_PATH)/mockery --name=SegmentManager --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_segment_manager.go --with-expecter --outpkg=segments --structname=MockSegmentManager --inpackage
$(INSTALL_PATH)/mockery --name=CollectionManager --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_collection_manager.go --with-expecter --outpkg=segments --structname=MockCollectionManager --inpackage
@@ -382,11 +428,12 @@ generate-mockery-datacoord: getdeps
generate-mockery-datanode: getdeps
$(INSTALL_PATH)/mockery --name=Allocator --dir=$(PWD)/internal/datanode/allocator --output=$(PWD)/internal/datanode/allocator --filename=mock_allocator.go --with-expecter --structname=MockAllocator --outpkg=allocator --inpackage
+ $(INSTALL_PATH)/mockery --name=Broker --dir=$(PWD)/internal/datanode/broker --output=$(PWD)/internal/datanode/broker/ --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=broker --inpackage
generate-mockery-metastore: getdeps
- $(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks
- $(INSTALL_PATH)/mockery --name=DataCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_datacoord_catalog.go --with-expecter --structname=DataCoordCatalog --outpkg=mocks
- $(INSTALL_PATH)/mockery --name=QueryCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_querycoord_catalog.go --with-expecter --structname=QueryCoordCatalog --outpkg=mocks
+ $(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks
+ $(INSTALL_PATH)/mockery --name=DataCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_datacoord_catalog.go --with-expecter --structname=DataCoordCatalog --outpkg=mocks
+ $(INSTALL_PATH)/mockery --name=QueryCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_querycoord_catalog.go --with-expecter --structname=QueryCoordCatalog --outpkg=mocks
generate-mockery-utils: getdeps
# dependency.Factory
@@ -399,9 +446,10 @@ generate-mockery-kv: getdeps
$(INSTALL_PATH)/mockery --name=MetaKv --dir=$(PWD)/internal/kv --output=$(PWD)/internal/kv/mocks --filename=meta_kv.go --with-expecter
$(INSTALL_PATH)/mockery --name=WatchKV --dir=$(PWD)/internal/kv --output=$(PWD)/internal/kv/mocks --filename=watch_kv.go --with-expecter
$(INSTALL_PATH)/mockery --name=SnapShotKV --dir=$(PWD)/internal/kv --output=$(PWD)/internal/kv/mocks --filename=snapshot_kv.go --with-expecter
+ $(INSTALL_PATH)/mockery --name=Predicate --dir=$(PWD)/internal/kv/predicates --output=$(PWD)/internal/kv/predicates --filename=mock_predicate.go --with-expecter --inpackage
generate-mockery-pkg:
$(MAKE) -C pkg generate-mockery
generate-mockery: generate-mockery-types generate-mockery-kv generate-mockery-rootcoord generate-mockery-proxy generate-mockery-querycoord generate-mockery-querynode generate-mockery-datacoord generate-mockery-pkg
-
+
diff --git a/README.md b/README.md
index 8eb5351170e63..707e0b419e5c8 100644
--- a/README.md
+++ b/README.md
@@ -169,7 +169,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
### All contributors
-
+
@@ -188,6 +188,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -262,6 +263,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -276,6 +278,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -319,6 +322,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -399,6 +403,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -409,6 +414,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -447,6 +453,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -472,6 +479,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
@@ -481,7 +489,9 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
+
+
diff --git a/README_CN.md b/README_CN.md
index 11785188531b1..3cf9a188af826 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -154,7 +154,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
### All contributors
-
+
@@ -173,6 +173,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -247,6 +248,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -261,6 +263,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -304,6 +307,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -384,6 +388,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -394,6 +399,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -432,6 +438,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -457,6 +464,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
@@ -466,7 +474,9 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
+
+
diff --git a/build/builder.sh b/build/builder.sh
index 9802764c7aa0e..5e58b7f07f088 100755
--- a/build/builder.sh
+++ b/build/builder.sh
@@ -9,8 +9,6 @@ if [[ -f "$toplevel/.env" ]]; then
export $(cat $toplevel/.env | xargs)
fi
-export OS_NAME="${OS_NAME:-ubuntu20.04}"
-
pushd "${toplevel}"
if [[ "${1-}" == "pull" ]]; then
@@ -27,8 +25,6 @@ PLATFORM_ARCH="${PLATFORM_ARCH:-${IMAGE_ARCH}}"
export IMAGE_ARCH=${PLATFORM_ARCH}
-echo ${IMAGE_ARCH}
-
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache"
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-go-mod"
mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions"
diff --git a/build/docker/builder/cpu/amazonlinux2023/Dockerfile b/build/docker/builder/cpu/amazonlinux2023/Dockerfile
index d7d0aba5ebf0b..c9f3f15cecdb2 100644
--- a/build/docker/builder/cpu/amazonlinux2023/Dockerfile
+++ b/build/docker/builder/cpu/amazonlinux2023/Dockerfile
@@ -13,31 +13,22 @@ FROM amazonlinux:2023
ARG TARGETARCH
-RUN yum install -y wget g++ gcc gdb libstdc++-static git make zip unzip tar which \
+RUN yum install -y wget g++ gcc gdb libatomic libstdc++-static git make zip unzip tar which \
autoconf automake golang python3 python3-pip perl-FindBin texinfo \
- pkg-config libuuid-devel libaio && \
+ pkg-config libuuid-devel libaio perl-IPC-Cmd && \
rm -rf /var/cache/yum/*
RUN pip3 install conan==1.58.0
RUN echo "target arch $TARGETARCH"
-RUN if [ "$TARGETARCH" = "amd64" ]; then CMAKE_SUFFIX=x86_64; else CMAKE_SUFFIX=aarch64; fi &&\
- wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-$CMAKE_SUFFIX.tar.gz" | tar --strip-components=1 -xz -C /usr/local
+RUN wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
-RUN mkdir /tmp/ccache && wget -qO- https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2.tar.gz | tar --strip-components=1 -xz -C /tmp/ccache &&\
- cd /tmp/ccache && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. && make && make install &&\
+RUN mkdir /tmp/ccache && cd /tmp/ccache &&\
+ wget https://dl.fedoraproject.org/pub/epel/9/Everything/`uname -m`/Packages/h/hiredis-1.0.2-1.el9.`uname -m`.rpm &&\
+ wget https://dl.fedoraproject.org/pub/epel/9/Everything/`uname -m`/Packages/c/ccache-4.5.1-2.el9.`uname -m`.rpm &&\
+ rpm -i hiredis-1.0.2-1.el9.`uname -m`.rpm ccache-4.5.1-2.el9.`uname -m`.rpm &&\
rm -rf /tmp/ccache
-# https://github.com/golang/go/issues/22040 Due to this issue, ld.gold cannot be used in the ARM environment, so the official golang package cannot be used. You need to use the golang package that comes with yum.
-# Install Go
-# ENV GOPATH /go
-# ENV GOROOT /usr/local/go
-# ENV GO111MODULE on
-# ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
-# RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.20.7.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
-# mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
-# go clean --modcache && \
-# chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
diff --git a/build/docker/builder/cpu/ubuntu20.04/Dockerfile b/build/docker/builder/cpu/ubuntu20.04/Dockerfile
index 444a971132518..3e95294dfd5f2 100644
--- a/build/docker/builder/cpu/ubuntu20.04/Dockerfile
+++ b/build/docker/builder/cpu/ubuntu20.04/Dockerfile
@@ -13,6 +13,8 @@ FROM ubuntu:focal-20220426
ARG TARGETARCH
+RUN if [ "$TARGETARCH" = "arm64" ]; then apt-get update && apt-get install -y ninja-build; fi
+
RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \
g++ gcc gdb gdbserver git make ccache libssl-dev zlib1g-dev zip unzip \
clang-format-10 clang-tidy-10 lcov libtool m4 autoconf automake python3 python3-pip \
@@ -23,8 +25,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
RUN pip3 install conan==1.58.0
RUN echo "target arch $TARGETARCH"
-RUN if [ "$TARGETARCH" = "amd64" ]; then CMAKE_SUFFIX=x86_64; else CMAKE_SUFFIX=aarch64; fi &&\
- wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-$CMAKE_SUFFIX.tar.gz" | tar --strip-components=1 -xz -C /usr/local
+RUN wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local
# Install Go
ENV GOPATH /go
diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy
index f28f2f7f48bf0..37d7f656cba17 100644
--- a/ci/jenkins/PR.groovy
+++ b/ci/jenkins/PR.groovy
@@ -3,7 +3,7 @@
int total_timeout_minutes = 60 * 5
int e2e_timeout_seconds = 120 * 60
def imageTag=''
-int case_timeout_seconds = 10 * 60
+int case_timeout_seconds = 20 * 60
def chart_version='4.0.6'
pipeline {
options {
diff --git a/ci/jenkins/pod/e2e.yaml b/ci/jenkins/pod/e2e.yaml
index 3a87861174455..ebf856ba5eea3 100644
--- a/ci/jenkins/pod/e2e.yaml
+++ b/ci/jenkins/pod/e2e.yaml
@@ -9,7 +9,7 @@ spec:
enableServiceLinks: false
containers:
- name: pytest
- image: harbor.milvus.io/dockerhub/milvusdb/pytest:20230830-a8e5dc3
+ image: harbor.milvus.io/dockerhub/milvusdb/pytest:20231019-020ad9a
resources:
limits:
cpu: "6"
diff --git a/ci/jenkins/pod/rte.yaml b/ci/jenkins/pod/rte.yaml
index 43adc7d78688c..ae81cd7e6314e 100644
--- a/ci/jenkins/pod/rte.yaml
+++ b/ci/jenkins/pod/rte.yaml
@@ -45,7 +45,7 @@ spec:
- mountPath: /ci-logs
name: ci-logs
- name: pytest
- image: harbor.milvus.io/dockerhub/milvusdb/pytest:20230830-a8e5dc3
+ image: harbor.milvus.io/dockerhub/milvusdb/pytest:20231019-020ad9a
resources:
limits:
cpu: "6"
diff --git a/cmd/components/data_coord.go b/cmd/components/data_coord.go
index 2e556242c3a11..f7878314739f3 100644
--- a/cmd/components/data_coord.go
+++ b/cmd/components/data_coord.go
@@ -19,13 +19,14 @@ package components
import (
"context"
+ "go.uber.org/zap"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
grpcdatacoordclient "github.com/milvus-io/milvus/internal/distributed/datacoord"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "go.uber.org/zap"
)
// DataCoord implements grpc server of DataCoord server
diff --git a/cmd/components/data_node.go b/cmd/components/data_node.go
index 734d1e1686a54..25a7b9a91c37c 100644
--- a/cmd/components/data_node.go
+++ b/cmd/components/data_node.go
@@ -19,13 +19,14 @@ package components
import (
"context"
+ "go.uber.org/zap"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
grpcdatanode "github.com/milvus-io/milvus/internal/distributed/datanode"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "go.uber.org/zap"
)
// DataNode implements DataNode grpc server
diff --git a/cmd/components/index_coord.go b/cmd/components/index_coord.go
index adaad1a2757b5..ff03f83789318 100644
--- a/cmd/components/index_coord.go
+++ b/cmd/components/index_coord.go
@@ -26,8 +26,7 @@ import (
)
// IndexCoord implements IndexCoord grpc server
-type IndexCoord struct {
-}
+type IndexCoord struct{}
// NewIndexCoord creates a new IndexCoord
func NewIndexCoord(ctx context.Context, factory dependency.Factory) (*IndexCoord, error) {
@@ -48,7 +47,6 @@ func (s *IndexCoord) Stop() error {
// GetComponentStates returns indexnode's states
func (s *IndexCoord) Health(ctx context.Context) commonpb.StateCode {
- log.Info("IndexCoord is healthy")
return commonpb.StateCode_Healthy
}
diff --git a/cmd/components/index_node.go b/cmd/components/index_node.go
index 9c874a8801bac..4f947d35f4158 100644
--- a/cmd/components/index_node.go
+++ b/cmd/components/index_node.go
@@ -19,13 +19,14 @@ package components
import (
"context"
+ "go.uber.org/zap"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
grpcindexnode "github.com/milvus-io/milvus/internal/distributed/indexnode"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "go.uber.org/zap"
)
// IndexNode implements IndexNode grpc server
@@ -43,7 +44,6 @@ func NewIndexNode(ctx context.Context, factory dependency.Factory) (*IndexNode,
}
n.svr = svr
return n, nil
-
}
// Run starts service
diff --git a/cmd/components/proxy.go b/cmd/components/proxy.go
index 37bb9ef13873b..61a62df495538 100644
--- a/cmd/components/proxy.go
+++ b/cmd/components/proxy.go
@@ -19,13 +19,14 @@ package components
import (
"context"
+ "go.uber.org/zap"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
grpcproxy "github.com/milvus-io/milvus/internal/distributed/proxy"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "go.uber.org/zap"
)
// Proxy implements Proxy grpc server
diff --git a/cmd/components/query_node.go b/cmd/components/query_node.go
index f44cffa8ce1e0..50570ec152fe4 100644
--- a/cmd/components/query_node.go
+++ b/cmd/components/query_node.go
@@ -46,7 +46,6 @@ func NewQueryNode(ctx context.Context, factory dependency.Factory) (*QueryNode,
ctx: ctx,
svr: svr,
}, nil
-
}
// Run starts service
diff --git a/cmd/components/root_coord.go b/cmd/components/root_coord.go
index e26a5c50fd4e9..720511902a911 100644
--- a/cmd/components/root_coord.go
+++ b/cmd/components/root_coord.go
@@ -19,13 +19,14 @@ package components
import (
"context"
+ "go.uber.org/zap"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
rc "github.com/milvus-io/milvus/internal/distributed/rootcoord"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "go.uber.org/zap"
)
// RootCoord implements RoodCoord grpc server
diff --git a/cmd/embedded/embedded.go b/cmd/embedded/embedded.go
index 1f2148a093f28..34b979a612483 100644
--- a/cmd/embedded/embedded.go
+++ b/cmd/embedded/embedded.go
@@ -16,8 +16,9 @@
package main
+import "C"
+
import (
- "C"
"os"
"github.com/milvus-io/milvus/cmd/milvus"
diff --git a/cmd/main.go b/cmd/main.go
index 875126af32c33..9e02d743555ee 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -17,11 +17,81 @@
package main
import (
+ "log"
"os"
+ "os/exec"
+ "os/signal"
+ "strings"
+
+ "golang.org/x/exp/slices"
"github.com/milvus-io/milvus/cmd/milvus"
+ "github.com/milvus-io/milvus/internal/util/sessionutil"
+ "github.com/milvus-io/milvus/pkg/util/paramtable"
)
func main() {
- milvus.RunMilvus(os.Args)
+ idx := slices.Index(os.Args, "--run-with-subprocess")
+
+ // execute command as a subprocess if the command contains "--run-with-subprocess"
+ if idx > 0 {
+ args := slices.Delete(os.Args, idx, idx+1)
+ log.Println("run subprocess with cmd:", args)
+
+ /* #nosec G204 */
+ cmd := exec.Command(args[0], args[1:]...)
+
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+
+ if err := cmd.Start(); err != nil {
+ // Command not found on PATH, not executable, &c.
+ log.Fatal(err)
+ }
+
+ // wait for the command to finish
+ waitCh := make(chan error, 1)
+ go func() {
+ waitCh <- cmd.Wait()
+ close(waitCh)
+ }()
+
+ sc := make(chan os.Signal, 1)
+ signal.Notify(sc)
+
+ // Need a for loop to handle multiple signals
+ for {
+ select {
+ case sig := <-sc:
+ if err := cmd.Process.Signal(sig); err != nil {
+ log.Println("error sending signal", sig, err)
+ }
+ case err := <-waitCh:
+ // clean session
+ paramtable.Init()
+ params := paramtable.Get()
+ if len(args) >= 3 {
+ metaPath := params.EtcdCfg.MetaRootPath.GetValue()
+ endpoints := params.EtcdCfg.Endpoints.GetValue()
+ etcdEndpoints := strings.Split(endpoints, ",")
+
+ sessionSuffix := sessionutil.GetSessions(cmd.Process.Pid)
+ defer sessionutil.RemoveServerInfoFile(cmd.Process.Pid)
+
+ if err := milvus.CleanSession(metaPath, etcdEndpoints, sessionSuffix); err != nil {
+ log.Println("clean session failed", err.Error())
+ }
+ }
+
+ if err != nil {
+ log.Println("subprocess exit, ", err.Error())
+ } else {
+ log.Println("exit code:", cmd.ProcessState.ExitCode())
+ }
+ return
+ }
+ }
+ } else {
+ milvus.RunMilvus(os.Args)
+ }
}
diff --git a/cmd/milvus/help.go b/cmd/milvus/help.go
index c967b8228865b..3cb4d7c15912e 100644
--- a/cmd/milvus/help.go
+++ b/cmd/milvus/help.go
@@ -7,6 +7,11 @@ import (
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
+const (
+ RunCmd = "run"
+ RoleMixture = "mixture"
+)
+
var (
usageLine = fmt.Sprintf("Usage:\n"+
"%s\n%s\n%s\n%s\n", runLine, stopLine, mckLine, serverTypeLine)
diff --git a/cmd/milvus/mck.go b/cmd/milvus/mck.go
index c07758f7a7fc6..3775764bf07e9 100644
--- a/cmd/milvus/mck.go
+++ b/cmd/milvus/mck.go
@@ -364,7 +364,6 @@ func getTrashKey(taskType, key string) string {
}
func (c *mck) extractTask(prefix string, keys []string, values []string) {
-
for i := range keys {
taskID, err := strconv.ParseInt(filepath.Base(keys[i]), 10, 64)
if err != nil {
@@ -520,7 +519,6 @@ func (c *mck) extractVecFieldIndexInfo(taskID int64, infos []*querypb.FieldIndex
func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, error) {
header := commonpb.MsgHeader{}
err := proto.Unmarshal([]byte(t), &header)
-
if err != nil {
return errReturn(taskID, "MsgHeader", err)
}
diff --git a/cmd/milvus/run.go b/cmd/milvus/run.go
index b898e2d1c1a8e..bbb19eb88ab7a 100644
--- a/cmd/milvus/run.go
+++ b/cmd/milvus/run.go
@@ -10,96 +10,29 @@ import (
"go.uber.org/zap"
- "github.com/milvus-io/milvus/cmd/roles"
"github.com/milvus-io/milvus/pkg/log"
- "github.com/milvus-io/milvus/pkg/util/hardware"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
- "github.com/milvus-io/milvus/pkg/util/typeutil"
)
-const (
- RunCmd = "run"
- roleMixture = "mixture"
-)
-
-type run struct {
- serverType string
- // flags
- svrAlias string
- enableRootCoord bool
- enableQueryCoord bool
- enableDataCoord bool
- enableIndexCoord bool
- enableQueryNode bool
- enableDataNode bool
- enableIndexNode bool
- enableProxy bool
-}
-
-func (c *run) getHelp() string {
- return runLine + "\n" + serverTypeLine
-}
+type run struct{}
func (c *run) execute(args []string, flags *flag.FlagSet) {
if len(args) < 3 {
- fmt.Fprintln(os.Stderr, c.getHelp())
+ fmt.Fprintln(os.Stderr, getHelp())
return
}
flags.Usage = func() {
- fmt.Fprintln(os.Stderr, c.getHelp())
+ fmt.Fprintln(os.Stderr, getHelp())
}
- c.serverType = args[2]
- c.formatFlags(args, flags)
-
- // make go ignore SIGPIPE when all cgo threads set mask of SIGPIPE
+ // make go ignore SIGPIPE when all cgo thread set mask SIGPIPE
signal.Ignore(syscall.SIGPIPE)
- role := roles.NewMilvusRoles()
- role.Local = false
- switch c.serverType {
- case typeutil.RootCoordRole:
- role.EnableRootCoord = true
- case typeutil.ProxyRole:
- role.EnableProxy = true
- case typeutil.QueryCoordRole:
- role.EnableQueryCoord = true
- case typeutil.QueryNodeRole:
- role.EnableQueryNode = true
- case typeutil.DataCoordRole:
- role.EnableDataCoord = true
- case typeutil.DataNodeRole:
- role.EnableDataNode = true
- case typeutil.IndexCoordRole:
- role.EnableIndexCoord = true
- case typeutil.IndexNodeRole:
- role.EnableIndexNode = true
- case typeutil.StandaloneRole, typeutil.EmbeddedRole:
- role.EnableRootCoord = true
- role.EnableProxy = true
- role.EnableQueryCoord = true
- role.EnableQueryNode = true
- role.EnableDataCoord = true
- role.EnableDataNode = true
- role.EnableIndexCoord = true
- role.EnableIndexNode = true
- role.Local = true
- role.Embedded = c.serverType == typeutil.EmbeddedRole
- case roleMixture:
- role.EnableRootCoord = c.enableRootCoord
- role.EnableQueryCoord = c.enableQueryCoord
- role.EnableDataCoord = c.enableDataCoord
- role.EnableIndexCoord = c.enableIndexCoord
- role.EnableQueryNode = c.enableQueryNode
- role.EnableDataNode = c.enableDataNode
- role.EnableIndexNode = c.enableIndexNode
- role.EnableProxy = c.enableProxy
- default:
- fmt.Fprintf(os.Stderr, "Unknown server type = %s\n%s", c.serverType, c.getHelp())
- os.Exit(-1)
- }
+ serverType := args[2]
+ roles := GetMilvusRoles(args, flags)
+ // setup config for embedded milvus
- runtimeDir := createRuntimeDir(c.serverType)
- filename := getPidFileName(c.serverType, c.svrAlias)
+ runtimeDir := createRuntimeDir(serverType)
+ filename := getPidFileName(serverType, roles.Alias)
c.printBanner(flags.Output())
c.injectVariablesToEnv()
@@ -108,29 +41,7 @@ func (c *run) execute(args []string, flags *flag.FlagSet) {
panic(err)
}
defer removePidFile(lock)
- role.Run(c.svrAlias)
-}
-
-func (c *run) formatFlags(args []string, flags *flag.FlagSet) {
- flags.StringVar(&c.svrAlias, "alias", "", "set alias")
-
- flags.BoolVar(&c.enableRootCoord, typeutil.RootCoordRole, false, "enable root coordinator")
- flags.BoolVar(&c.enableQueryCoord, typeutil.QueryCoordRole, false, "enable query coordinator")
- flags.BoolVar(&c.enableIndexCoord, typeutil.IndexCoordRole, false, "enable index coordinator")
- flags.BoolVar(&c.enableDataCoord, typeutil.DataCoordRole, false, "enable data coordinator")
-
- flags.BoolVar(&c.enableQueryNode, typeutil.QueryNodeRole, false, "enable query node")
- flags.BoolVar(&c.enableDataNode, typeutil.DataNodeRole, false, "enable data node")
- flags.BoolVar(&c.enableIndexNode, typeutil.IndexNodeRole, false, "enable index node")
- flags.BoolVar(&c.enableProxy, typeutil.ProxyRole, false, "enable proxy node")
-
- if c.serverType == typeutil.EmbeddedRole {
- flags.SetOutput(io.Discard)
- }
- hardware.InitMaxprocs(c.serverType, flags)
- if err := flags.Parse(args[3:]); err != nil {
- os.Exit(-1)
- }
+ roles.Run()
}
func (c *run) printBanner(w io.Writer) {
diff --git a/cmd/milvus/stop.go b/cmd/milvus/stop.go
index 66517bd21b3d8..1392125e6f44e 100644
--- a/cmd/milvus/stop.go
+++ b/cmd/milvus/stop.go
@@ -61,7 +61,7 @@ func (c *stop) formatFlags(args []string, flags *flag.FlagSet) {
func (c *stop) stopPid(filename string, runtimeDir string) error {
var pid int
- fd, err := os.OpenFile(path.Join(runtimeDir, filename), os.O_RDONLY, 0664)
+ fd, err := os.OpenFile(path.Join(runtimeDir, filename), os.O_RDONLY, 0o664)
if err != nil {
return err
}
diff --git a/cmd/milvus/util.go b/cmd/milvus/util.go
index 1e40bb4048e69..e042190af2a3e 100644
--- a/cmd/milvus/util.go
+++ b/cmd/milvus/util.go
@@ -1,20 +1,34 @@
package milvus
import (
+ "context"
+ "encoding/json"
+ "flag"
"fmt"
"io"
- "io/ioutil"
"os"
"path"
"runtime"
+ "strconv"
+ "strings"
+ "time"
+ "github.com/cockroachdb/errors"
"github.com/gofrs/flock"
+ "github.com/samber/lo"
+ clientv3 "go.etcd.io/etcd/client/v3"
+ "go.uber.org/zap"
+ "github.com/milvus-io/milvus/cmd/roles"
+ "github.com/milvus-io/milvus/internal/util/sessionutil"
+ "github.com/milvus-io/milvus/pkg/log"
+ "github.com/milvus-io/milvus/pkg/util/etcd"
+ "github.com/milvus-io/milvus/pkg/util/hardware"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
func makeRuntimeDir(dir string) error {
- perm := os.FileMode(0755)
+ perm := os.FileMode(0o755)
// os.MkdirAll equal to `mkdir -p`
err := os.MkdirAll(dir, perm)
if err != nil {
@@ -22,7 +36,7 @@ func makeRuntimeDir(dir string) error {
return fmt.Errorf("create runtime dir %s failed, err: %s", dir, err.Error())
}
- tmpFile, err := ioutil.TempFile(dir, "tmp")
+ tmpFile, err := os.CreateTemp(dir, "tmp")
if err != nil {
return err
}
@@ -63,7 +77,7 @@ func createRuntimeDir(sType string) string {
func createPidFile(w io.Writer, filename string, runtimeDir string) (*flock.Flock, error) {
fileFullName := path.Join(runtimeDir, filename)
- fd, err := os.OpenFile(fileFullName, os.O_CREATE|os.O_RDWR, 0664)
+ fd, err := os.OpenFile(fileFullName, os.O_CREATE|os.O_RDWR, 0o664)
if err != nil {
return nil, fmt.Errorf("file %s is locked, error = %w", filename, err)
}
@@ -106,3 +120,190 @@ func removePidFile(lock *flock.Flock) {
lock.Close()
os.Remove(filename)
}
+
+func GetMilvusRoles(args []string, flags *flag.FlagSet) *roles.MilvusRoles {
+ alias, enableRootCoord, enableQueryCoord, enableIndexCoord, enableDataCoord, enableQueryNode,
+ enableDataNode, enableIndexNode, enableProxy := formatFlags(args, flags)
+
+ serverType := args[2]
+ role := roles.NewMilvusRoles()
+ role.Alias = alias
+
+ switch serverType {
+ case typeutil.RootCoordRole:
+ role.EnableRootCoord = true
+ case typeutil.ProxyRole:
+ role.EnableProxy = true
+ case typeutil.QueryCoordRole:
+ role.EnableQueryCoord = true
+ case typeutil.QueryNodeRole:
+ role.EnableQueryNode = true
+ case typeutil.DataCoordRole:
+ role.EnableDataCoord = true
+ case typeutil.DataNodeRole:
+ role.EnableDataNode = true
+ case typeutil.IndexCoordRole:
+ role.EnableIndexCoord = true
+ case typeutil.IndexNodeRole:
+ role.EnableIndexNode = true
+ case typeutil.StandaloneRole, typeutil.EmbeddedRole:
+ role.EnableRootCoord = true
+ role.EnableProxy = true
+ role.EnableQueryCoord = true
+ role.EnableQueryNode = true
+ role.EnableDataCoord = true
+ role.EnableDataNode = true
+ role.EnableIndexCoord = true
+ role.EnableIndexNode = true
+ role.Local = true
+ role.Embedded = serverType == typeutil.EmbeddedRole
+ case RoleMixture:
+ role.EnableRootCoord = enableRootCoord
+ role.EnableQueryCoord = enableQueryCoord
+ role.EnableDataCoord = enableDataCoord
+ role.EnableIndexCoord = enableIndexCoord
+ role.EnableQueryNode = enableQueryNode
+ role.EnableDataNode = enableDataNode
+ role.EnableIndexNode = enableIndexNode
+ role.EnableProxy = enableProxy
+ default:
+ fmt.Fprintf(os.Stderr, "Unknown server type = %s\n%s", serverType, getHelp())
+ os.Exit(-1)
+ }
+
+ return role
+}
+
+func formatFlags(args []string, flags *flag.FlagSet) (alias string, enableRootCoord, enableQueryCoord,
+ enableIndexCoord, enableDataCoord, enableQueryNode, enableDataNode, enableIndexNode, enableProxy bool,
+) {
+ flags.StringVar(&alias, "alias", "", "set alias")
+
+ flags.BoolVar(&enableRootCoord, typeutil.RootCoordRole, false, "enable root coordinator")
+ flags.BoolVar(&enableQueryCoord, typeutil.QueryCoordRole, false, "enable query coordinator")
+ flags.BoolVar(&enableIndexCoord, typeutil.IndexCoordRole, false, "enable index coordinator")
+ flags.BoolVar(&enableDataCoord, typeutil.DataCoordRole, false, "enable data coordinator")
+
+ flags.BoolVar(&enableQueryNode, typeutil.QueryNodeRole, false, "enable query node")
+ flags.BoolVar(&enableDataNode, typeutil.DataNodeRole, false, "enable data node")
+ flags.BoolVar(&enableIndexNode, typeutil.IndexNodeRole, false, "enable index node")
+ flags.BoolVar(&enableProxy, typeutil.ProxyRole, false, "enable proxy node")
+
+ serverType := args[2]
+ if serverType == typeutil.EmbeddedRole {
+ flags.SetOutput(io.Discard)
+ }
+ hardware.InitMaxprocs(serverType, flags)
+ if err := flags.Parse(args[3:]); err != nil {
+ os.Exit(-1)
+ }
+ return
+}
+
+func getHelp() string {
+ return runLine + "\n" + serverTypeLine
+}
+
+func CleanSession(metaPath string, etcdEndpoints []string, sessionSuffix []string) error {
+ if len(sessionSuffix) == 0 {
+ log.Warn("not found session info , skip to clean sessions")
+ return nil
+ }
+
+ etcdCli, err := etcd.GetRemoteEtcdClient(etcdEndpoints)
+ if err != nil {
+ return err
+ }
+ defer etcdCli.Close()
+
+ ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
+ defer cancel()
+
+ keys := getSessionPaths(ctx, etcdCli, metaPath, sessionSuffix)
+ if len(keys) == 0 {
+ return nil
+ }
+
+ for _, key := range keys {
+ _, _ = etcdCli.Delete(ctx, key, clientv3.WithPrefix())
+ }
+ log.Info("clean sessions from etcd", zap.Any("keys", keys))
+ return nil
+}
+
+func getSessionPaths(ctx context.Context, client *clientv3.Client, metaPath string, sessionSuffix []string) []string {
+ sessionKeys := make([]string, 0)
+ sessionPathPrefix := path.Join(metaPath, sessionutil.DefaultServiceRoot)
+ newSessionSuffixSet := addActiveKeySuffix(ctx, client, sessionPathPrefix, sessionSuffix)
+ for _, suffix := range newSessionSuffixSet {
+ key := path.Join(sessionPathPrefix, suffix)
+ sessionKeys = append(sessionKeys, key)
+ }
+ return sessionKeys
+}
+
+// filterUnmatchedKey skip active keys that don't match completed key, the latest active key may from standby server
+func addActiveKeySuffix(ctx context.Context, client *clientv3.Client, sessionPathPrefix string, sessionSuffix []string) []string {
+ suffixSet := lo.SliceToMap(sessionSuffix, func(t string) (string, struct{}) {
+ return t, struct{}{}
+ })
+
+ for _, suffix := range sessionSuffix {
+ if strings.Contains(suffix, "-") && (strings.HasPrefix(suffix, typeutil.RootCoordRole) ||
+ strings.HasPrefix(suffix, typeutil.QueryCoordRole) || strings.HasPrefix(suffix, typeutil.DataCoordRole) ||
+ strings.HasPrefix(suffix, typeutil.IndexCoordRole)) {
+ res := strings.Split(suffix, "-")
+ if len(res) != 2 {
+ // skip illegal keys
+ log.Warn("skip illegal key", zap.String("suffix", suffix))
+ continue
+ }
+
+ serverType := res[0]
+ targetServerID, err := strconv.ParseInt(res[1], 10, 64)
+ if err != nil {
+ log.Warn("get server id failed from key", zap.String("suffix", suffix), zap.Error(err))
+ continue
+ }
+
+ key := path.Join(sessionPathPrefix, serverType)
+ serverID, err := getServerID(ctx, client, key)
+ if err != nil {
+ log.Warn("get server id failed from key", zap.String("suffix", suffix), zap.Error(err))
+ continue
+ }
+
+ if serverID == targetServerID {
+ log.Info("add active serverID key", zap.String("suffix", suffix), zap.String("key", key))
+ suffixSet[serverType] = struct{}{}
+ }
+
+ // also remove a faked indexcoord seesion if role is a datacoord
+ if strings.HasPrefix(suffix, typeutil.DataCoordRole) {
+ suffixSet[typeutil.IndexCoordRole] = struct{}{}
+ }
+ }
+ }
+
+ return lo.MapToSlice(suffixSet, func(key string, v struct{}) string { return key })
+}
+
+func getServerID(ctx context.Context, client *clientv3.Client, key string) (int64, error) {
+ resp, err := client.Get(ctx, key)
+ if err != nil {
+ return 0, err
+ }
+
+ if len(resp.Kvs) == 0 {
+ return 0, errors.New("not found value")
+ }
+
+ value := resp.Kvs[0].Value
+ session := &sessionutil.SessionRaw{}
+ err = json.Unmarshal(value, &session)
+ if err != nil {
+ return 0, err
+ }
+
+ return session.ServerID, nil
+}
diff --git a/cmd/roles/roles.go b/cmd/roles/roles.go
index 550b575719d11..7d3d62e49a22d 100644
--- a/cmd/roles/roles.go
+++ b/cmd/roles/roles.go
@@ -27,6 +27,10 @@ import (
"syscall"
"time"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+ "go.uber.org/zap"
+
"github.com/milvus-io/milvus/cmd/components"
"github.com/milvus-io/milvus/internal/http"
"github.com/milvus-io/milvus/internal/http/healthz"
@@ -37,14 +41,12 @@ import (
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/tracer"
"github.com/milvus-io/milvus/pkg/util/etcd"
+ "github.com/milvus-io/milvus/pkg/util/generic"
"github.com/milvus-io/milvus/pkg/util/logutil"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
_ "github.com/milvus-io/milvus/pkg/util/symbolizer" // support symbolizer and crash dump
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
- "go.uber.org/zap"
)
// all milvus related metrics is in a separate registry
@@ -91,7 +93,7 @@ func runComponent[T component](ctx context.Context,
runWg *sync.WaitGroup,
creator func(context.Context, dependency.Factory) (T, error),
metricRegister func(*prometheus.Registry),
-) T {
+) component {
var role T
sign := make(chan struct{})
@@ -118,6 +120,9 @@ func runComponent[T component](ctx context.Context,
healthz.Register(role)
metricRegister(Registry.GoRegistry)
+ if generic.IsZero(role) {
+ return nil
+ }
return role
}
@@ -133,9 +138,11 @@ type MilvusRoles struct {
EnableIndexNode bool `env:"ENABLE_INDEX_NODE"`
Local bool
+ Alias string
Embedded bool
- closed chan struct{}
- once sync.Once
+
+ closed chan struct{}
+ once sync.Once
}
// NewMilvusRoles creates a new MilvusRoles with private fields initialized.
@@ -161,46 +168,52 @@ func (mr *MilvusRoles) printLDPreLoad() {
}
}
-func (mr *MilvusRoles) runRootCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.RootCoord {
+func (mr *MilvusRoles) runRootCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
return runComponent(ctx, localMsg, wg, components.NewRootCoord, metrics.RegisterRootCoord)
}
-func (mr *MilvusRoles) runProxy(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.Proxy {
+func (mr *MilvusRoles) runProxy(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
return runComponent(ctx, localMsg, wg, components.NewProxy, metrics.RegisterProxy)
}
-func (mr *MilvusRoles) runQueryCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.QueryCoord {
+func (mr *MilvusRoles) runQueryCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
return runComponent(ctx, localMsg, wg, components.NewQueryCoord, metrics.RegisterQueryCoord)
}
-func (mr *MilvusRoles) runQueryNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.QueryNode {
+func (mr *MilvusRoles) runQueryNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
+ // clear local storage
rootPath := paramtable.Get().LocalStorageCfg.Path.GetValue()
queryDataLocalPath := filepath.Join(rootPath, typeutil.QueryNodeRole)
cleanLocalDir(queryDataLocalPath)
+ // clear mmap dir
+ mmapDir := paramtable.Get().QueryNodeCfg.MmapDirPath.GetValue()
+ if len(mmapDir) > 0 {
+ cleanLocalDir(mmapDir)
+ }
return runComponent(ctx, localMsg, wg, components.NewQueryNode, metrics.RegisterQueryNode)
}
-func (mr *MilvusRoles) runDataCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.DataCoord {
+func (mr *MilvusRoles) runDataCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
return runComponent(ctx, localMsg, wg, components.NewDataCoord, metrics.RegisterDataCoord)
}
-func (mr *MilvusRoles) runDataNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.DataNode {
+func (mr *MilvusRoles) runDataNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
return runComponent(ctx, localMsg, wg, components.NewDataNode, metrics.RegisterDataNode)
}
-func (mr *MilvusRoles) runIndexCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.IndexCoord {
+func (mr *MilvusRoles) runIndexCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
return runComponent(ctx, localMsg, wg, components.NewIndexCoord, func(registry *prometheus.Registry) {})
}
-func (mr *MilvusRoles) runIndexNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) *components.IndexNode {
+func (mr *MilvusRoles) runIndexNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
wg.Add(1)
rootPath := paramtable.Get().LocalStorageCfg.Path.GetValue()
indexDataLocalPath := filepath.Join(rootPath, typeutil.IndexNodeRole)
@@ -237,6 +250,7 @@ func (mr *MilvusRoles) setupLogger() {
// Register serves prometheus http service
func setupPrometheusHTTPServer(r *internalmetrics.MilvusRegistry) {
+ log.Info("setupPrometheusHTTPServer")
http.Register(&http.Handler{
Path: "/metrics",
Handler: promhttp.HandlerFor(r, promhttp.HandlerOpts{}),
@@ -282,7 +296,7 @@ func (mr *MilvusRoles) handleSignals() func() {
}
// Run Milvus components.
-func (mr *MilvusRoles) Run(alias string) {
+func (mr *MilvusRoles) Run() {
// start signal handler, defer close func
closeFn := mr.handleSignals()
defer closeFn()
@@ -325,54 +339,49 @@ func (mr *MilvusRoles) Run(alias string) {
}
http.ServeHTTP()
+ setupPrometheusHTTPServer(Registry)
- var rc *components.RootCoord
var wg sync.WaitGroup
local := mr.Local
+
+ var rootCoord, queryCoord, indexCoord, dataCoord component
+ var proxy, dataNode, indexNode, queryNode component
if mr.EnableRootCoord {
- rc = mr.runRootCoord(ctx, local, &wg)
+ rootCoord = mr.runRootCoord(ctx, local, &wg)
}
- var pn *components.Proxy
if mr.EnableProxy {
- pn = mr.runProxy(ctx, local, &wg)
+ proxy = mr.runProxy(ctx, local, &wg)
}
- var qs *components.QueryCoord
if mr.EnableQueryCoord {
- qs = mr.runQueryCoord(ctx, local, &wg)
+ queryCoord = mr.runQueryCoord(ctx, local, &wg)
}
- var qn *components.QueryNode
if mr.EnableQueryNode {
- qn = mr.runQueryNode(ctx, local, &wg)
+ queryNode = mr.runQueryNode(ctx, local, &wg)
}
- var ds *components.DataCoord
if mr.EnableDataCoord {
- ds = mr.runDataCoord(ctx, local, &wg)
+ dataCoord = mr.runDataCoord(ctx, local, &wg)
}
- var dn *components.DataNode
if mr.EnableDataNode {
- dn = mr.runDataNode(ctx, local, &wg)
+ dataNode = mr.runDataNode(ctx, local, &wg)
}
- var is *components.IndexCoord
if mr.EnableIndexCoord {
- is = mr.runIndexCoord(ctx, local, &wg)
+ indexCoord = mr.runIndexCoord(ctx, local, &wg)
}
- var in *components.IndexNode
if mr.EnableIndexNode {
- in = mr.runIndexNode(ctx, local, &wg)
+ indexNode = mr.runIndexNode(ctx, local, &wg)
}
wg.Wait()
mr.setupLogger()
tracer.Init()
- setupPrometheusHTTPServer(Registry)
paramtable.SetCreateTime(time.Now())
paramtable.SetUpdateTime(time.Now())
@@ -381,9 +390,11 @@ func (mr *MilvusRoles) Run(alias string) {
// stop coordinators first
// var component
- coordinators := []component{rc, qs, ds, is}
- for _, coord := range coordinators {
+ coordinators := []component{rootCoord, queryCoord, dataCoord, indexCoord}
+ for idx, coord := range coordinators {
+ log.Warn("stop processing")
if coord != nil {
+ log.Warn("stop coord", zap.Int("idx", idx), zap.Any("coord", coord))
wg.Add(1)
go func(coord component) {
defer wg.Done()
@@ -395,7 +406,7 @@ func (mr *MilvusRoles) Run(alias string) {
log.Info("All coordinators have stopped")
// stop nodes
- nodes := []component{qn, in, dn}
+ nodes := []component{queryNode, indexNode, dataNode}
for _, node := range nodes {
if node != nil {
wg.Add(1)
@@ -409,10 +420,39 @@ func (mr *MilvusRoles) Run(alias string) {
log.Info("All nodes have stopped")
// stop proxy
- if pn != nil {
- pn.Stop()
+ if proxy != nil {
+ proxy.Stop()
log.Info("proxy stopped")
}
log.Info("Milvus components graceful stop done")
}
+
+func (mr *MilvusRoles) GetRoles() []string {
+ roles := make([]string, 0)
+ if mr.EnableRootCoord {
+ roles = append(roles, typeutil.RootCoordRole)
+ }
+ if mr.EnableProxy {
+ roles = append(roles, typeutil.ProxyRole)
+ }
+ if mr.EnableQueryCoord {
+ roles = append(roles, typeutil.QueryCoordRole)
+ }
+ if mr.EnableQueryNode {
+ roles = append(roles, typeutil.QueryNodeRole)
+ }
+ if mr.EnableDataCoord {
+ roles = append(roles, typeutil.DataCoordRole)
+ }
+ if mr.EnableDataNode {
+ roles = append(roles, typeutil.DataNodeRole)
+ }
+ if mr.EnableIndexCoord {
+ roles = append(roles, typeutil.IndexCoordRole)
+ }
+ if mr.EnableIndexNode {
+ roles = append(roles, typeutil.IndexNodeRole)
+ }
+ return roles
+}
diff --git a/cmd/tools/config/generate.go b/cmd/tools/config/generate.go
index a30f2a46c8b80..0e6a4d5571f52 100644
--- a/cmd/tools/config/generate.go
+++ b/cmd/tools/config/generate.go
@@ -7,12 +7,13 @@ import (
"reflect"
"strings"
- "github.com/milvus-io/milvus/pkg/log"
- "github.com/milvus-io/milvus/pkg/util/paramtable"
- "github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/samber/lo"
"go.uber.org/zap"
"golang.org/x/exp/slices"
+
+ "github.com/milvus-io/milvus/pkg/log"
+ "github.com/milvus-io/milvus/pkg/util/paramtable"
+ "github.com/milvus-io/milvus/pkg/util/typeutil"
)
type DocContent struct {
@@ -106,7 +107,7 @@ type YamlMarshaller struct {
}
func (m *YamlMarshaller) writeYamlRecursive(data []DocContent, level int) {
- var topLevels = typeutil.NewOrderedMap[string, []DocContent]()
+ topLevels := typeutil.NewOrderedMap[string, []DocContent]()
for _, d := range data {
key := strings.Split(d.key, ".")[level]
diff --git a/cmd/tools/config/main.go b/cmd/tools/config/main.go
index 73aa670c3ef4c..8d6d0abfe1c2a 100644
--- a/cmd/tools/config/main.go
+++ b/cmd/tools/config/main.go
@@ -36,5 +36,4 @@ func main() {
default:
log.Error(fmt.Sprintf("unknown argument %s", args[1]))
}
-
}
diff --git a/cmd/tools/config/printer.go b/cmd/tools/config/printer.go
index 54cd90b4bf4dd..20300694868dc 100644
--- a/cmd/tools/config/printer.go
+++ b/cmd/tools/config/printer.go
@@ -5,9 +5,10 @@ import (
"os"
"sort"
- "github.com/milvus-io/milvus/pkg/log"
"github.com/spf13/viper"
"go.uber.org/zap"
+
+ "github.com/milvus-io/milvus/pkg/log"
)
func ShowYaml(filepath string) {
diff --git a/cmd/tools/datameta/main.go b/cmd/tools/datameta/main.go
index c5e20035f6ac7..d6bdffc35c2cb 100644
--- a/cmd/tools/datameta/main.go
+++ b/cmd/tools/datameta/main.go
@@ -7,12 +7,13 @@ import (
"strings"
"github.com/golang/protobuf/proto"
+ "go.uber.org/zap"
+
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
- "go.uber.org/zap"
)
var (
diff --git a/cmd/tools/migration/allocator/atomic_allocator.go b/cmd/tools/migration/allocator/atomic_allocator.go
index 894747ec06d4e..cce5a4c0b3cba 100644
--- a/cmd/tools/migration/allocator/atomic_allocator.go
+++ b/cmd/tools/migration/allocator/atomic_allocator.go
@@ -1,8 +1,9 @@
package allocator
import (
- "github.com/milvus-io/milvus/pkg/util/typeutil"
"go.uber.org/atomic"
+
+ "github.com/milvus-io/milvus/pkg/util/typeutil"
)
const (
diff --git a/cmd/tools/migration/backend/backend.go b/cmd/tools/migration/backend/backend.go
index 915477f359ee0..91c33e431d240 100644
--- a/cmd/tools/migration/backend/backend.go
+++ b/cmd/tools/migration/backend/backend.go
@@ -6,10 +6,8 @@ import (
"github.com/blang/semver/v4"
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
-
- "github.com/milvus-io/milvus/cmd/tools/migration/versions"
-
"github.com/milvus-io/milvus/cmd/tools/migration/meta"
+ "github.com/milvus-io/milvus/cmd/tools/migration/versions"
"github.com/milvus-io/milvus/pkg/util"
)
diff --git a/cmd/tools/migration/backend/backup_header.go b/cmd/tools/migration/backend/backup_header.go
index 40e30ed8e9dd6..59436505ed7cb 100644
--- a/cmd/tools/migration/backend/backup_header.go
+++ b/cmd/tools/migration/backend/backup_header.go
@@ -3,9 +3,9 @@ package backend
import (
"encoding/json"
- "github.com/milvus-io/milvus/cmd/tools/migration/console"
-
"github.com/golang/protobuf/proto"
+
+ "github.com/milvus-io/milvus/cmd/tools/migration/console"
)
type BackupHeaderVersion int32
@@ -78,7 +78,7 @@ func (v *BackupHeaderExtra) ToJSONBytes() []byte {
}
func GetExtra(extra []byte) *BackupHeaderExtra {
- var v = newDefaultBackupHeaderExtra()
+ v := newDefaultBackupHeaderExtra()
err := json.Unmarshal(extra, v)
if err != nil {
console.Error(err.Error())
diff --git a/cmd/tools/migration/backend/backup_restore.go b/cmd/tools/migration/backend/backup_restore.go
index 6980c25bee2e0..c21d096cdf1be 100644
--- a/cmd/tools/migration/backend/backup_restore.go
+++ b/cmd/tools/migration/backend/backup_restore.go
@@ -6,6 +6,7 @@ import (
"io"
"github.com/golang/protobuf/proto"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
)
diff --git a/cmd/tools/migration/backend/etcd.go b/cmd/tools/migration/backend/etcd.go
index b4ad68f56d49c..0f5e4d28a507c 100644
--- a/cmd/tools/migration/backend/etcd.go
+++ b/cmd/tools/migration/backend/etcd.go
@@ -1,11 +1,12 @@
package backend
import (
+ clientv3 "go.etcd.io/etcd/client/v3"
+
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/pkg/util/etcd"
- clientv3 "go.etcd.io/etcd/client/v3"
)
type etcdBasedBackend struct {
diff --git a/cmd/tools/migration/backend/etcd210.go b/cmd/tools/migration/backend/etcd210.go
index 527ec44fe6a83..e66cbd2a815b9 100644
--- a/cmd/tools/migration/backend/etcd210.go
+++ b/cmd/tools/migration/backend/etcd210.go
@@ -3,20 +3,18 @@ package backend
import (
"context"
"fmt"
- "io/ioutil"
+ "os"
"path"
"strconv"
"strings"
+ "github.com/golang/protobuf/proto"
clientv3 "go.etcd.io/etcd/client/v3"
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
+ "github.com/milvus-io/milvus/cmd/tools/migration/console"
"github.com/milvus-io/milvus/cmd/tools/migration/legacy"
-
"github.com/milvus-io/milvus/cmd/tools/migration/legacy/legacypb"
-
- "github.com/golang/protobuf/proto"
- "github.com/milvus-io/milvus/cmd/tools/migration/console"
"github.com/milvus-io/milvus/cmd/tools/migration/meta"
"github.com/milvus-io/milvus/cmd/tools/migration/utils"
"github.com/milvus-io/milvus/cmd/tools/migration/versions"
@@ -56,7 +54,7 @@ func (b etcd210) loadTtAliases() (meta.TtAliasesMeta210, error) {
tsKey := keys[i]
tsValue := values[i]
valueIsTombstone := rootcoord.IsTombstone(tsValue)
- var aliasInfo = &pb.CollectionInfo{} // alias stored in collection info.
+ aliasInfo := &pb.CollectionInfo{} // alias stored in collection info.
if valueIsTombstone {
aliasInfo = nil
} else {
@@ -88,7 +86,7 @@ func (b etcd210) loadAliases() (meta.AliasesMeta210, error) {
key := keys[i]
value := values[i]
valueIsTombstone := rootcoord.IsTombstone(value)
- var aliasInfo = &pb.CollectionInfo{} // alias stored in collection info.
+ aliasInfo := &pb.CollectionInfo{} // alias stored in collection info.
if valueIsTombstone {
aliasInfo = nil
} else {
@@ -122,7 +120,7 @@ func (b etcd210) loadTtCollections() (meta.TtCollectionsMeta210, error) {
}
valueIsTombstone := rootcoord.IsTombstone(tsValue)
- var coll = &pb.CollectionInfo{}
+ coll := &pb.CollectionInfo{}
if valueIsTombstone {
coll = nil
} else {
@@ -164,7 +162,7 @@ func (b etcd210) loadCollections() (meta.CollectionsMeta210, error) {
}
valueIsTombstone := rootcoord.IsTombstone(value)
- var coll = &pb.CollectionInfo{}
+ coll := &pb.CollectionInfo{}
if valueIsTombstone {
coll = nil
} else {
@@ -213,7 +211,7 @@ func (b etcd210) loadCollectionIndexes() (meta.CollectionIndexesMeta210, error)
key := keys[i]
value := values[i]
- var index = &pb.IndexInfo{}
+ index := &pb.IndexInfo{}
if err := proto.Unmarshal([]byte(value), index); err != nil {
return nil, err
}
@@ -240,7 +238,7 @@ func (b etcd210) loadSegmentIndexes() (meta.SegmentIndexesMeta210, error) {
for i := 0; i < l; i++ {
value := values[i]
- var index = &pb.SegmentIndexInfo{}
+ index := &pb.SegmentIndexInfo{}
if err := proto.Unmarshal([]byte(value), index); err != nil {
return nil, err
}
@@ -263,7 +261,7 @@ func (b etcd210) loadIndexBuildMeta() (meta.IndexBuildMeta210, error) {
for i := 0; i < l; i++ {
value := values[i]
- var record = &legacypb.IndexMeta{}
+ record := &legacypb.IndexMeta{}
if err := proto.Unmarshal([]byte(value), record); err != nil {
return nil, err
}
@@ -434,7 +432,7 @@ func (b etcd210) Backup(meta *meta.Meta, backupFile string) error {
return err
}
console.Warning(fmt.Sprintf("backup to: %s", backupFile))
- return ioutil.WriteFile(backupFile, backup, 0600)
+ return os.WriteFile(backupFile, backup, 0o600)
}
func (b etcd210) BackupV2(file string) error {
@@ -489,11 +487,11 @@ func (b etcd210) BackupV2(file string) error {
}
console.Warning(fmt.Sprintf("backup to: %s", file))
- return ioutil.WriteFile(file, backup, 0600)
+ return os.WriteFile(file, backup, 0o600)
}
func (b etcd210) Restore(backupFile string) error {
- backup, err := ioutil.ReadFile(backupFile)
+ backup, err := os.ReadFile(backupFile)
if err != nil {
return err
}
diff --git a/cmd/tools/migration/backend/etcd220.go b/cmd/tools/migration/backend/etcd220.go
index 4eae60d43e49a..d00805786537e 100644
--- a/cmd/tools/migration/backend/etcd220.go
+++ b/cmd/tools/migration/backend/etcd220.go
@@ -3,14 +3,11 @@ package backend
import (
"fmt"
- "github.com/milvus-io/milvus/internal/metastore/kv/querycoord"
-
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
-
- "github.com/milvus-io/milvus/pkg/util"
-
"github.com/milvus-io/milvus/cmd/tools/migration/meta"
+ "github.com/milvus-io/milvus/internal/metastore/kv/querycoord"
"github.com/milvus-io/milvus/internal/metastore/kv/rootcoord"
+ "github.com/milvus-io/milvus/pkg/util"
)
// etcd220 implements Backend.
diff --git a/cmd/tools/migration/command/main.go b/cmd/tools/migration/command/main.go
index 5c393f4c5af15..40185ef4695f3 100644
--- a/cmd/tools/migration/command/main.go
+++ b/cmd/tools/migration/command/main.go
@@ -5,9 +5,8 @@ import (
"fmt"
"os"
- "github.com/milvus-io/milvus/cmd/tools/migration/console"
-
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
+ "github.com/milvus-io/milvus/cmd/tools/migration/console"
)
func Execute(args []string) {
diff --git a/cmd/tools/migration/command/run.go b/cmd/tools/migration/command/run.go
index e90faacfcd64e..d30abdbd36d67 100644
--- a/cmd/tools/migration/command/run.go
+++ b/cmd/tools/migration/command/run.go
@@ -4,9 +4,7 @@ import (
"context"
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
-
"github.com/milvus-io/milvus/cmd/tools/migration/console"
-
"github.com/milvus-io/milvus/cmd/tools/migration/migration"
)
diff --git a/cmd/tools/migration/meta/210_to_220.go b/cmd/tools/migration/meta/210_to_220.go
index 3ae5ee7722a7c..1fceb29d0e083 100644
--- a/cmd/tools/migration/meta/210_to_220.go
+++ b/cmd/tools/migration/meta/210_to_220.go
@@ -6,11 +6,11 @@ import (
"strconv"
"strings"
- "github.com/milvus-io/milvus/cmd/tools/migration/legacy/legacypb"
-
- "github.com/milvus-io/milvus/cmd/tools/migration/allocator"
+ "go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
+ "github.com/milvus-io/milvus/cmd/tools/migration/allocator"
+ "github.com/milvus-io/milvus/cmd/tools/migration/legacy/legacypb"
"github.com/milvus-io/milvus/cmd/tools/migration/versions"
"github.com/milvus-io/milvus/internal/metastore/model"
pb "github.com/milvus-io/milvus/internal/proto/etcdpb"
@@ -19,7 +19,6 @@ import (
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
- "go.uber.org/zap"
)
func alias210ToAlias220(record *pb.CollectionInfo, ts Timestamp) *model.Alias {
diff --git a/cmd/tools/migration/meta/meta.go b/cmd/tools/migration/meta/meta.go
index b2d36d429196f..f76e6b174eb7c 100644
--- a/cmd/tools/migration/meta/meta.go
+++ b/cmd/tools/migration/meta/meta.go
@@ -2,11 +2,14 @@ package meta
import (
"github.com/blang/semver/v4"
+
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
-type UniqueID = typeutil.UniqueID
-type Timestamp = typeutil.Timestamp
+type (
+ UniqueID = typeutil.UniqueID
+ Timestamp = typeutil.Timestamp
+)
type Meta struct {
SourceVersion semver.Version
diff --git a/cmd/tools/migration/meta/meta210.go b/cmd/tools/migration/meta/meta210.go
index 9f3c65f97a057..23016210660f9 100644
--- a/cmd/tools/migration/meta/meta210.go
+++ b/cmd/tools/migration/meta/meta210.go
@@ -21,14 +21,20 @@ type FieldIndexesWithSchema struct {
type FieldIndexes210 map[UniqueID]*FieldIndexesWithSchema // coll_id -> field indexes.
-type TtCollectionsMeta210 map[UniqueID]map[Timestamp]*pb.CollectionInfo // coll_id -> ts -> coll
-type CollectionsMeta210 map[UniqueID]*pb.CollectionInfo // coll_id -> coll
+type (
+ TtCollectionsMeta210 map[UniqueID]map[Timestamp]*pb.CollectionInfo // coll_id -> ts -> coll
+ CollectionsMeta210 map[UniqueID]*pb.CollectionInfo // coll_id -> coll
+)
-type TtAliasesMeta210 map[string]map[Timestamp]*pb.CollectionInfo // alias name -> ts -> coll
-type AliasesMeta210 map[string]*pb.CollectionInfo // alias name -> coll
+type (
+ TtAliasesMeta210 map[string]map[Timestamp]*pb.CollectionInfo // alias name -> ts -> coll
+ AliasesMeta210 map[string]*pb.CollectionInfo // alias name -> coll
+)
-type CollectionIndexesMeta210 map[UniqueID]map[UniqueID]*pb.IndexInfo // coll_id -> index_id -> index
-type SegmentIndexesMeta210 map[UniqueID]map[UniqueID]*pb.SegmentIndexInfo // seg_id -> index_id -> segment index
+type (
+ CollectionIndexesMeta210 map[UniqueID]map[UniqueID]*pb.IndexInfo // coll_id -> index_id -> index
+ SegmentIndexesMeta210 map[UniqueID]map[UniqueID]*pb.SegmentIndexInfo // seg_id -> index_id -> segment index
+)
type IndexBuildMeta210 map[UniqueID]*legacypb.IndexMeta // index_build_id -> index
diff --git a/cmd/tools/migration/meta/meta220.go b/cmd/tools/migration/meta/meta220.go
index 63044fdb8c1b8..f190b4061c651 100644
--- a/cmd/tools/migration/meta/meta220.go
+++ b/cmd/tools/migration/meta/meta220.go
@@ -13,23 +13,35 @@ import (
"github.com/milvus-io/milvus/pkg/util"
)
-type TtCollectionsMeta220 map[UniqueID]map[Timestamp]*model.Collection // coll_id -> ts -> coll
-type CollectionsMeta220 map[UniqueID]*model.Collection // coll_id -> coll
+type (
+ TtCollectionsMeta220 map[UniqueID]map[Timestamp]*model.Collection // coll_id -> ts -> coll
+ CollectionsMeta220 map[UniqueID]*model.Collection // coll_id -> coll
+)
-type TtAliasesMeta220 map[string]map[Timestamp]*model.Alias // alias name -> ts -> coll
-type AliasesMeta220 map[string]*model.Alias // alias name -> coll
+type (
+ TtAliasesMeta220 map[string]map[Timestamp]*model.Alias // alias name -> ts -> coll
+ AliasesMeta220 map[string]*model.Alias // alias name -> coll
+)
-type TtPartitionsMeta220 map[UniqueID]map[Timestamp][]*model.Partition // coll_id -> ts -> partitions
-type PartitionsMeta220 map[UniqueID][]*model.Partition // coll_id -> ts -> partitions
+type (
+ TtPartitionsMeta220 map[UniqueID]map[Timestamp][]*model.Partition // coll_id -> ts -> partitions
+ PartitionsMeta220 map[UniqueID][]*model.Partition // coll_id -> ts -> partitions
+)
-type TtFieldsMeta220 map[UniqueID]map[Timestamp][]*model.Field // coll_id -> ts -> fields
-type FieldsMeta220 map[UniqueID][]*model.Field // coll_id -> ts -> fields
+type (
+ TtFieldsMeta220 map[UniqueID]map[Timestamp][]*model.Field // coll_id -> ts -> fields
+ FieldsMeta220 map[UniqueID][]*model.Field // coll_id -> ts -> fields
+)
-type CollectionIndexesMeta220 map[UniqueID]map[UniqueID]*model.Index // coll_id -> index_id -> index
-type SegmentIndexesMeta220 map[UniqueID]map[UniqueID]*model.SegmentIndex // seg_id -> index_id -> segment index
+type (
+ CollectionIndexesMeta220 map[UniqueID]map[UniqueID]*model.Index // coll_id -> index_id -> index
+ SegmentIndexesMeta220 map[UniqueID]map[UniqueID]*model.SegmentIndex // seg_id -> index_id -> segment index
+)
-type CollectionLoadInfo220 map[UniqueID]*model.CollectionLoadInfo // collectionID -> CollectionLoadInfo
-type PartitionLoadInfo220 map[UniqueID]map[UniqueID]*model.PartitionLoadInfo // collectionID, partitionID -> PartitionLoadInfo
+type (
+ CollectionLoadInfo220 map[UniqueID]*model.CollectionLoadInfo // collectionID -> CollectionLoadInfo
+ PartitionLoadInfo220 map[UniqueID]map[UniqueID]*model.PartitionLoadInfo // collectionID, partitionID -> PartitionLoadInfo
+)
func (meta *TtCollectionsMeta220) GenerateSaves(sourceVersion semver.Version) (map[string]string, error) {
saves := make(map[string]string)
diff --git a/cmd/tools/migration/migration/210_to_220.go b/cmd/tools/migration/migration/210_to_220.go
index edb28f2673fbb..79aaba7a5f44a 100644
--- a/cmd/tools/migration/migration/210_to_220.go
+++ b/cmd/tools/migration/migration/210_to_220.go
@@ -4,8 +4,7 @@ import (
"github.com/milvus-io/milvus/cmd/tools/migration/meta"
)
-type migrator210To220 struct {
-}
+type migrator210To220 struct{}
func (m migrator210To220) Migrate(metas *meta.Meta) (*meta.Meta, error) {
return meta.From210To220(metas)
diff --git a/cmd/tools/migration/migration/migrator.go b/cmd/tools/migration/migration/migrator.go
index e220f71dca82d..c02f35c343fa6 100644
--- a/cmd/tools/migration/migration/migrator.go
+++ b/cmd/tools/migration/migration/migrator.go
@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/blang/semver/v4"
+
"github.com/milvus-io/milvus/cmd/tools/migration/meta"
"github.com/milvus-io/milvus/cmd/tools/migration/versions"
)
diff --git a/cmd/tools/migration/migration/runner.go b/cmd/tools/migration/migration/runner.go
index 204c0c2d76841..87d8d664dbfdf 100644
--- a/cmd/tools/migration/migration/runner.go
+++ b/cmd/tools/migration/migration/runner.go
@@ -7,20 +7,15 @@ import (
"sync"
"time"
- "github.com/milvus-io/milvus/internal/util/sessionutil"
- "go.uber.org/atomic"
-
- "github.com/milvus-io/milvus/cmd/tools/migration/versions"
-
"github.com/blang/semver/v4"
+ clientv3 "go.etcd.io/etcd/client/v3"
+ "go.uber.org/atomic"
+ "github.com/milvus-io/milvus/cmd/tools/migration/backend"
"github.com/milvus-io/milvus/cmd/tools/migration/configs"
-
"github.com/milvus-io/milvus/cmd/tools/migration/console"
-
- "github.com/milvus-io/milvus/cmd/tools/migration/backend"
- clientv3 "go.etcd.io/etcd/client/v3"
-
+ "github.com/milvus-io/milvus/cmd/tools/migration/versions"
+ "github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/util/etcd"
)
diff --git a/cmd/tools/migration/utils/util.go b/cmd/tools/migration/utils/util.go
index 6fa47a4f8997f..e9dc2caa5776e 100644
--- a/cmd/tools/migration/utils/util.go
+++ b/cmd/tools/migration/utils/util.go
@@ -5,13 +5,14 @@ import (
"strconv"
"strings"
- "github.com/milvus-io/milvus/pkg/util/typeutil"
-
"github.com/milvus-io/milvus/internal/metastore/kv/rootcoord"
+ "github.com/milvus-io/milvus/pkg/util/typeutil"
)
-type UniqueID = typeutil.UniqueID
-type Timestamp = typeutil.Timestamp
+type (
+ UniqueID = typeutil.UniqueID
+ Timestamp = typeutil.Timestamp
+)
type errNotOfTsKey struct {
key string
diff --git a/configs/milvus.yaml b/configs/milvus.yaml
index eff16f90ed9ec..adea48aee559e 100644
--- a/configs/milvus.yaml
+++ b/configs/milvus.yaml
@@ -45,9 +45,19 @@ etcd:
metastore:
# Default value: etcd
- # Valid values: etcd
+ # Valid values: [etcd, tikv]
type: etcd
+# Related configuration of tikv, used to store Milvus metadata.
+# Notice that when TiKV is enabled for metastore, you still need to have etcd for service discovery.
+# TiKV is a good option when the metadata size requires better horizontal scalability.
+tikv:
+ # Note that the default pd port of tikv is 2379, which conflicts with etcd.
+ endpoints: 127.0.0.1:2389
+ rootPath: by-dev # The root path where data is stored
+ metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath
+ kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath
+
localStorage:
path: /tmp/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/
@@ -84,6 +94,8 @@ minio:
region: ""
# Cloud whether use virtual host bucket mode
useVirtualHost: false
+ # timeout for request time in milliseconds
+ requestTimeoutMs: 3000
# Milvus supports four MQ: rocksmq(based on RockDB), natsmq(embedded nats-server), Pulsar and Kafka.
# You can change your mq by setting mq.type field.
@@ -104,6 +116,7 @@ pulsar:
tenant: public
namespace: default
requestTimeout: 60 # pulsar client global request timeout in seconds
+ enableClientMetrics: false # Whether to register pulsar client metrics into milvus metrics path.
# If you want to enable kafka, needs to comment the pulsar configs
# kafka:
@@ -112,6 +125,7 @@ pulsar:
# saslPassword:
# saslMechanisms: PLAIN
# securityProtocol: SASL_SSL
+# readTimeout: 10 # read message timeout in seconds
rocksmq:
# The path where the message is stored in rocksmq
@@ -157,6 +171,9 @@ rootCoord:
importTaskExpiration: 900 # (in seconds) Duration after which an import task will expire (be killed). Default 900 seconds (15 minutes).
importTaskRetention: 86400 # (in seconds) Milvus will keep the record of import tasks for at least `importTaskRetention` seconds. Default 86400, seconds (24 hours).
enableActiveStandby: false
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 53100
grpc:
serverMaxSendSize: 536870912
@@ -188,6 +205,9 @@ proxy:
http:
enabled: true # Whether to enable the http server
debug_mode: false # Whether to enable http server debug mode
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 19530
internalPort: 19529
grpc:
@@ -214,6 +234,9 @@ queryCoord:
heartbeatAvailableInterval: 10000 # 10s, Only QueryNodes which fetched heartbeats within the duration are available
loadTimeoutSeconds: 600
checkHandoffInterval: 5000
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 19531
grpc:
serverMaxSendSize: 536870912
@@ -248,8 +271,9 @@ queryNode:
enableDisk: false # enable querynode load disk index, and search on disk index
maxDiskUsagePercentage: 95
cache:
- enabled: true
- memoryLimit: 2147483648 # 2 GB, 2 * 1024 *1024 *1024
+ enabled: true # deprecated, TODO: remove it
+ memoryLimit: 2147483648 # 2 GB, 2 * 1024 *1024 *1024 # deprecated, TODO: remove it
+ readAheadPolicy: willneed # The read ahead policy of chunk cache, options: `normal, random, sequential, willneed, dontneed`
grouping:
enabled: true
maxNQ: 1000
@@ -281,6 +305,9 @@ queryNode:
enableCrossUserGrouping: false # false by default Enable Cross user grouping when using user-task-polling policy. (close it if task of any user can not merge others).
maxPendingTaskPerUser: 1024 # 50 by default, max pending task in scheduler per user.
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 21123
grpc:
serverMaxSendSize: 536870912
@@ -302,6 +329,9 @@ indexNode:
buildParallel: 1
enableDisk: true # enable index node build disk vector index
maxDiskUsagePercentage: 95
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 21121
grpc:
serverMaxSendSize: 536870912
@@ -354,6 +384,9 @@ dataCoord:
missingTolerance: 3600 # file meta missing tolerance duration in seconds, 3600
dropTolerance: 10800 # file belongs to dropped entity tolerance duration in seconds. 10800
enableActiveStandby: false
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 13333
grpc:
serverMaxSendSize: 536870912
@@ -371,6 +404,9 @@ dataNode:
insertBufSize: 16777216 # Max buffer size to flush for a single segment.
deleteBufBytes: 67108864 # Max buffer size to flush del for a single channel
syncPeriod: 600 # The period to sync segments if buffer is not empty.
+ # can specify ip for example
+ # ip: 127.0.0.1
+ ip: # if not specify address, will use the first unicastable address as local ip
port: 21124
grpc:
serverMaxSendSize: 536870912
@@ -384,6 +420,11 @@ dataNode:
watermarkCluster: 0.5 # memory watermark for cluster, upon reaching this watermark, segments will be synced.
timetick:
byRPC: true
+ channel:
+ # specify the size of global work pool of all channels
+ # if this parameter <= 0, will set it as the maximum number of CPUs that can be executing
+ # suggest to set it bigger on large collection numbers to avoid blocking
+ workPoolSize: -1
# Configures the system log output.
log:
@@ -402,14 +443,14 @@ grpc:
serverMaxSendSize: 536870912
serverMaxRecvSize: 536870912
client:
- compressionEnabled: false
+ compressionEnabled: true
dialTimeout: 200
keepAliveTime: 10000
keepAliveTimeout: 20000
- maxMaxAttempts: 5
- initialBackoff: 1
- maxBackoff: 10
- backoffMultiplier: 2
+ maxMaxAttempts: 10
+ initialBackOff: 0.2 # seconds
+ maxBackoff: 10 # seconds
+ backoffMultiplier: 2.0 # deprecated
clientMaxSendSize: 268435456
clientMaxRecvSize: 268435456
@@ -425,6 +466,7 @@ common:
rootCoordTimeTick: rootcoord-timetick
rootCoordStatistics: rootcoord-statistics
rootCoordDml: rootcoord-dml
+ replicateMsg: replicate-msg
rootCoordDelta: rootcoord-delta
search: search
searchResult: searchResult
@@ -492,6 +534,7 @@ common:
threshold:
info: 500 # minimum milliseconds for printing durations in info level
warn: 1000 # minimum milliseconds for printing durations in warn level
+ ttMsgEnabled: true # Whether the instance disable sending ts messages
# QuotaConfig, configurations of Milvus quota and limits.
# By default, we enable:
diff --git a/docker-compose.yml b/docker-compose.yml
index 6166fe8c7ba2e..c8b0b8f7a19e9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -28,6 +28,7 @@ services:
ETCD_ENDPOINTS: ${ETCD_ENDPOINTS}
MINIO_ADDRESS: ${MINIO_ADDRESS}
CONAN_USER_HOME: /home/milvus
+ AZURE_STORAGE_CONNECTION_STRING: ${AZURITE_CONNECTION_STRING}
volumes: &builder-volumes
- .:/go/src/github.com/milvus-io/milvus:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache:/ccache:delegated
@@ -39,6 +40,7 @@ services:
- etcd
- minio
- pulsar
+ - azurite
# Command
command: &builder-command >
/bin/bash -c "
@@ -64,6 +66,7 @@ services:
ETCD_ENDPOINTS: ${ETCD_ENDPOINTS}
MINIO_ADDRESS: ${MINIO_ADDRESS}
CONAN_USER_HOME: /home/milvus
+ AZURE_STORAGE_CONNECTION_STRING: ${AZURITE_CONNECTION_STRING}
volumes: &builder-volumes-gpu
- .:/go/src/github.com/milvus-io/milvus:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-ccache:/ccache:delegated
@@ -75,6 +78,7 @@ services:
- etcd
- minio
- pulsar
+ - azurite
# Command
command: &builder-command-gpu >
/bin/bash -c "
@@ -110,6 +114,10 @@ services:
timeout: 20s
retries: 3
+ azurite:
+ image: mcr.microsoft.com/azure-storage/azurite
+ command: azurite-blob --blobHost 0.0.0.0
+
jaeger:
image: jaegertracing/all-in-one:latest
diff --git a/docs/design_docs/20230918-datanode_remove_datacoord_dependency.md b/docs/design_docs/20230918-datanode_remove_datacoord_dependency.md
new file mode 100644
index 0000000000000..338aaa986ecbc
--- /dev/null
+++ b/docs/design_docs/20230918-datanode_remove_datacoord_dependency.md
@@ -0,0 +1,121 @@
+# MEP: Datanode remove dependency of `Datacoord`
+
+Current state: "Accepted"
+
+ISSUE: https://github.com/milvus-io/milvus/issues/26758
+
+Keywords: datacoord, datanode, flush, dependency, roll-upgrade
+
+## Summary
+
+Remove the dependency of `Datacoord` for `Datanodes`.
+
+## Motivation
+
+1. Datanodes shall be always be running even when the data coordinator is not alive
+
+If datanodes performs `sync` during rolling upgrade, it needs datacoord to change the related meta in metastore. If datacoord happens to be offline or it is during some period of rolling-upgrade, datanode has to panic to ensure there is no data lost.
+
+2. Flush operation is complex and error-prone due since the whole procedure involves datacoord, datanodes and grpc
+
+This proposal means to remove the dependency of datacoord ensuring:
+
+- the data is integrate and no duplicate data is kept in records
+- no compatibility issue during or after rolling upgrade
+- `Datacoord` shall be able to detect the segment meta updates and provides recent targets for `QueryCoord`
+
+## Design Details
+
+The most brief description if this proposal is to:
+
+- Make `Datanode` operating the segment meta directly
+- Make `Datacoord` refresh the latest segment change periodically
+
+
+### Preventing multiple writers
+
+There is a major concern that if multiple `Datanodes` are handling the same dml channel, there shall be only one `DataNode` could update segment meta successfully.
+
+This guarantee is previously implemented by singleton writer in `Datacoord`: it checks the valid watcher id before update the segment meta when receiving the `SaveBinlogPaths` grpc call.
+
+In this proposal, `DataNodes` update segment meta on its own, so we need to introduce a new mechanism to prevent this error from happening:
+
+{% note %}
+
+**Note:** Like the "etcd lease for key", the ownership of each dml channel is bound to a lease id. This lease id shall be recorded in metastore (etcd/tikv or any other implementation).
+When a `DataNode` start to watch a dml channel, it shall read this lease id (via etcd or grpc call). ANY operations on this dml channel shall under a transaction with the lease id is equal to previously read value.
+If a `datanode` finds the lease id is revoke or updated, it shall close the flowgraph/pipeline and cancel all pending operations instead of panicking.
+
+{% endnote %}
+
+- [] Add lease id field in etcd channel watch info/ grpc watch request
+- [] Add `TransactionIf` like APIs in `TxnKV` interface
+
+### Updating channel checkpoint
+
+Likewise, all channel checkpoints update operations are performed by `Datacoord` invoking by grpc calls from `DataNodes`. So it has the same problem in previously stated scenarios.
+
+So, "updating channel checkpoint" shall also be processed in `DataNodes` while removing the dependency of `DataCoord`.
+
+The rules system shall follow is:
+
+{% note %}
+
+**Note:** Segments meta shall be updated *BEFORE* changing the channel checkpoint in case of datanode crashing during the prodedure. Under this premise, reconsuming from the old checkpoint shall recover all the data and duplidated entires will be discarded by segment checkpoints.
+
+{% endnote %}
+
+### Updating segment status in `DataCoord`
+
+As previous described, `DataCoord` shall refresh the segment meta and channel checkpoint periodically to provide recent target for `QueryCoord`.
+
+The `watching via Etcd` strategy is ruled out first since `Watch` operation shall avoided in the future design: currently Milvus system tends to not use `Watch` operation and try to remove it from metastore.
+Also `Watch` is heavy and has caused lots of issue before.
+
+The winning option is to:
+
+{% note %}
+
+**Note:** `Datacoord` reloads from metastore periodically.
+Optimization 1: reload channel checkpoint first, then reload segment meta if newly read revision is greater than in-memory one.
+Optimization 2: After `L0 segemnt` is implemented, datacoord shall refresh growing segments only.
+
+{% endnote %}
+
+
+## Compatibility, Deprecation, and Migration Plan
+
+This change shall guarantee that:
+
+- When new `Datacoord` starts, it shall be able to upgrade the old watch info and add lease id into it
+ - For watch info, release then watch
+ - For grpc, `release then watch` is the second choice, try call watch with lease id
+- Older `DataNodes` could invoking `SaveBinlogPaths` and other legacy grpc calls without panicking
+- The new `DataNodes` receiving old watch request(without lease id) shall fallback to older strategy, which is to update meta via grpc
+- `SaveBinlogPaths`, `UpdateChannelCheckpoints` APIs shall be kept until next break change
+
+## Test Plan
+
+### Unit test
+Coverage over 90%
+
+### Integration Test
+
+#### Datacoord offline
+
+1. Insert data without datanodes online
+2. Start datanodes
+3. Make datacoord go offline after channel assignment
+4. Assert no datanode panicking and all data shall be intact
+5. Bring back datacoord and test `GetRecoveryInfo`, which shall returns latest target
+
+
+#### Compatibility
+
+1. Start mock datacoord
+2. construct a watch info (without lease)
+3. Datanode start to watch dml channel and all meta update shall be performed via grpc
+
+## Rejected Alternatives
+
+DataCoord refresh meta via Etcd watch
diff --git a/go.mod b/go.mod
index 9f7771d16642e..3501b30ecb576 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,12 @@ module github.com/milvus-io/milvus
go 1.18
require (
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
+ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
github.com/aliyun/credentials-go v1.2.7
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e
- github.com/antonmedv/expr v1.8.9
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7
github.com/bits-and-blooms/bloom/v3 v3.0.1
github.com/blang/semver/v4 v4.0.0
@@ -20,7 +23,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.7
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
- github.com/milvus-io/milvus-proto/go-api/v2 v2.3.1-0.20230907032509-23756009c643
+ github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8
github.com/milvus-io/milvus/pkg v0.0.1
github.com/minio/minio-go/v7 v7.0.61
github.com/prometheus/client_golang v1.14.0
@@ -35,6 +38,7 @@ require (
github.com/stretchr/testify v1.8.4
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.14.4
+ github.com/tikv/client-go/v2 v2.0.4
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5
@@ -44,11 +48,11 @@ require (
go.uber.org/atomic v1.11.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.24.0
- golang.org/x/crypto v0.11.0
+ golang.org/x/crypto v0.14.0
golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691
golang.org/x/oauth2 v0.8.0
golang.org/x/sync v0.3.0
- golang.org/x/text v0.11.0
+ golang.org/x/text v0.13.0
google.golang.org/grpc v1.57.0
google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f
stathat.com/c/consistent v1.0.0
@@ -57,6 +61,7 @@ require (
require (
github.com/apache/arrow/go/v12 v12.0.0-20230223012627-e0e740bd7a24
github.com/milvus-io/milvus-storage/go v0.0.0-20231017063757-b4720fe2ec8f
+ github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
)
require (
@@ -65,6 +70,7 @@ require (
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.1 // indirect
github.com/AthenZ/athenz v1.10.39 // indirect
+ github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/DataDog/zstd v1.5.0 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
@@ -88,8 +94,10 @@ require (
github.com/containerd/cgroups v1.1.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
@@ -111,6 +119,7 @@ require (
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+ github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/uuid v1.3.0 // indirect
@@ -127,6 +136,7 @@ require (
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
+ github.com/kylelemons/godebug v1.1.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/linkedin/goavro/v2 v2.11.1 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
@@ -149,16 +159,22 @@ require (
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
+ github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/panjf2000/ants/v2 v2.7.2 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
+ github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
+ github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
+ github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
+ github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/procfs v0.9.0 // indirect
+ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/shirou/gopsutil/v3 v3.22.9 // indirect
@@ -167,15 +183,19 @@ require (
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
+ github.com/stathat/consistent v1.0.0 // indirect
github.com/streamnative/pulsarctl v0.5.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
+ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
+ github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+ github.com/twmb/murmur3 v1.1.3 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
@@ -197,9 +217,9 @@ require (
go.uber.org/automaxprocs v1.5.2 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/mod v0.12.0 // indirect
- golang.org/x/net v0.12.0 // indirect
- golang.org/x/sys v0.10.0 // indirect
- golang.org/x/term v0.10.0 // indirect
+ golang.org/x/net v0.17.0 // indirect
+ golang.org/x/sys v0.13.0 // indirect
+ golang.org/x/term v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.11.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
diff --git a/go.sum b/go.sum
index 232315ee7f40b..8e5e5d28b0e25 100644
--- a/go.sum
+++ b/go.sum
@@ -49,12 +49,22 @@ github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwR
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY=
github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
@@ -82,11 +92,10 @@ github.com/aliyun/credentials-go v1.2.7 h1:gLtFylxLZ1TWi1pStIt1O6a53GFU1zkNwjtJi
github.com/aliyun/credentials-go v1.2.7/go.mod h1:/KowD1cfGSLrLsH28Jr8W+xwoId0ywIy5lNzDz6O1vw=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e h1:GCzyKMDDjSGnlpl3clrdAK7I1AaVoaiKDOYkUzChZzg=
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
-github.com/antonmedv/expr v1.8.9 h1:O9stiHmHHww9b4ozhPx7T6BK7fXfOCHJ8ybxf0833zw=
-github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
github.com/apache/arrow/go/v12 v12.0.0-20230223012627-e0e740bd7a24 h1:3klg6Gtrm0jGkiXWLYricKhI1pYYFuBFXhGzOT5B1eo=
github.com/apache/arrow/go/v12 v12.0.0-20230223012627-e0e740bd7a24/go.mod h1:3JcT3bSZFdc7wLPKSlQXhf3L0GjPz0TOmLlG1YXnBfU=
github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg=
@@ -186,19 +195,22 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
+github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=
github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
-github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
+github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -244,8 +256,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
github.com/getsentry/sentry-go v0.12.0 h1:era7g0re5iY13bHSdN/xMkyV+5zZppjRVQhZrXCaEIk=
github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c=
@@ -314,6 +324,8 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
@@ -414,6 +426,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
@@ -524,6 +537,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kris-nova/logger v0.0.0-20181127235838-fd0d87064b06 h1:vN4d3jSss3ExzUn2cE0WctxztfOgiKvMKnDrydBsg00=
github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b h1:xYEM2oBUhBEhQjrV+KJ9lEWDWYZoNVZUaBF++Wyljq4=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
@@ -535,8 +550,6 @@ github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF
github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/linkedin/goavro/v2 v2.11.1 h1:4cuAtbDfqkKnBXp9E+tRkIJGa6W6iAjwonwt8O1f4U0=
github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
-github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
-github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -556,9 +569,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
-github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
@@ -570,18 +581,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.1-0.20230907032509-23756009c643 h1:3MXEYckliGnyepZeLDrhn+speelsoRKU1IwD8JrxXMo=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.1-0.20230907032509-23756009c643/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008080526-5593abc6e20e h1:p7Gzp5fAePnDd+ikHek4u9OhAewKazBVI720uP6QtgY=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008080526-5593abc6e20e/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008084415-18e12222e1e0 h1:tRuzrCHnFXxYpcGd4VebQoqTmty69A8IrR8ZYgMM9Dk=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008084415-18e12222e1e0/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008085610-f9cad594aa32 h1:cfUa5LlfXguXubqS+6Lv1UaV0Qs9jYHkC0VWBQ/d6Pw=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008085610-f9cad594aa32/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008092056-1a44517beb7d h1:3rjVfWdW/NvwBHWa6H6c3XJQKW4t1VODvybxZfIe/sM=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231008092056-1a44517beb7d/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231009032726-c040a793ebf4 h1:oXPeTLfI7BEdBJHKXdnKx7ACv8jW7H0Uo0Mo4h57d04=
-github.com/milvus-io/milvus-storage/go v0.0.0-20231009032726-c040a793ebf4/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8 h1:GoGErEOhdWjwSfQilXso3eINqb11yEBDLtoBMNdlve0=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-storage/go v0.0.0-20231017063757-b4720fe2ec8f h1:NGI0nposnZ2S0K1W/pEWRNzDkaLraaI10aMgKCMnVYs=
github.com/milvus-io/milvus-storage/go v0.0.0-20231017063757-b4720fe2ec8f/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
@@ -655,6 +656,7 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/panjf2000/ants/v2 v2.7.2 h1:2NUt9BaZFO5kQzrieOmK/wdb/tQ/K+QHaxN8sOgD63U=
github.com/panjf2000/ants/v2 v2.7.2/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
@@ -671,16 +673,27 @@ github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUM
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
+github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
+github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
+github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
+github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
+github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a h1:LzIZsQpXQlj8yF7+yvyOg680OaPq7bmPuDuszgXfHsw=
+github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY=
+github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
+github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
-github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -692,6 +705,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
@@ -718,8 +732,8 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
-github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -739,13 +753,13 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.27.0 h1:GOyDWxsblvqYobqsmUuMddPa2/mMzkKyojlXol4+LaQ=
github.com/samber/lo v1.27.0/go.mod h1:it33p9UtPMS7z72fP4gw/EIfQB2eI8ke7GR2wc6+Rhg=
-github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
github.com/sbinet/npyio v0.6.0 h1:IyqqQIzRjDym9xnIXsToCKei/qCzxDP+Y74KoMlMgXo=
github.com/sbinet/npyio v0.6.0/go.mod h1:/q3BNr6dJOy+t6h7RZchTJ0nwRJO52mivaem29WE1j8=
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA=
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -784,6 +798,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
+github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
+github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@@ -791,7 +807,6 @@ github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -809,12 +824,18 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
+github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
+github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2Xxo=
+github.com/tikv/client-go/v2 v2.0.4/go.mod h1:v52O5zDtv2BBus4lm5yrSQhxGW4Z4RaXWfg0U1Kuyqo=
+github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
+github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
@@ -824,6 +845,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
+github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@@ -920,17 +943,23 @@ go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
+go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
+go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
@@ -950,8 +979,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1022,6 +1051,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1052,8 +1082,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1101,7 +1131,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1152,6 +1181,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1164,12 +1194,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1179,8 +1209,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1212,6 +1242,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1250,6 +1281,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1305,6 +1337,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@@ -1354,6 +1387,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -1374,6 +1408,7 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
diff --git a/internal/allocator/cached_allocator.go b/internal/allocator/cached_allocator.go
index 6308070038b7f..fe63b2f3c3eee 100644
--- a/internal/allocator/cached_allocator.go
+++ b/internal/allocator/cached_allocator.go
@@ -23,8 +23,9 @@ import (
"time"
"github.com/cockroachdb/errors"
- "github.com/milvus-io/milvus/pkg/log"
"go.uber.org/zap"
+
+ "github.com/milvus-io/milvus/pkg/log"
)
const (
diff --git a/internal/allocator/global_id_allocator_test.go b/internal/allocator/global_id_allocator_test.go
index f752ffb03d43b..74d856313eac9 100644
--- a/internal/allocator/global_id_allocator_test.go
+++ b/internal/allocator/global_id_allocator_test.go
@@ -17,7 +17,6 @@
package allocator
import (
- "io/ioutil"
"net/url"
"os"
"testing"
@@ -37,7 +36,7 @@ var Params paramtable.ComponentParam
var embedEtcdServer *embed.Etcd
func startEmbedEtcdServer() (*embed.Etcd, error) {
- dir, err := ioutil.TempDir(os.TempDir(), "milvus_ut")
+ dir, err := os.MkdirTemp(os.TempDir(), "milvus_ut")
if err != nil {
return nil, err
}
diff --git a/internal/allocator/id_allocator.go b/internal/allocator/id_allocator.go
index a71e7214a15b0..6ea7f8fca3885 100644
--- a/internal/allocator/id_allocator.go
+++ b/internal/allocator/id_allocator.go
@@ -22,6 +22,7 @@ import (
"time"
"github.com/cockroachdb/errors"
+
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
@@ -83,7 +84,6 @@ func (ia *IDAllocator) gatherReqIDCount() uint32 {
}
func (ia *IDAllocator) syncID() (bool, error) {
-
need := ia.gatherReqIDCount()
if need < ia.countPerRPC {
need = ia.countPerRPC
diff --git a/internal/allocator/id_allocator_test.go b/internal/allocator/id_allocator_test.go
index b56c4e5f18ad1..3d83f42944bc3 100644
--- a/internal/allocator/id_allocator_test.go
+++ b/internal/allocator/id_allocator_test.go
@@ -20,23 +20,21 @@ import (
"context"
"testing"
- "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
- "github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+
+ "github.com/milvus-io/milvus/internal/proto/rootcoordpb"
+ "github.com/milvus-io/milvus/pkg/util/merr"
)
-type mockIDAllocator struct {
-}
+type mockIDAllocator struct{}
-func (tso *mockIDAllocator) AllocID(ctx context.Context, req *rootcoordpb.AllocIDRequest) (*rootcoordpb.AllocIDResponse, error) {
+func (tso *mockIDAllocator) AllocID(ctx context.Context, req *rootcoordpb.AllocIDRequest, opts ...grpc.CallOption) (*rootcoordpb.AllocIDResponse, error) {
return &rootcoordpb.AllocIDResponse{
- Status: &commonpb.Status{
- ErrorCode: commonpb.ErrorCode_Success,
- Reason: "",
- },
- ID: int64(1),
- Count: req.Count,
+ Status: merr.Success(),
+ ID: int64(1),
+ Count: req.Count,
}, nil
}
diff --git a/internal/allocator/remote_interface.go b/internal/allocator/remote_interface.go
index 6a9cddd572e2a..2f70ef55cd9ab 100644
--- a/internal/allocator/remote_interface.go
+++ b/internal/allocator/remote_interface.go
@@ -19,9 +19,11 @@ package allocator
import (
"context"
+ "google.golang.org/grpc"
+
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
)
type remoteInterface interface {
- AllocID(ctx context.Context, req *rootcoordpb.AllocIDRequest) (*rootcoordpb.AllocIDResponse, error)
+ AllocID(ctx context.Context, req *rootcoordpb.AllocIDRequest, opts ...grpc.CallOption) (*rootcoordpb.AllocIDResponse, error)
}
diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt
index a4dc18bd758a3..f5bd368bfac11 100644
--- a/internal/core/CMakeLists.txt
+++ b/internal/core/CMakeLists.txt
@@ -29,7 +29,7 @@ if ( MILVUS_GPU_VERSION )
add_definitions(-DMILVUS_GPU_VERSION)
endif ()
-if ( USE_DYNAMIC_SIMD )
+if ( USE_DYNAMIC_SIMD )
add_definitions(-DUSE_DYNAMIC_SIMD)
endif()
@@ -151,19 +151,6 @@ if ( APPLE )
)
endif ()
-# Set SIMD to CMAKE_CXX_FLAGS
-if (OPEN_SIMD)
- message(STATUS "open simd function, CPU_ARCH:${CPU_ARCH}")
- if (${CPU_ARCH} STREQUAL "avx")
- #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftree-vectorize -mavx2 -mfma -mavx -mf16c ")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mfma -mavx -mf16c ")
- elseif (${CPU_ARCH} STREQUAL "sse")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 ")
- elseif (${CPU_ARCH} STREQUAL "arm64")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=apple-m1+crc ")
- endif()
-endif ()
-
# **************************** Coding style check tools ****************************
find_package( ClangTools )
set( BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support" )
diff --git a/internal/core/build.sh b/internal/core/build.sh
index ec918b78a67f2..051a42e5f5e4a 100755
--- a/internal/core/build.sh
+++ b/internal/core/build.sh
@@ -37,8 +37,9 @@ WITH_PROMETHEUS="ON"
CUDA_ARCH="DEFAULT"
CUSTOM_THIRDPARTY_PATH=""
BUILD_DISK_ANN="OFF"
+INDEX_ENGINE="knowhere"
-while getopts "p:t:s:f:o:ulrcghzme" arg; do
+while getopts "p:t:s:f:o:ulrcghzmex" arg; do
case $arg in
f)
CUSTOM_THIRDPARTY_PATH=$OPTARG
@@ -82,6 +83,9 @@ while getopts "p:t:s:f:o:ulrcghzme" arg; do
n)
BUILD_DISK_ANN="OFF"
;;
+ x)
+ INDEX_ENGINE=$OPTARG
+ ;;
h) # help
echo "
parameter:
@@ -154,6 +158,7 @@ CMAKE_CMD="cmake \
-DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} \
-DKNOWHERE_GPU_VERSION=${SUPPORT_GPU} \
-DBUILD_DISK_ANN=${BUILD_DISK_ANN} \
+-DINDEX_ENGINE=${INDEX_ENGINE} \
${SCRIPTS_DIR}"
echo ${CMAKE_CMD}
${CMAKE_CMD}
diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py
index e93db20872986..9bcea02577b93 100644
--- a/internal/core/conanfile.py
+++ b/internal/core/conanfile.py
@@ -13,7 +13,7 @@ class MilvusConan(ConanFile):
"snappy/1.1.9",
"lzo/2.10",
"arrow/12.0.1",
- "openssl/1.1.1q",
+ "openssl/3.1.2",
"s2n/1.3.31@milvus/dev",
"aws-c-common/0.8.2@milvus/dev",
"aws-c-compression/0.2.15@milvus/dev",
diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt
index 70bc827fde22b..bdebcedf4483f 100644
--- a/internal/core/src/CMakeLists.txt
+++ b/internal/core/src/CMakeLists.txt
@@ -23,10 +23,7 @@ endif()
include_directories(${MILVUS_ENGINE_SRC})
include_directories(${MILVUS_THIRDPARTY_SRC})
-set(FOUND_OPENBLAS "unknown")
add_subdirectory( pb )
-add_subdirectory( exceptions )
-add_subdirectory( utils )
add_subdirectory( log )
add_subdirectory( config )
add_subdirectory( common )
diff --git a/internal/core/src/common/Array.h b/internal/core/src/common/Array.h
new file mode 100644
index 0000000000000..4cf311f0ac548
--- /dev/null
+++ b/internal/core/src/common/Array.h
@@ -0,0 +1,645 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include
+#include
+
+#include
+#include
+#include
+
+#include "FieldMeta.h"
+#include "Types.h"
+
+namespace milvus {
+
+class Array {
+ public:
+ Array() = default;
+
+ ~Array() {
+ delete[] data_;
+ }
+
+ explicit Array(const ScalarArray& field_data) {
+ switch (field_data.data_case()) {
+ case ScalarArray::kBoolData: {
+ element_type_ = DataType::BOOL;
+ length_ = field_data.bool_data().data().size();
+ auto data = new bool[length_];
+ size_ = length_;
+ offsets_.reserve(length_);
+ for (int i = 0; i < length_; ++i) {
+ data[i] = field_data.bool_data().data(i);
+ offsets_.push_back(sizeof(bool) * i);
+ }
+ data_ = reinterpret_cast(data);
+ break;
+ }
+ case ScalarArray::kIntData: {
+ element_type_ = DataType::INT32;
+ length_ = field_data.int_data().data().size();
+ size_ = length_ * sizeof(int32_t);
+ data_ = new char[size_];
+ offsets_.reserve(length_);
+ for (int i = 0; i < length_; ++i) {
+ reinterpret_cast(data_)[i] =
+ field_data.int_data().data(i);
+ offsets_.push_back(sizeof(int32_t) * i);
+ }
+ break;
+ }
+ case ScalarArray::kLongData: {
+ element_type_ = DataType::INT64;
+ length_ = field_data.long_data().data().size();
+ size_ = length_ * sizeof(int64_t);
+ data_ = new char[size_];
+ offsets_.reserve(length_);
+ for (int i = 0; i < length_; ++i) {
+ reinterpret_cast(data_)[i] =
+ field_data.long_data().data(i);
+ offsets_.push_back(sizeof(int64_t) * i);
+ }
+ break;
+ }
+ case ScalarArray::kFloatData: {
+ element_type_ = DataType::FLOAT;
+ length_ = field_data.float_data().data().size();
+ size_ = length_ * sizeof(float);
+ data_ = new char[size_];
+ offsets_.reserve(length_);
+ for (int i = 0; i < length_; ++i) {
+ reinterpret_cast(data_)[i] =
+ field_data.float_data().data(i);
+ offsets_.push_back(sizeof(float) * i);
+ }
+ break;
+ }
+ case ScalarArray::kDoubleData: {
+ element_type_ = DataType::DOUBLE;
+ length_ = field_data.double_data().data().size();
+ size_ = length_ * sizeof(double);
+ data_ = new char[size_];
+ offsets_.reserve(length_);
+ for (int i = 0; i < length_; ++i) {
+ reinterpret_cast(data_)[i] =
+ field_data.double_data().data(i);
+ offsets_.push_back(sizeof(double) * i);
+ }
+ break;
+ }
+ case ScalarArray::kStringData: {
+ element_type_ = DataType::STRING;
+ length_ = field_data.string_data().data().size();
+ offsets_.reserve(length_);
+ for (int i = 0; i < length_; ++i) {
+ offsets_.push_back(size_);
+ size_ += field_data.string_data().data(i).size();
+ }
+
+ data_ = new char[size_];
+ for (int i = 0; i < length_; ++i) {
+ std::copy_n(field_data.string_data().data(i).data(),
+ field_data.string_data().data(i).size(),
+ data_ + offsets_[i]);
+ }
+ break;
+ }
+ default: {
+ // empty array
+ }
+ }
+ }
+
+ Array(char* data,
+ size_t size,
+ DataType element_type,
+ std::vector&& element_offsets)
+ : length_(element_offsets.size()),
+ size_(size),
+ offsets_(std::move(element_offsets)),
+ element_type_(element_type) {
+ delete[] data_;
+ data_ = new char[size];
+ std::copy(data, data + size, data_);
+ }
+
+ Array(const Array& array) noexcept
+ : length_{array.length_},
+ size_{array.size_},
+ element_type_{array.element_type_} {
+ delete[] data_;
+ data_ = new char[array.size_];
+ std::copy(array.data_, array.data_ + array.size_, data_);
+ offsets_ = array.offsets_;
+ }
+
+ Array&
+ operator=(const Array& array) {
+ delete[] data_;
+
+ data_ = new char[array.size_];
+ std::copy(array.data_, array.data_ + array.size_, data_);
+ length_ = array.length_;
+ size_ = array.size_;
+ offsets_ = array.offsets_;
+ element_type_ = array.element_type_;
+ return *this;
+ }
+
+ bool
+ operator==(const Array& arr) const {
+ if (element_type_ != arr.element_type_) {
+ return false;
+ }
+ if (length_ != arr.length_) {
+ return false;
+ }
+ if (length_ == 0) {
+ return true;
+ }
+ switch (element_type_) {
+ case DataType::INT64: {
+ for (int i = 0; i < length_; ++i) {
+ if (get_data(i) != arr.get_data(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::BOOL: {
+ for (int i = 0; i < length_; ++i) {
+ if (get_data(i) != arr.get_data(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::DOUBLE: {
+ for (int i = 0; i < length_; ++i) {
+ if (get_data(i) != arr.get_data(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::FLOAT: {
+ for (int i = 0; i < length_; ++i) {
+ if (get_data(i) != arr.get_data(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::INT32:
+ case DataType::INT16:
+ case DataType::INT8: {
+ for (int i = 0; i < length_; ++i) {
+ if (get_data(i) != arr.get_data(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::STRING:
+ case DataType::VARCHAR: {
+ for (int i = 0; i < length_; ++i) {
+ if (get_data(i) !=
+ arr.get_data(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ default:
+ PanicInfo(Unsupported, "unsupported element type for array");
+ }
+ }
+
+ template
+ T
+ get_data(const int index) const {
+ AssertInfo(
+ index >= 0 && index < length_,
+ fmt::format(
+ "index out of range, index={}, length={}", index, length_));
+ size_t element_length = (index == length_ - 1)
+ ? size_ - offsets_.back()
+ : offsets_[index + 1] - offsets_[index];
+ if constexpr (std::is_same_v ||
+ std::is_same_v) {
+ return T(data_ + offsets_[index], element_length);
+ }
+ if constexpr (std::is_same_v || std::is_same_v ||
+ std::is_same_v || std::is_same_v) {
+ switch (element_type_) {
+ case DataType::INT8:
+ case DataType::INT16:
+ case DataType::INT32:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ case DataType::INT64:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ case DataType::FLOAT:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ case DataType::DOUBLE:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ default:
+ PanicInfo(Unsupported,
+ "unsupported element type for array");
+ }
+ }
+ return reinterpret_cast(data_)[index];
+ }
+
+ const std::vector&
+ get_offsets() const {
+ return offsets_;
+ }
+
+ ScalarArray
+ output_data() const {
+ ScalarArray data_array;
+ switch (element_type_) {
+ case DataType::BOOL: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_bool_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::INT8:
+ case DataType::INT16:
+ case DataType::INT32: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_int_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::INT64: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_long_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::STRING:
+ case DataType::VARCHAR: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_string_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::FLOAT: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_float_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::DOUBLE: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_double_data()->add_data(element);
+ }
+ break;
+ }
+ default: {
+ // empty array
+ }
+ }
+ return data_array;
+ }
+
+ int
+ length() const {
+ return length_;
+ }
+
+ size_t
+ byte_size() const {
+ return size_;
+ }
+
+ DataType
+ get_element_type() const {
+ return element_type_;
+ }
+
+ const char*
+ data() const {
+ return data_;
+ }
+
+ bool
+ is_same_array(const proto::plan::Array& arr2) const {
+ if (arr2.array_size() != length_) {
+ return false;
+ }
+ if (length_ == 0) {
+ return true;
+ }
+ if (!arr2.same_type()) {
+ return false;
+ }
+ switch (element_type_) {
+ case DataType::BOOL: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).bool_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::INT8:
+ case DataType::INT16:
+ case DataType::INT32: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).int64_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::INT64: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).int64_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::FLOAT: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).float_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::DOUBLE: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).float_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::VARCHAR:
+ case DataType::STRING: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).string_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ default:
+ return false;
+ }
+ }
+
+ private:
+ char* data_{nullptr};
+ int length_ = 0;
+ int size_ = 0;
+ std::vector offsets_{};
+ DataType element_type_ = DataType::NONE;
+};
+
+class ArrayView {
+ public:
+ ArrayView() = default;
+
+ ArrayView(char* data,
+ size_t size,
+ DataType element_type,
+ std::vector&& element_offsets)
+ : size_(size),
+ element_type_(element_type),
+ offsets_(std::move(element_offsets)),
+ length_(element_offsets.size()) {
+ data_ = data;
+ }
+
+ template
+ T
+ get_data(const int index) const {
+ AssertInfo(
+ index >= 0 && index < length_,
+ fmt::format(
+ "index out of range, index={}, length={}", index, length_));
+ size_t element_length = (index == length_ - 1)
+ ? size_ - offsets_.back()
+ : offsets_[index + 1] - offsets_[index];
+ if constexpr (std::is_same_v ||
+ std::is_same_v) {
+ return T(data_ + offsets_[index], element_length);
+ }
+ if constexpr (std::is_same_v || std::is_same_v ||
+ std::is_same_v || std::is_same_v) {
+ switch (element_type_) {
+ case DataType::INT8:
+ case DataType::INT16:
+ case DataType::INT32:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ case DataType::INT64:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ case DataType::FLOAT:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ case DataType::DOUBLE:
+ return static_cast(
+ reinterpret_cast(data_)[index]);
+ default:
+ PanicInfo(Unsupported,
+ "unsupported element type for array");
+ }
+ }
+ return reinterpret_cast(data_)[index];
+ }
+
+ ScalarArray
+ output_data() const {
+ ScalarArray data_array;
+ switch (element_type_) {
+ case DataType::BOOL: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_bool_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::INT8:
+ case DataType::INT16:
+ case DataType::INT32: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_int_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::INT64: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_long_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::STRING:
+ case DataType::VARCHAR: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_string_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::FLOAT: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_float_data()->add_data(element);
+ }
+ break;
+ }
+ case DataType::DOUBLE: {
+ for (int j = 0; j < length_; ++j) {
+ auto element = get_data(j);
+ data_array.mutable_double_data()->add_data(element);
+ }
+ break;
+ }
+ default: {
+ // empty array
+ }
+ }
+ return data_array;
+ }
+
+ int
+ length() const {
+ return length_;
+ }
+
+ size_t
+ byte_size() const {
+ return size_;
+ }
+
+ DataType
+ get_element_type() const {
+ return element_type_;
+ }
+
+ const void*
+ data() const {
+ return data_;
+ }
+
+ bool
+ is_same_array(const proto::plan::Array& arr2) const {
+ if (arr2.array_size() != length_) {
+ return false;
+ }
+ if (!arr2.same_type()) {
+ return false;
+ }
+ switch (element_type_) {
+ case DataType::BOOL: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).bool_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::INT8:
+ case DataType::INT16:
+ case DataType::INT32: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).int64_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::INT64: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).int64_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::FLOAT: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).float_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::DOUBLE: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).float_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ case DataType::VARCHAR:
+ case DataType::STRING: {
+ for (int i = 0; i < length_; i++) {
+ auto val = get_data(i);
+ if (val != arr2.array(i).string_val()) {
+ return false;
+ }
+ }
+ return true;
+ }
+ default:
+ return length_ == 0;
+ }
+ }
+
+ private:
+ char* data_{nullptr};
+ int length_ = 0;
+ int size_ = 0;
+ std::vector offsets_{};
+ DataType element_type_ = DataType::NONE;
+};
+
+} // namespace milvus
diff --git a/internal/core/src/common/BitsetView.h b/internal/core/src/common/BitsetView.h
index 777c3ef6bb3d9..57ce86a97e912 100644
--- a/internal/core/src/common/BitsetView.h
+++ b/internal/core/src/common/BitsetView.h
@@ -22,7 +22,7 @@
#include
#include "common/Types.h"
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "knowhere/bitsetview.h"
namespace milvus {
diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt
index 8996aa734811f..7fdac692ed909 100644
--- a/internal/core/src/common/CMakeLists.txt
+++ b/internal/core/src/common/CMakeLists.txt
@@ -20,14 +20,16 @@ set(COMMON_SRC
Common.cpp
RangeSearchHelper.cpp
Tracer.cpp
- IndexMeta.cpp)
+ IndexMeta.cpp
+ EasyAssert.cpp
+)
add_library(milvus_common SHARED ${COMMON_SRC})
target_link_libraries(milvus_common
milvus_config
- milvus_utils
milvus_log
+ milvus_proto
yaml-cpp
boost_bitset_ext
simdjson
diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h
index 50a4f2dc3202d..9de3b0e48b566 100644
--- a/internal/core/src/common/Consts.h
+++ b/internal/core/src/common/Consts.h
@@ -52,3 +52,5 @@ constexpr const char* RADIUS = knowhere::meta::RADIUS;
constexpr const char* RANGE_FILTER = knowhere::meta::RANGE_FILTER;
const int64_t DEFAULT_MAX_OUTPUT_SIZE = 67108864; // bytes, 64MB
+
+const int64_t DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS = 3000;
diff --git a/internal/core/src/exceptions/EasyAssert.cpp b/internal/core/src/common/EasyAssert.cpp
similarity index 83%
rename from internal/core/src/exceptions/EasyAssert.cpp
rename to internal/core/src/common/EasyAssert.cpp
index 77d80ccd6d0c6..fceb87e35f37f 100644
--- a/internal/core/src/exceptions/EasyAssert.cpp
+++ b/internal/core/src/common/EasyAssert.cpp
@@ -16,7 +16,7 @@
#include
#include "EasyAssert.h"
-// #define BOOST_STACKTRACE_USE_BACKTRACE
+#include "fmt/core.h"
#include
#include
@@ -40,13 +40,16 @@ EasyAssertInfo(bool value,
std::string_view filename,
int lineno,
std::string_view extra_info,
- ErrorCodeEnum error_code) {
+ ErrorCode error_code) {
// enable error code
if (!value) {
std::string info;
- info += "Assert \"" + std::string(expr_str) + "\"";
- info += " at " + std::string(filename) + ":" + std::to_string(lineno) +
- "\n";
+ if (!expr_str.empty()) {
+ info += fmt::format("Assert \"{}\" at {}:{}\n",
+ expr_str,
+ std::string(filename),
+ std::to_string(lineno));
+ }
if (!extra_info.empty()) {
info += " => " + std::string(extra_info);
}
diff --git a/internal/core/src/exceptions/EasyAssert.h b/internal/core/src/common/EasyAssert.h
similarity index 53%
rename from internal/core/src/exceptions/EasyAssert.h
rename to internal/core/src/common/EasyAssert.h
index cabacdad5ceea..07a444c4a70b9 100644
--- a/internal/core/src/exceptions/EasyAssert.h
+++ b/internal/core/src/common/EasyAssert.h
@@ -18,14 +18,46 @@
#include
#include
#include
-#include
-#include
+#include
+#include
#include
#include "pb/common.pb.h"
+#include "common/type_c.h"
/* Paste this on the file if you want to debug. */
namespace milvus {
-using ErrorCodeEnum = proto::common::ErrorCode;
+enum ErrorCode {
+ Success = 0,
+ UnexpectedError = 2001,
+ NotImplemented = 2002,
+ Unsupported = 2003,
+ IndexBuildError = 2004,
+ IndexAlreadyBuild = 2005,
+ ConfigInvalid = 2006,
+ DataTypeInvalid = 2007,
+ PathInvalid = 2009,
+ PathAlreadyExist = 2010,
+ PathNotExist = 2011,
+ FileOpenFailed = 2012,
+ FileCreateFailed = 2013,
+ FileReadFailed = 2014,
+ FileWriteFailed = 2015,
+ BucketInvalid = 2016,
+ ObjectNotExist = 2017,
+ S3Error = 2018,
+ RetrieveError = 2019,
+ FieldIDInvalid = 2020,
+ FieldAlreadyExist = 2021,
+ OpTypeInvalid = 2022,
+ DataIsEmpty = 2023,
+ DataFormatBroken = 2024,
+ JsonKeyInvalid = 2025,
+ MetricTypeInvalid = 2026,
+ FieldNotLoaded = 2027,
+ ExprInvalid = 2028,
+ UnistdError = 2030,
+ KnowhereError = 2100,
+};
namespace impl {
void
EasyAssertInfo(bool value,
@@ -33,25 +65,55 @@ EasyAssertInfo(bool value,
std::string_view filename,
int lineno,
std::string_view extra_info,
- ErrorCodeEnum error_code = ErrorCodeEnum::UnexpectedError);
+ ErrorCode error_code = ErrorCode::UnexpectedError);
} // namespace impl
class SegcoreError : public std::runtime_error {
public:
- SegcoreError(ErrorCodeEnum error_code, const std::string& error_msg)
+ static SegcoreError
+ success() {
+ return {ErrorCode::Success, ""};
+ }
+
+ SegcoreError(ErrorCode error_code, const std::string& error_msg)
: std::runtime_error(error_msg), error_code_(error_code) {
}
- ErrorCodeEnum
+ ErrorCode
get_error_code() {
return error_code_;
}
+ bool
+ ok() {
+ return error_code_ == ErrorCode::Success;
+ }
+
private:
- ErrorCodeEnum error_code_;
+ ErrorCode error_code_;
};
+inline CStatus
+SuccessCStatus() {
+ return CStatus{Success, ""};
+}
+
+inline CStatus
+FailureCStatus(int code, const std::string& msg) {
+ return CStatus{code, strdup(msg.data())};
+}
+
+inline CStatus
+FailureCStatus(std::exception* ex) {
+ if (dynamic_cast(ex) != nullptr) {
+ auto segcore_error = dynamic_cast(ex);
+ return CStatus{static_cast(segcore_error->get_error_code()),
+ strdup(ex->what())};
+ }
+ return CStatus{static_cast(UnexpectedError), strdup(ex->what())};
+}
+
} // namespace milvus
#define AssertInfo(expr, info) \
@@ -65,15 +127,10 @@ class SegcoreError : public std::runtime_error {
} while (0)
#define Assert(expr) AssertInfo((expr), "")
-#define PanicInfo(info) \
- do { \
- milvus::impl::EasyAssertInfo(false, (info), __FILE__, __LINE__, ""); \
- __builtin_unreachable(); \
- } while (0)
-#define PanicCodeInfo(errcode, info) \
+#define PanicInfo(errcode, info) \
do { \
milvus::impl::EasyAssertInfo( \
- false, (info), __FILE__, __LINE__, "", errcode); \
+ false, "", __FILE__, __LINE__, (info), errcode); \
__builtin_unreachable(); \
} while (0)
diff --git a/internal/core/src/common/FieldMeta.h b/internal/core/src/common/FieldMeta.h
index 35098dcf3a8a0..c6d1775595068 100644
--- a/internal/core/src/common/FieldMeta.h
+++ b/internal/core/src/common/FieldMeta.h
@@ -21,8 +21,7 @@
#include
#include "common/Types.h"
-#include "exceptions/EasyAssert.h"
-#include "utils/Status.h"
+#include "common/EasyAssert.h"
namespace milvus {
@@ -49,8 +48,12 @@ datatype_sizeof(DataType data_type, int dim = 1) {
AssertInfo(dim % 8 == 0, "dim=" + std::to_string(dim));
return dim / 8;
}
+ case DataType::VECTOR_FLOAT16: {
+ return sizeof(float16) * dim;
+ }
default: {
- throw std::invalid_argument("unsupported data type");
+ throw SegcoreError(DataTypeInvalid,
+ fmt::format("invalid type is {}", data_type));
}
}
}
@@ -59,6 +62,8 @@ datatype_sizeof(DataType data_type, int dim = 1) {
inline std::string
datatype_name(DataType data_type) {
switch (data_type) {
+ case DataType::NONE:
+ return "none";
case DataType::BOOL:
return "bool";
case DataType::INT8:
@@ -73,6 +78,8 @@ datatype_name(DataType data_type) {
return "float";
case DataType::DOUBLE:
return "double";
+ case DataType::STRING:
+ return "string";
case DataType::VARCHAR:
return "varChar";
case DataType::ARRAY:
@@ -84,10 +91,12 @@ datatype_name(DataType data_type) {
case DataType::VECTOR_BINARY: {
return "vector_binary";
}
+ case DataType::VECTOR_FLOAT16: {
+ return "vector_float16";
+ }
default: {
- auto err_msg =
- "Unsupported DataType(" + std::to_string((int)data_type) + ")";
- PanicInfo(err_msg);
+ PanicInfo(DataTypeInvalid,
+ fmt::format("Unsupported DataType({})", data_type));
}
}
}
@@ -95,7 +104,8 @@ datatype_name(DataType data_type) {
inline bool
datatype_is_vector(DataType datatype) {
return datatype == DataType::VECTOR_BINARY ||
- datatype == DataType::VECTOR_FLOAT;
+ datatype == DataType::VECTOR_FLOAT ||
+ datatype == DataType::VECTOR_FLOAT16;
}
inline bool
@@ -120,6 +130,16 @@ datatype_is_binary(DataType datatype) {
}
}
+inline bool
+datatype_is_json(DataType datatype) {
+ return datatype == DataType::JSON;
+}
+
+inline bool
+datatype_is_array(DataType datatype) {
+ return datatype == DataType::ARRAY;
+}
+
inline bool
datatype_is_variable(DataType datatype) {
switch (datatype) {
@@ -183,6 +203,14 @@ class FieldMeta {
Assert(datatype_is_string(type_));
}
+ FieldMeta(const FieldName& name,
+ FieldId id,
+ DataType type,
+ DataType element_type)
+ : name_(name), id_(id), type_(type), element_type_(element_type) {
+ Assert(datatype_is_array(type_));
+ }
+
FieldMeta(const FieldName& name,
FieldId id,
DataType type,
@@ -231,6 +259,11 @@ class FieldMeta {
return type_;
}
+ DataType
+ get_element_type() const {
+ return element_type_;
+ }
+
bool
is_vector() const {
return datatype_is_vector(type_);
@@ -267,6 +300,7 @@ class FieldMeta {
FieldName name_;
FieldId id_;
DataType type_ = DataType::NONE;
+ DataType element_type_ = DataType::NONE;
std::optional vector_info_;
std::optional string_info_;
};
diff --git a/internal/core/src/utils/File.h b/internal/core/src/common/File.h
similarity index 96%
rename from internal/core/src/utils/File.h
rename to internal/core/src/common/File.h
index 6ce8ae5722210..3622bcbf033c0 100644
--- a/internal/core/src/utils/File.h
+++ b/internal/core/src/common/File.h
@@ -12,7 +12,7 @@
#pragma once
#include
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "fmt/core.h"
#include
#include
@@ -62,4 +62,4 @@ class File {
}
int fd_{-1};
};
-} // namespace milvus
\ No newline at end of file
+} // namespace milvus
diff --git a/internal/core/src/common/IndexMeta.h b/internal/core/src/common/IndexMeta.h
index 89ad9fd01c3b4..132c1fd34b170 100644
--- a/internal/core/src/common/IndexMeta.h
+++ b/internal/core/src/common/IndexMeta.h
@@ -21,6 +21,7 @@
#include "pb/common.pb.h"
#include "pb/segcore.pb.h"
+#include "knowhere/utils.h"
#include "Types.h"
namespace milvus {
@@ -44,6 +45,11 @@ class FieldIndexMeta {
return index_params_.at(knowhere::meta::INDEX_TYPE);
}
+ bool
+ IsFlatIndex() const {
+ return knowhere::IsFlatIndex(GetIndexType());
+ }
+
const std::map&
GetIndexParams() const {
return index_params_;
diff --git a/internal/core/src/common/Json.h b/internal/core/src/common/Json.h
index 91df2b94144b7..892652955a366 100644
--- a/internal/core/src/common/Json.h
+++ b/internal/core/src/common/Json.h
@@ -25,7 +25,7 @@
#include
#include
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "simdjson.h"
#include "fmt/core.h"
#include "simdjson/common_defs.h"
diff --git a/internal/core/src/common/LoadInfo.h b/internal/core/src/common/LoadInfo.h
index 554d70d5f5cbc..f58a002b237f6 100644
--- a/internal/core/src/common/LoadInfo.h
+++ b/internal/core/src/common/LoadInfo.h
@@ -28,6 +28,7 @@
struct FieldBinlogInfo {
int64_t field_id;
int64_t row_count = -1;
+ std::vector entries_nums;
std::vector insert_files;
};
diff --git a/internal/core/src/common/RangeSearchHelper.cpp b/internal/core/src/common/RangeSearchHelper.cpp
index 36c68f35bd627..9e51dac1e6541 100644
--- a/internal/core/src/common/RangeSearchHelper.cpp
+++ b/internal/core/src/common/RangeSearchHelper.cpp
@@ -88,7 +88,7 @@ ReGenRangeSearchResult(DatasetPtr data_set,
pq(cmp);
auto capacity = std::min(lims[i + 1] - lims[i], topk);
- for (int j = lims[i]; j < lims[i + 1]; j++) {
+ for (size_t j = lims[i]; j < lims[i + 1]; j++) {
auto curr = ResultPair(dist[j], id[j]);
if (pq.size() < capacity) {
pq.push(curr);
diff --git a/internal/core/src/common/Schema.cpp b/internal/core/src/common/Schema.cpp
index e35ff7368c2c4..7265ba10871ad 100644
--- a/internal/core/src/common/Schema.cpp
+++ b/internal/core/src/common/Schema.cpp
@@ -69,6 +69,9 @@ Schema::ParseFrom(const milvus::proto::schema::CollectionSchema& schema_proto) {
auto max_len =
boost::lexical_cast(type_map.at(MAX_LENGTH));
schema->AddField(name, field_id, data_type, max_len);
+ } else if (datatype_is_array(data_type)) {
+ schema->AddField(
+ name, field_id, data_type, DataType(child.element_type()));
} else {
schema->AddField(name, field_id, data_type);
}
diff --git a/internal/core/src/common/Schema.h b/internal/core/src/common/Schema.h
index 6e2a933095968..71187f1004564 100644
--- a/internal/core/src/common/Schema.h
+++ b/internal/core/src/common/Schema.h
@@ -41,6 +41,16 @@ class Schema {
return field_id;
}
+ FieldId
+ AddDebugField(const std::string& name,
+ DataType data_type,
+ DataType element_type) {
+ auto field_id = FieldId(debug_id);
+ debug_id++;
+ this->AddField(FieldName(name), field_id, data_type, element_type);
+ return field_id;
+ }
+
// auto gen field_id for convenience
FieldId
AddDebugField(const std::string& name,
@@ -62,6 +72,16 @@ class Schema {
this->AddField(std::move(field_meta));
}
+ // array type
+ void
+ AddField(const FieldName& name,
+ const FieldId id,
+ DataType data_type,
+ DataType element_type) {
+ auto field_meta = FieldMeta(name, id, data_type, element_type);
+ this->AddField(std::move(field_meta));
+ }
+
// string type
void
AddField(const FieldName& name,
diff --git a/internal/core/src/common/SystemProperty.cpp b/internal/core/src/common/SystemProperty.cpp
index 49f889a88c57a..ad42ea541a745 100644
--- a/internal/core/src/common/SystemProperty.cpp
+++ b/internal/core/src/common/SystemProperty.cpp
@@ -18,7 +18,7 @@
#include "SystemProperty.h"
#include "Consts.h"
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
namespace milvus {
class SystemPropertyImpl : public SystemProperty {
diff --git a/internal/core/src/common/SystemProperty.h b/internal/core/src/common/SystemProperty.h
index 373e4e42a3f4f..65800aea38dbf 100644
--- a/internal/core/src/common/SystemProperty.h
+++ b/internal/core/src/common/SystemProperty.h
@@ -18,8 +18,7 @@
#include
-#include "Types.h"
-#include "utils/Json.h"
+#include "common/Types.h"
namespace milvus {
@@ -64,3 +63,23 @@ class SystemProperty {
};
} // namespace milvus
+
+template <>
+struct fmt::formatter : formatter {
+ auto
+ format(milvus::SystemFieldType c, format_context& ctx) const {
+ string_view name = "unknown";
+ switch (c) {
+ case milvus::SystemFieldType::Invalid:
+ name = "Invalid";
+ break;
+ case milvus::SystemFieldType::RowId:
+ name = "RowId";
+ break;
+ case milvus::SystemFieldType::Timestamp:
+ name = "Timestamp";
+ break;
+ }
+ return formatter::format(name, ctx);
+ }
+};
diff --git a/internal/core/src/common/Tracer.cpp b/internal/core/src/common/Tracer.cpp
index 69fc899fc8eed..21a4c637092f2 100644
--- a/internal/core/src/common/Tracer.cpp
+++ b/internal/core/src/common/Tracer.cpp
@@ -11,6 +11,8 @@
#include "log/Log.h"
#include "Tracer.h"
+#include
+
#include "opentelemetry/exporters/ostream/span_exporter_factory.h"
#include "opentelemetry/exporters/jaeger/jaeger_exporter_factory.h"
#include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h"
@@ -103,7 +105,7 @@ thread_local std::shared_ptr local_span;
void
SetRootSpan(std::shared_ptr span) {
if (enable_trace) {
- local_span = span;
+ local_span = std::move(span);
}
}
@@ -123,7 +125,7 @@ AddEvent(std::string event_label) {
bool
isEmptyID(const uint8_t* id, int length) {
- for (size_t i = 0; i < length; i++) {
+ for (int i = 0; i < length; i++) {
if (id[i] != 0) {
return false;
}
diff --git a/internal/core/src/common/Types.h b/internal/core/src/common/Types.h
index 0846b8cb536ed..2db86a0390002 100644
--- a/internal/core/src/common/Types.h
+++ b/internal/core/src/common/Types.h
@@ -35,6 +35,7 @@
#include
#include
+#include "fmt/core.h"
#include "knowhere/binaryset.h"
#include "knowhere/comp/index_param.h"
#include "knowhere/dataset.h"
@@ -51,6 +52,33 @@ using offset_t = int32_t;
using date_t = int32_t;
using distance_t = float;
+union float16 {
+ unsigned short bits;
+ struct {
+ unsigned short mantissa : 10;
+ unsigned short exponent : 5;
+ unsigned short sign : 1;
+ } parts;
+ float16() {
+ }
+ float16(float f) {
+ unsigned int i = *(unsigned int*)&f;
+ unsigned int sign = (i >> 31) & 0x0001;
+ unsigned int exponent = ((i >> 23) & 0xff) - 127 + 15;
+ unsigned int mantissa = (i >> 13) & 0x3ff;
+ parts.sign = sign;
+ parts.exponent = exponent;
+ parts.mantissa = mantissa;
+ }
+ operator float() const {
+ unsigned int sign = parts.sign << 31;
+ unsigned int exponent = (parts.exponent - 15 + 127) << 23;
+ unsigned int mantissa = parts.mantissa << 13;
+ unsigned int bits = sign | exponent | mantissa;
+ return *(float*)&bits;
+ }
+};
+
enum class DataType {
NONE = 0,
BOOL = 1,
@@ -69,6 +97,7 @@ enum class DataType {
VECTOR_BINARY = 100,
VECTOR_FLOAT = 101,
+ VECTOR_FLOAT16 = 102,
};
using Timestamp = uint64_t; // TODO: use TiKV-like timestamp
@@ -137,6 +166,7 @@ using BinarySet = knowhere::BinarySet;
using Dataset = knowhere::DataSet;
using DatasetPtr = knowhere::DataSetPtr;
using MetricType = knowhere::MetricType;
+using IndexVersion = knowhere::IndexVersion;
// TODO :: type define milvus index type(vector index type and scalar index type)
using IndexType = knowhere::IndexType;
@@ -202,6 +232,65 @@ struct fmt::formatter : formatter {
case milvus::DataType::VECTOR_FLOAT:
name = "VECTOR_FLOAT";
break;
+ case milvus::DataType::VECTOR_FLOAT16:
+ name = "VECTOR_FLOAT16";
+ break;
+ }
+ return formatter::format(name, ctx);
+ }
+};
+
+template <>
+struct fmt::formatter : formatter {
+ auto
+ format(milvus::OpType c, format_context& ctx) const {
+ string_view name = "unknown";
+ switch (c) {
+ case milvus::OpType::Invalid:
+ name = "Invalid";
+ break;
+ case milvus::OpType::GreaterThan:
+ name = "GreaterThan";
+ break;
+ case milvus::OpType::GreaterEqual:
+ name = "GreaterEqual";
+ break;
+ case milvus::OpType::LessThan:
+ name = "LessThan";
+ break;
+ case milvus::OpType::LessEqual:
+ name = "LessEqual";
+ break;
+ case milvus::OpType::Equal:
+ name = "Equal";
+ break;
+ case milvus::OpType::NotEqual:
+ name = "NotEqual";
+ break;
+ case milvus::OpType::PrefixMatch:
+ name = "PrefixMatch";
+ break;
+ case milvus::OpType::PostfixMatch:
+ name = "PostfixMatch";
+ break;
+ case milvus::OpType::Match:
+ name = "Match";
+ break;
+ case milvus::OpType::Range:
+ name = "Range";
+ break;
+ case milvus::OpType::In:
+ name = "In";
+ break;
+ case milvus::OpType::NotIn:
+ name = "NotIn";
+ break;
+ case milvus::OpType::OpType_INT_MIN_SENTINEL_DO_NOT_USE_:
+ name = "OpType_INT_MIN_SENTINEL_DO_NOT_USE";
+ break;
+ case milvus::OpType::OpType_INT_MAX_SENTINEL_DO_NOT_USE_:
+ name = "OpType_INT_MAX_SENTINEL_DO_NOT_USE";
+ break;
}
return formatter::format(name, ctx);
}
diff --git a/internal/core/src/common/Utils.h b/internal/core/src/common/Utils.h
index d90a72deba23b..2e8e245af0c8c 100644
--- a/internal/core/src/common/Utils.h
+++ b/internal/core/src/common/Utils.h
@@ -28,7 +28,7 @@
#include "common/FieldMeta.h"
#include "common/LoadInfo.h"
#include "common/Types.h"
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "knowhere/dataset.h"
#include "knowhere/expected.h"
#include "simdjson.h"
@@ -166,46 +166,7 @@ PositivelyRelated(const knowhere::MetricType& metric_type) {
inline std::string
KnowhereStatusString(knowhere::Status status) {
- switch (status) {
- case knowhere::Status::invalid_args:
- return "invalid args";
- case knowhere::Status::invalid_param_in_json:
- return "invalid param in json";
- case knowhere::Status::out_of_range_in_json:
- return "out of range in json";
- case knowhere::Status::type_conflict_in_json:
- return "type conflict in json";
- case knowhere::Status::invalid_metric_type:
- return "invalid metric type";
- case knowhere::Status::empty_index:
- return "empty index";
- case knowhere::Status::not_implemented:
- return "not implemented";
- case knowhere::Status::index_not_trained:
- return "index not trained";
- case knowhere::Status::index_already_trained:
- return "index already trained";
- case knowhere::Status::faiss_inner_error:
- return "faiss inner error";
- case knowhere::Status::hnsw_inner_error:
- return "hnsw inner error";
- case knowhere::Status::malloc_error:
- return "malloc error";
- case knowhere::Status::diskann_inner_error:
- return "diskann inner error";
- case knowhere::Status::diskann_file_error:
- return "diskann file error";
- case knowhere::Status::invalid_value_in_json:
- return "invalid value in json";
- case knowhere::Status::arithmetic_overflow:
- return "arithmetic overflow";
- case knowhere::Status::raft_inner_error:
- return "raft inner error";
- case knowhere::Status::invalid_binary_set:
- return "invalid binary set";
- default:
- return "unexpected status";
- }
+ return knowhere::Status2String(status);
}
inline std::vector
diff --git a/internal/core/src/common/VectorTrait.h b/internal/core/src/common/VectorTrait.h
index bd8805bcad286..a6a899abf03fb 100644
--- a/internal/core/src/common/VectorTrait.h
+++ b/internal/core/src/common/VectorTrait.h
@@ -18,6 +18,7 @@
#include "Types.h"
#include
#include
+#include "Array.h"
namespace milvus {
@@ -35,12 +36,20 @@ class BinaryVector : public VectorTrait {
static constexpr auto metric_type = DataType::VECTOR_BINARY;
};
+class Float16Vector : public VectorTrait {
+ public:
+ using embedded_type = float16;
+ static constexpr auto metric_type = DataType::VECTOR_FLOAT16;
+};
+
template
inline constexpr int64_t
element_sizeof(int64_t dim) {
static_assert(std::is_base_of_v);
if constexpr (std::is_same_v) {
return dim * sizeof(float);
+ } else if constexpr (std::is_same_v) {
+ return dim * sizeof(float16);
} else {
return dim / 8;
}
@@ -52,7 +61,9 @@ constexpr bool IsVector = std::is_base_of_v;
template
constexpr bool IsScalar =
std::is_fundamental_v || std::is_same_v ||
- std::is_same_v || std::is_same_v;
+ std::is_same_v || std::is_same_v ||
+ std::is_same_v || std::is_same_v ||
+ std::is_same_v;
template
struct EmbeddedTypeImpl;
@@ -64,8 +75,10 @@ struct EmbeddedTypeImpl>> {
template
struct EmbeddedTypeImpl>> {
- using type =
- std::conditional_t, float, uint8_t>;
+ using type = std::conditional_t<
+ std::is_same_v,
+ float,
+ std::conditional_t, float16, uint8_t>>;
};
template
diff --git a/internal/core/src/common/binary_set_c.cpp b/internal/core/src/common/binary_set_c.cpp
index 8495ae73f6bd0..5de5fa0f18bb2 100644
--- a/internal/core/src/common/binary_set_c.cpp
+++ b/internal/core/src/common/binary_set_c.cpp
@@ -14,6 +14,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "common/EasyAssert.h"
#include "knowhere/binaryset.h"
#include "common/binary_set_c.h"
@@ -23,12 +24,12 @@ NewBinarySet(CBinarySet* c_binary_set) {
auto binary_set = std::make_unique();
*c_binary_set = binary_set.release();
auto status = CStatus();
- status.error_code = Success;
+ status.error_code = milvus::ErrorCode::Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
- status.error_code = UnexpectedError;
+ status.error_code = milvus::ErrorCode::UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
@@ -55,10 +56,10 @@ AppendIndexBinary(CBinarySet c_binary_set,
std::shared_ptr data(dup);
binary_set->Append(index_key, data, index_size);
- status.error_code = Success;
+ status.error_code = milvus::ErrorCode::Success;
status.error_msg = "";
} catch (std::exception& e) {
- status.error_code = UnexpectedError;
+ status.error_code = milvus::ErrorCode::UnexpectedError;
status.error_msg = strdup(e.what());
}
return status;
@@ -100,11 +101,11 @@ CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) {
auto binary_set = (knowhere::BinarySet*)c_binary_set;
try {
auto binary = binary_set->GetByName(key);
- status.error_code = Success;
+ status.error_code = milvus::ErrorCode::Success;
status.error_msg = "";
memcpy((uint8_t*)data, binary->data.get(), binary->size);
} catch (std::exception& e) {
- status.error_code = UnexpectedError;
+ status.error_code = milvus::ErrorCode::UnexpectedError;
status.error_msg = strdup(e.what());
}
return status;
diff --git a/internal/core/src/common/protobuf_utils.h b/internal/core/src/common/protobuf_utils.h
index 387b168de9ef7..4a6502fc4f95d 100644
--- a/internal/core/src/common/protobuf_utils.h
+++ b/internal/core/src/common/protobuf_utils.h
@@ -21,7 +21,7 @@
#include
#include "pb/schema.pb.h"
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
using std::string;
@@ -38,4 +38,4 @@ RepeatedKeyValToMap(
}
return mapping;
}
-} //namespace milvus
\ No newline at end of file
+} //namespace milvus
diff --git a/internal/core/src/common/type_c.h b/internal/core/src/common/type_c.h
index 1ab6176e0045e..e432e899e6f69 100644
--- a/internal/core/src/common/type_c.h
+++ b/internal/core/src/common/type_c.h
@@ -31,12 +31,6 @@ enum SegmentType {
typedef enum SegmentType SegmentType;
-enum ErrorCode {
- Success = 0,
- UnexpectedError = 1,
- IllegalArgument = 5,
-};
-
// pure C don't support that we use schemapb.DataType directly.
// Note: the value of all enumerations must match the corresponding schemapb.DataType.
// TODO: what if there are increments in schemapb.DataType.
@@ -56,6 +50,7 @@ enum CDataType {
BinaryVector = 100,
FloatVector = 101,
+ Float16Vector = 102,
};
typedef enum CDataType CDataType;
@@ -83,12 +78,14 @@ typedef struct CStorageConfig {
const char* access_key_value;
const char* root_path;
const char* storage_type;
+ const char* cloud_provider;
const char* iam_endpoint;
const char* log_level;
const char* region;
bool useSSL;
bool useIAM;
bool useVirtualHost;
+ int64_t requestTimeoutMs;
} CStorageConfig;
typedef struct CTraceConfig {
@@ -107,4 +104,5 @@ typedef struct CTraceContext {
} CTraceContext;
#ifdef __cplusplus
}
+
#endif
diff --git a/internal/core/src/config/CMakeLists.txt b/internal/core/src/config/CMakeLists.txt
index a167af69b34fc..36f3ccc6a785f 100644
--- a/internal/core/src/config/CMakeLists.txt
+++ b/internal/core/src/config/CMakeLists.txt
@@ -20,13 +20,8 @@ if ( EMBEDDED_MILVUS )
add_compile_definitions( EMBEDDED_MILVUS )
endif()
-set(CONFIG_SRC
- ConfigKnowhere.cpp
- )
+set(CONFIG_SRC ConfigKnowhere.cpp)
add_library(milvus_config STATIC ${CONFIG_SRC})
-target_link_libraries(milvus_config
- milvus_exceptions
- knowhere
- )
+target_link_libraries(milvus_config knowhere)
diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp
index d57681f6865c7..cfe235876221c 100644
--- a/internal/core/src/config/ConfigKnowhere.cpp
+++ b/internal/core/src/config/ConfigKnowhere.cpp
@@ -17,11 +17,11 @@
#include
#include "ConfigKnowhere.h"
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "glog/logging.h"
#include "log/Log.h"
-#include "knowhere/comp/thread_pool.h"
#include "knowhere/comp/knowhere_config.h"
+#include "knowhere/version.h"
namespace milvus::config {
@@ -40,10 +40,11 @@ KnowhereInitImpl(const char* conf_file) {
#ifdef EMBEDDED_MILVUS
// always disable all logs for embedded milvus
google::SetCommandLineOption("minloglevel", "4");
-#endif
+#else
if (conf_file != nullptr) {
gflags::SetCommandLineOption("flagfile", conf_file);
}
+#endif
};
std::call_once(init_knowhere_once_, init);
@@ -61,28 +62,39 @@ KnowhereSetSimdType(const char* value) {
} else if (strcmp(value, "avx") == 0 || strcmp(value, "sse4_2") == 0) {
simd_type = knowhere::KnowhereConfig::SimdType::SSE4_2;
} else {
- PanicInfo("invalid SIMD type: " + std::string(value));
+ PanicInfo(ConfigInvalid, "invalid SIMD type: " + std::string(value));
}
try {
return knowhere::KnowhereConfig::SetSimdType(simd_type);
} catch (std::exception& e) {
LOG_SERVER_ERROR_ << e.what();
- PanicInfo(e.what());
+ PanicInfo(ConfigInvalid, e.what());
}
}
void
KnowhereInitBuildThreadPool(const uint32_t num_threads) {
- knowhere::ThreadPool::InitGlobalBuildThreadPool(num_threads);
+ knowhere::KnowhereConfig::SetBuildThreadPoolSize(num_threads);
}
void
KnowhereInitSearchThreadPool(const uint32_t num_threads) {
- knowhere::ThreadPool::InitGlobalSearchThreadPool(num_threads);
+ knowhere::KnowhereConfig::SetSearchThreadPoolSize(num_threads);
if (!knowhere::KnowhereConfig::SetAioContextPool(num_threads)) {
- PanicInfo("Failed to set aio context pool with num_threads " +
- std::to_string(num_threads));
+ PanicInfo(ConfigInvalid,
+ "Failed to set aio context pool with num_threads " +
+ std::to_string(num_threads));
}
}
+int32_t
+GetMinimalIndexVersion() {
+ return knowhere::Version::GetMinimalVersion().VersionNumber();
+}
+
+int32_t
+GetCurrentIndexVersion() {
+ return knowhere::Version::GetCurrentVersion().VersionNumber();
+}
+
} // namespace milvus::config
diff --git a/internal/core/src/config/ConfigKnowhere.h b/internal/core/src/config/ConfigKnowhere.h
index d76a898946480..c7584f2e7d96c 100644
--- a/internal/core/src/config/ConfigKnowhere.h
+++ b/internal/core/src/config/ConfigKnowhere.h
@@ -30,4 +30,11 @@ KnowhereInitBuildThreadPool(const uint32_t);
void
KnowhereInitSearchThreadPool(const uint32_t);
+
+int32_t
+GetMinimalIndexVersion();
+
+int32_t
+GetCurrentIndexVersion();
+
} // namespace milvus::config
diff --git a/internal/core/src/exceptions/CMakeLists.txt b/internal/core/src/exceptions/CMakeLists.txt
deleted file mode 100644
index b92906ca8f2a1..0000000000000
--- a/internal/core/src/exceptions/CMakeLists.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the LF AI & Data foundation under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set(exceptions_files
- EasyAssert.cpp
-)
-
-add_library(milvus_exceptions STATIC ${exceptions_files})
-
-target_link_libraries(milvus_exceptions milvus_proto)
diff --git a/internal/core/src/index/BoolIndex.h b/internal/core/src/index/BoolIndex.h
index 3e18a127f0a18..fe9b3df66a8dc 100644
--- a/internal/core/src/index/BoolIndex.h
+++ b/internal/core/src/index/BoolIndex.h
@@ -25,7 +25,8 @@ namespace milvus::index {
using BoolIndexPtr = std::shared_ptr>;
inline BoolIndexPtr
-CreateBoolIndex(storage::FileManagerImplPtr file_manager = nullptr) {
- return std::make_unique>(file_manager);
+CreateBoolIndex(const storage::FileManagerContext& file_manager_context =
+ storage::FileManagerContext()) {
+ return std::make_unique>(file_manager_context);
}
} // namespace milvus::index
diff --git a/internal/core/src/index/CMakeLists.txt b/internal/core/src/index/CMakeLists.txt
index c429b575ddfd8..1a5dbf200a8ee 100644
--- a/internal/core/src/index/CMakeLists.txt
+++ b/internal/core/src/index/CMakeLists.txt
@@ -14,31 +14,12 @@ set(INDEX_FILES
Utils.cpp
VectorMemIndex.cpp
IndexFactory.cpp
- VectorMemNMIndex.cpp
+ VectorDiskIndex.cpp
)
-if ( BUILD_DISK_ANN STREQUAL "ON" )
- set(INDEX_FILES
- ${INDEX_FILES}
- VectorDiskIndex.cpp
- )
-endif ()
-
milvus_add_pkg_config("milvus_index")
add_library(milvus_index SHARED ${INDEX_FILES})
-set(PLATFORM_LIBS )
-if ( LINUX OR APPLE )
- set(PLATFORM_LIBS marisa)
-endif()
-if (MSYS)
- set(PLATFORM_LIBS -Wl,--allow-multiple-definition)
-endif ()
-
-target_link_libraries(milvus_index
- milvus_storage
- ${PLATFORM_LIBS}
- milvus-storage
- )
+target_link_libraries(milvus_index milvus_storage milvus-storage)
install(TARGETS milvus_index DESTINATION "${CMAKE_INSTALL_LIBDIR}")
diff --git a/internal/core/src/index/Exception.h b/internal/core/src/index/Exception.h
deleted file mode 100644
index dde9a9e24bfd6..0000000000000
--- a/internal/core/src/index/Exception.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Licensed to the LF AI & Data foundation under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include
-#include
-#include
-
-namespace milvus::index {
-
-class UnistdException : public std::runtime_error {
- public:
- explicit UnistdException(const std::string& msg) : std::runtime_error(msg) {
- }
-
- virtual ~UnistdException() {
- }
-};
-
-} // namespace milvus::index
diff --git a/internal/core/src/index/Index.h b/internal/core/src/index/Index.h
index d2accc10a2c5f..39d6011dbfb82 100644
--- a/internal/core/src/index/Index.h
+++ b/internal/core/src/index/Index.h
@@ -18,7 +18,7 @@
#include
#include
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "knowhere/comp/index_param.h"
#include "knowhere/dataset.h"
#include "common/Types.h"
diff --git a/internal/core/src/index/IndexFactory-inl.h b/internal/core/src/index/IndexFactory-inl.h
index 64e19d1e2b116..a03cc3e2d93a6 100644
--- a/internal/core/src/index/IndexFactory-inl.h
+++ b/internal/core/src/index/IndexFactory-inl.h
@@ -23,45 +23,35 @@ namespace milvus::index {
template
inline ScalarIndexPtr
-IndexFactory::CreateScalarIndex(const IndexType& index_type,
- storage::FileManagerImplPtr file_manager) {
- return CreateScalarIndexSort(file_manager);
+IndexFactory::CreateScalarIndex(
+ const IndexType& index_type,
+ const storage::FileManagerContext& file_manager_context) {
+ return CreateScalarIndexSort(file_manager_context);
}
template
inline ScalarIndexPtr
-IndexFactory::CreateScalarIndex(const IndexType& index_type,
- storage::FileManagerImplPtr file_manager,
- std::shared_ptr space) {
- return CreateScalarIndexSort(file_manager, space);
+IndexFactory::CreateScalarIndex(
+ const IndexType& index_type,
+ const storage::FileManagerContext& file_manager_context,
+ std::shared_ptr space) {
+ return CreateScalarIndexSort(file_manager_context, space);
}
-// template <>
-// inline ScalarIndexPtr
-// IndexFactory::CreateScalarIndex(const IndexType& index_type) {
-// return CreateBoolIndex();
-//}
-
template <>
inline ScalarIndexPtr
-IndexFactory::CreateScalarIndex(const IndexType& index_type,
- storage::FileManagerImplPtr file_manager) {
-#if defined(__linux__) || defined(__APPLE__)
- return CreateStringIndexMarisa(file_manager);
-#else
- throw std::runtime_error("unsupported platform");
-#endif
+IndexFactory::CreateScalarIndex(
+ const IndexType& index_type,
+ const storage::FileManagerContext& file_manager_context) {
+ return CreateStringIndexMarisa(file_manager_context);
}
template <>
inline ScalarIndexPtr
-IndexFactory::CreateScalarIndex(const IndexType& index_type,
- storage::FileManagerImplPtr file_manager,
- std::shared_ptr space) {
-#if defined(__linux__) || defined(__APPLE__)
- return CreateStringIndexMarisa(file_manager, space);
-#else
- throw std::runtime_error("unsupported platform");
-#endif
+IndexFactory::CreateScalarIndex(
+ const IndexType& index_type,
+ const storage::FileManagerContext& file_manager_context,
+ std::shared_ptr space) {
+ return CreateStringIndexMarisa(file_manager_context, space);
}
} // namespace milvus::index
diff --git a/internal/core/src/index/IndexFactory.cpp b/internal/core/src/index/IndexFactory.cpp
index 4757a06b7001e..501a3efb711ac 100644
--- a/internal/core/src/index/IndexFactory.cpp
+++ b/internal/core/src/index/IndexFactory.cpp
@@ -16,106 +16,107 @@
#include "index/IndexFactory.h"
#include "index/VectorMemIndex.h"
-#include "index/VectorMemNMIndex.h"
#include "index/Utils.h"
#include "index/Meta.h"
+#include "knowhere/utils.h"
-#ifdef BUILD_DISK_ANN
#include "index/VectorDiskIndex.h"
-#endif
namespace milvus::index {
IndexBasePtr
-IndexFactory::CreateIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager) {
+IndexFactory::CreateIndex(
+ const CreateIndexInfo& create_index_info,
+ const storage::FileManagerContext& file_manager_context) {
if (datatype_is_vector(create_index_info.field_type)) {
- return CreateVectorIndex(create_index_info, file_manager);
+ return CreateVectorIndex(create_index_info, file_manager_context);
}
- return CreateScalarIndex(create_index_info, file_manager);
+ return CreateScalarIndex(create_index_info, file_manager_context);
}
IndexBasePtr
-IndexFactory::CreateIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager,
- std::shared_ptr space) {
+IndexFactory::CreateIndex(
+ const CreateIndexInfo& create_index_info,
+ const storage::FileManagerContext& file_manager_context,
+ std::shared_ptr space) {
if (datatype_is_vector(create_index_info.field_type)) {
- return CreateVectorIndex(create_index_info, file_manager, space);
+ return CreateVectorIndex(
+ create_index_info, file_manager_context, space);
}
- return CreateScalarIndex(create_index_info, file_manager, space);
+ return CreateScalarIndex(create_index_info, file_manager_context, space);
}
IndexBasePtr
-IndexFactory::CreateScalarIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager) {
+IndexFactory::CreateScalarIndex(
+ const CreateIndexInfo& create_index_info,
+ const storage::FileManagerContext& file_manager_context) {
auto data_type = create_index_info.field_type;
auto index_type = create_index_info.index_type;
switch (data_type) {
// create scalar index
case DataType::BOOL:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
case DataType::INT8:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
case DataType::INT16:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
case DataType::INT32:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
case DataType::INT64:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
case DataType::FLOAT:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
case DataType::DOUBLE:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type, file_manager_context);
// create string index
case DataType::STRING:
case DataType::VARCHAR:
- return CreateScalarIndex(index_type, file_manager);
+ return CreateScalarIndex(index_type,
+ file_manager_context);
default:
- throw std::invalid_argument(
- std::string("invalid data type to build index: ") +
- std::to_string(int(data_type)));
+ throw SegcoreError(
+ DataTypeInvalid,
+ fmt::format("invalid data type to build index: {}", data_type));
}
}
IndexBasePtr
-IndexFactory::CreateVectorIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager) {
- auto data_type = create_index_info.field_type;
+IndexFactory::CreateVectorIndex(
+ const CreateIndexInfo& create_index_info,
+ const storage::FileManagerContext& file_manager_context) {
auto index_type = create_index_info.index_type;
auto metric_type = create_index_info.metric_type;
-
-#ifdef BUILD_DISK_ANN
+ auto version = create_index_info.index_engine_version;
// create disk index
- if (is_in_disk_list(index_type)) {
+#ifdef BUILD_DISK_ANN
+ auto data_type = create_index_info.field_type;
+ if (knowhere::UseDiskLoad(index_type, version)) {
switch (data_type) {
case DataType::VECTOR_FLOAT: {
return std::make_unique>(
- index_type, metric_type, file_manager);
+ index_type, metric_type, version, file_manager_context);
}
default:
- throw std::invalid_argument(
- std::string("invalid data type to build disk index: ") +
- std::to_string(int(data_type)));
+ throw SegcoreError(
+ DataTypeInvalid,
+ fmt::format("invalid data type to build disk index: {}",
+ data_type));
}
}
#endif
- if (is_in_nm_list(index_type)) {
- return std::make_unique(
- index_type, metric_type, file_manager);
- }
// create mem index
return std::make_unique(
- index_type, metric_type, file_manager);
+ index_type, metric_type, version, file_manager_context);
}
IndexBasePtr
IndexFactory::CreateScalarIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager,
std::shared_ptr space) {
auto data_type = create_index_info.field_type;
auto index_type = create_index_info.index_type;
@@ -150,35 +151,34 @@ IndexFactory::CreateScalarIndex(const CreateIndexInfo& create_index_info,
}
IndexBasePtr
-IndexFactory::CreateVectorIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager,
- std::shared_ptr space) {
+IndexFactory::CreateVectorIndex(
+ const CreateIndexInfo& create_index_info,
+ const storage::FileManagerContext& file_manager_context,
+ std::shared_ptr space) {
auto data_type = create_index_info.field_type;
auto index_type = create_index_info.index_type;
auto metric_type = create_index_info.metric_type;
+ auto version = create_index_info.index_engine_version;
-#ifdef BUILD_DISK_ANN
// create disk index
- if (is_in_disk_list(index_type)) {
+#ifdef BUILD_DISK_ANN
+ if (knowhere::UseDiskLoad(index_type, version)) {
switch (data_type) {
case DataType::VECTOR_FLOAT: {
return std::make_unique>(
- index_type, metric_type, space);
+ index_type, metric_type, version, file_manager_context);
}
default:
- throw std::invalid_argument(
- std::string("invalid data type to build disk index: ") +
- std::to_string(int(data_type)));
+ throw SegcoreError(
+ DataTypeInvalid,
+ fmt::format("invalid data type to build disk index: {}",
+ data_type));
}
}
#endif
- if (is_in_nm_list(index_type)) {
- return std::make_unique(
- create_index_info, file_manager, space);
- }
// create mem index
return std::make_unique(
- create_index_info, file_manager, space);
+ create_index_info, file_manager_context, space);
}
} // namespace milvus::index
diff --git a/internal/core/src/index/IndexFactory.h b/internal/core/src/index/IndexFactory.h
index 5581b007fb158..0754639422121 100644
--- a/internal/core/src/index/IndexFactory.h
+++ b/internal/core/src/index/IndexFactory.h
@@ -50,28 +50,29 @@ class IndexFactory {
IndexBasePtr
CreateIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager);
+ const storage::FileManagerContext& file_manager_context);
IndexBasePtr
CreateIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager_context,
std::shared_ptr space);
IndexBasePtr
CreateVectorIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager);
+ const storage::FileManagerContext& file_manager_context);
IndexBasePtr
CreateScalarIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager = nullptr);
+ const storage::FileManagerContext& file_manager_context =
+ storage::FileManagerContext());
IndexBasePtr
CreateVectorIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager_context,
std::shared_ptr space);
IndexBasePtr
CreateScalarIndex(const CreateIndexInfo& create_index_info,
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager_context,
std::shared_ptr space);
// IndexBasePtr
@@ -80,12 +81,12 @@ class IndexFactory {
template
ScalarIndexPtr
CreateScalarIndex(const IndexType& index_type,
- storage::FileManagerImplPtr file_manager = nullptr);
+ const storage::FileManagerContext& file_manager);
template
ScalarIndexPtr
CreateScalarIndex(const IndexType& index_type,
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager,
std::shared_ptr space);
};
diff --git a/internal/core/src/index/IndexInfo.h b/internal/core/src/index/IndexInfo.h
index 4b4b6925addde..632cc3210cffc 100644
--- a/internal/core/src/index/IndexInfo.h
+++ b/internal/core/src/index/IndexInfo.h
@@ -26,6 +26,7 @@ struct CreateIndexInfo {
MetricType metric_type;
std::string field_name;
int64_t dim;
+ IndexVersion index_engine_version;
};
} // namespace milvus::index
diff --git a/internal/core/src/index/Meta.h b/internal/core/src/index/Meta.h
index 489a8afe50c3f..77024a13aa073 100644
--- a/internal/core/src/index/Meta.h
+++ b/internal/core/src/index/Meta.h
@@ -45,10 +45,13 @@ constexpr const char* FIELD_ID = "field_id";
constexpr const char* INDEX_BUILD_ID = "index_build_id";
constexpr const char* INDEX_ID = "index_id";
constexpr const char* INDEX_VERSION = "index_version";
+constexpr const char* INDEX_ENGINE_VERSION = "index_engine_version";
-// DiskAnn build params
+// VecIndex file metas
constexpr const char* DISK_ANN_PREFIX_PATH = "index_prefix";
constexpr const char* DISK_ANN_RAW_DATA_PATH = "data_path";
+
+// DiskAnn build params
constexpr const char* DISK_ANN_MAX_DEGREE = "max_degree";
constexpr const char* DISK_ANN_SEARCH_LIST_SIZE = "search_list_size";
constexpr const char* DISK_ANN_PQ_CODE_BUDGET = "pq_code_budget_gb";
diff --git a/internal/core/src/index/ScalarIndex-inl.h b/internal/core/src/index/ScalarIndex-inl.h
index 5ddf60d2195d1..b59aadf27d47f 100644
--- a/internal/core/src/index/ScalarIndex-inl.h
+++ b/internal/core/src/index/ScalarIndex-inl.h
@@ -20,6 +20,7 @@
#include "index/Meta.h"
#include "knowhere/dataset.h"
+#include "common/Types.h"
namespace milvus::index {
template
@@ -63,8 +64,9 @@ ScalarIndex::Query(const DatasetPtr& dataset) {
case OpType::PrefixMatch:
case OpType::PostfixMatch:
default:
- throw std::invalid_argument(std::string(
- "unsupported operator type: " + std::to_string(op)));
+ throw SegcoreError(
+ OpTypeInvalid,
+ fmt::format("unsupported operator type: {}", op));
}
}
@@ -73,7 +75,6 @@ inline void
ScalarIndex::BuildWithRawData(size_t n,
const void* values,
const Config& config) {
- // TODO :: use arrow
proto::schema::StringArray arr;
auto ok = arr.ParseFromArray(values, n);
Assert(ok);
diff --git a/internal/core/src/index/ScalarIndex.h b/internal/core/src/index/ScalarIndex.h
index ec67f79b80024..3fb4c27425615 100644
--- a/internal/core/src/index/ScalarIndex.h
+++ b/internal/core/src/index/ScalarIndex.h
@@ -22,8 +22,9 @@
#include
#include "common/Types.h"
-#include "exceptions/EasyAssert.h"
+#include "common/EasyAssert.h"
#include "index/Index.h"
+#include "fmt/format.h"
namespace milvus::index {
@@ -38,7 +39,8 @@ class ScalarIndex : public IndexBase {
void
BuildWithDataset(const DatasetPtr& dataset,
const Config& config = {}) override {
- PanicInfo("scalar index don't support build index with dataset");
+ PanicInfo(Unsupported,
+ "scalar index don't support build index with dataset");
};
public:
diff --git a/internal/core/src/index/ScalarIndexSort-inl.h b/internal/core/src/index/ScalarIndexSort-inl.h
index 1759692a245ff..d1b459dd0545e 100644
--- a/internal/core/src/index/ScalarIndexSort-inl.h
+++ b/internal/core/src/index/ScalarIndexSort-inl.h
@@ -25,28 +25,31 @@
#include "Meta.h"
#include "common/Utils.h"
#include "common/Slice.h"
+#include "common/Types.h"
#include "index/Utils.h"
namespace milvus::index {
template
inline ScalarIndexSort::ScalarIndexSort(
- storage::FileManagerImplPtr file_manager)
+ const storage::FileManagerContext& file_manager_context)
: is_built_(false), data_() {
- if (file_manager != nullptr) {
- file_manager_ = std::dynamic_pointer_cast(
- file_manager);
+ if (file_manager_context.Valid()) {
+ file_manager_ =
+ std::make_shared(file_manager_context);
+ AssertInfo(file_manager_ != nullptr, "create file manager failed!");
}
}
template
inline ScalarIndexSort::ScalarIndexSort(
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager_context,
std::shared_ptr space)
: is_built_(false), data_(), space_(space) {
- if (file_manager != nullptr) {
- file_manager_ = std::dynamic_pointer_cast(
- file_manager);
+ if (file_manager_context.Valid()) {
+ file_manager_ =
+ std::make_shared(file_manager_context, space_);
+ AssertInfo(file_manager_ != nullptr, "create file manager failed!");
}
}
template
@@ -71,9 +74,8 @@ ScalarIndexSort::Build(const Config& config) {
total_num_rows += data->get_num_rows();
}
if (total_num_rows == 0) {
- // todo: throw an exception
- throw std::invalid_argument(
- "ScalarIndexSort cannot build null values!");
+ throw SegcoreError(DataIsEmpty,
+ "ScalarIndexSort cannot build null values!");
}
data_.reserve(total_num_rows);
@@ -101,9 +103,8 @@ ScalarIndexSort::Build(size_t n, const T* values) {
if (is_built_)
return;
if (n == 0) {
- // todo: throw an exception
- throw std::invalid_argument(
- "ScalarIndexSort cannot build null values!");
+ throw SegcoreError(DataIsEmpty,
+ "ScalarIndexSort cannot build null values!");
}
data_.reserve(n);
idx_to_offsets_.resize(n);
@@ -226,14 +227,14 @@ ScalarIndexSort::LoadV2(const Config& config) {
for (auto& file_name : index_files.value()) {
auto res = space_->GetBlobByteSize(file_name);
if (!res.ok()) {
- PanicCodeInfo(ErrorCodeEnum::UnexpectedError,
+ PanicInfo(S3Error,
"unable to read index blob");
}
auto index_blob_data =
std::shared_ptr(new uint8_t[res.value()]);
auto status = space_->ReadBlob(file_name, index_blob_data.get());
if (!status.ok()) {
- PanicCodeInfo(ErrorCodeEnum::UnexpectedError,
+ PanicInfo(S3Error,
"unable to read index blob");
}
auto raw_index_blob =
@@ -322,8 +323,8 @@ ScalarIndexSort::Range(const T value, const OpType op) {
data_.begin(), data_.end(), IndexStructure(value));
break;
default:
- throw std::invalid_argument(std::string("Invalid OperatorType: ") +
- std::to_string((int)op) + "!");
+ throw SegcoreError(OpTypeInvalid,
+ fmt::format("Invalid OperatorType: {}", op));
}
for (; lb < ub; ++lb) {
bitset[lb->idx_] = true;
diff --git a/internal/core/src/index/ScalarIndexSort.h b/internal/core/src/index/ScalarIndexSort.h
index 48c0bcb0f0b48..10f32d910b372 100644
--- a/internal/core/src/index/ScalarIndexSort.h
+++ b/internal/core/src/index/ScalarIndexSort.h
@@ -34,9 +34,10 @@ template
class ScalarIndexSort : public ScalarIndex {
public:
explicit ScalarIndexSort(
- storage::FileManagerImplPtr file_manager = nullptr);
+ const storage::FileManagerContext& file_manager_context =
+ storage::FileManagerContext());
- explicit ScalarIndexSort(storage::FileManagerImplPtr file_manager,
+ explicit ScalarIndexSort(const storage::FileManagerContext& file_manager_context,
std::shared_ptr space);
BinarySet
@@ -126,13 +127,14 @@ using ScalarIndexSortPtr = std::unique_ptr>;
namespace milvus::index {
template
inline ScalarIndexSortPtr
-CreateScalarIndexSort(storage::FileManagerImplPtr file_manager = nullptr) {
- return std::make_unique>(file_manager);
+CreateScalarIndexSort(const storage::FileManagerContext& file_manager_context =
+ storage::FileManagerContext()) {
+ return std::make_unique>(file_manager_context);
}
template
inline ScalarIndexSortPtr
-CreateScalarIndexSort(storage::FileManagerImplPtr file_manager,
+CreateScalarIndexSort(const storage::FileManagerContext& file_manager_context,
std::shared_ptr space) {
- return std::make_unique>(file_manager, space);
+ return std::make_unique>(file_manager_context, space);
}
} // namespace milvus::index
diff --git a/internal/core/src/index/StringIndexMarisa.cpp b/internal/core/src/index/StringIndexMarisa.cpp
index ae5c2d6708216..f24080e45bb10 100644
--- a/internal/core/src/index/StringIndexMarisa.cpp
+++ b/internal/core/src/index/StringIndexMarisa.cpp
@@ -23,10 +23,10 @@
#include
#include "common/Types.h"
+#include "common/EasyAssert.h"
#include "index/StringIndexMarisa.h"
#include "index/Utils.h"
#include "index/Index.h"
-#include "index/Exception.h"
#include "common/Utils.h"
#include "common/Slice.h"
#include "storage/Util.h"
@@ -34,22 +34,21 @@
namespace milvus::index {
-#if defined(__linux__) || defined(__APPLE__)
-
-StringIndexMarisa::StringIndexMarisa(storage::FileManagerImplPtr file_manager) {
- if (file_manager != nullptr) {
- file_manager_ = std::dynamic_pointer_cast(
- file_manager);
+StringIndexMarisa::StringIndexMarisa(
+ const storage::FileManagerContext& file_manager_context) {
+ if (file_manager_context.Valid()) {
+ file_manager_ =
+ std::make_shared(file_manager_context);
}
}
StringIndexMarisa::StringIndexMarisa(
- storage::FileManagerImplPtr file_manager,
+ const storage::FileManagerContext& file_manager_context,
std::shared_ptr space)
: space_(space) {
- if (file_manager != nullptr) {
- file_manager_ = std::dynamic_pointer_cast(
- file_manager);
+ if (file_manager_context.Valid()) {
+ file_manager_ = std::make_shared