Merge remote-tracking branch 'upstream/master' into storage_v2

.env

@@ -1,8 +1,8 @@
 IMAGE_REPO=milvusdb
 IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
-DATE_VERSION=20231011-11b5213
-LATEST_DATE_VERSION=20231011-11b5213
+DATE_VERSION=20231024-4faba61
+LATEST_DATE_VERSION=20231024-4faba61
 GPU_DATE_VERSION=20230822-a64488a
 LATEST_GPU_DATE_VERSION=20230317-a1c7b0c
 MINIO_ADDRESS=minio:9000

.github/workflows/mac.yaml

@@ -28,7 +28,7 @@ jobs:
   mac:
     name: Code Checker MacOS 12
     runs-on: macos-12
-    timeout-minutes: 300 
+    timeout-minutes: 300
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -73,7 +73,7 @@ jobs:
           fi
           ls -alh /var/tmp/ccache
           brew install libomp ninja openblas ccache pkg-config
-          pip3 install conan==1.58.0
+          pip3 install conan==1.61.0
           if [[ ! -d "/usr/local/opt/llvm" ]]; then
             ln -s /usr/local/opt/llvm@14 /usr/local/opt/llvm
           fi

DEVELOPMENT.md

@@ -118,7 +118,7 @@ Milvus uses Conan to manage third-party dependencies for c++.
 Install Conan
 
 ```shell
-pip install conan==1.58.0
+pip install conan==1.61.0
 ```
 
 Note: Conan version 2.x is not currently supported, please use version 1.58.
@@ -288,7 +288,7 @@ start the cluster on your host machine
 
 ```shell
 $ ./build/builder.sh make install // build milvus
-$ ./build/build_image.sh // build milvus lastest docker image
+$ ./build/build_image.sh // build milvus latest docker image
 $ docker images // check if milvus latest image is ready
 REPOSITORY                 TAG                                 IMAGE ID       CREATED          SIZE
 milvusdb/milvus            latest                              63c62ff7c1b7   52 minutes ago   570MB

README.md

@@ -169,7 +169,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
 ### All contributors
 
 <br><!-- Do not remove start of hero-bot -->
-<img src="https://img.shields.io/badge/all--contributors-336-orange"><br>
+<img src="https://img.shields.io/badge/all--contributors-337-orange"><br>
 <a href="https://github.com/0xflotus"><img src="https://avatars.githubusercontent.com/u/26602940?v=4" width="30px" /></a>
 <a href="https://github.com/ABNER-1"><img src="https://avatars.githubusercontent.com/u/24547351?v=4" width="30px" /></a>
 <a href="https://github.com/Accagain2014"><img src="https://avatars.githubusercontent.com/u/9635216?v=4" width="30px" /></a>
@@ -290,6 +290,7 @@ Contributions to Milvus are welcome from everyone. See [Guidelines for Contribut
 <a href="https://github.com/chenxingqiang"><img src="https://avatars.githubusercontent.com/u/12387235?v=4" width="30px" /></a>
 <a href="https://github.com/chinalu"><img src="https://avatars.githubusercontent.com/u/167307?v=4" width="30px" /></a>
 <a href="https://github.com/chinamcafee"><img src="https://avatars.githubusercontent.com/u/3439961?v=4" width="30px" /></a>
+<a href="https://github.com/christy"><img src="https://avatars.githubusercontent.com/u/94918?v=4" width="30px" /></a>
 <a href="https://github.com/chuangfengwang"><img src="https://avatars.githubusercontent.com/u/24692397?v=4" width="30px" /></a>
 <a href="https://github.com/chyezh"><img src="https://avatars.githubusercontent.com/u/20332315?v=4" width="30px" /></a>
 <a href="https://github.com/cjrh"><img src="https://avatars.githubusercontent.com/u/480395?v=4" width="30px" /></a>

README_CN.md

@@ -154,7 +154,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
 ### All contributors
 
 <br><!-- Do not remove start of hero-bot -->
-<img src="https://img.shields.io/badge/all--contributors-336-orange"><br>
+<img src="https://img.shields.io/badge/all--contributors-337-orange"><br>
 <a href="https://github.com/0xflotus"><img src="https://avatars.githubusercontent.com/u/26602940?v=4" width="30px" /></a>
 <a href="https://github.com/ABNER-1"><img src="https://avatars.githubusercontent.com/u/24547351?v=4" width="30px" /></a>
 <a href="https://github.com/Accagain2014"><img src="https://avatars.githubusercontent.com/u/9635216?v=4" width="30px" /></a>
@@ -275,6 +275,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
 <a href="https://github.com/chenxingqiang"><img src="https://avatars.githubusercontent.com/u/12387235?v=4" width="30px" /></a>
 <a href="https://github.com/chinalu"><img src="https://avatars.githubusercontent.com/u/167307?v=4" width="30px" /></a>
 <a href="https://github.com/chinamcafee"><img src="https://avatars.githubusercontent.com/u/3439961?v=4" width="30px" /></a>
+<a href="https://github.com/christy"><img src="https://avatars.githubusercontent.com/u/94918?v=4" width="30px" /></a>
 <a href="https://github.com/chuangfengwang"><img src="https://avatars.githubusercontent.com/u/24692397?v=4" width="30px" /></a>
 <a href="https://github.com/chyezh"><img src="https://avatars.githubusercontent.com/u/20332315?v=4" width="30px" /></a>
 <a href="https://github.com/cjrh"><img src="https://avatars.githubusercontent.com/u/480395?v=4" width="30px" /></a>

build/docker/builder/cpu/amazonlinux2023/Dockerfile

@@ -18,7 +18,7 @@ RUN yum install -y wget g++ gcc gdb libatomic libstdc++-static git make zip unzi
     pkg-config libuuid-devel libaio perl-IPC-Cmd && \
     rm -rf /var/cache/yum/*
 
-RUN pip3 install conan==1.58.0
+RUN pip3 install conan==1.61.0
 
 RUN echo "target arch $TARGETARCH"
 RUN wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local

build/docker/builder/cpu/ubuntu20.04/Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
     apt-get remove --purge -y && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install conan==1.58.0
+RUN pip3 install conan==1.61.0
 
 RUN echo "target arch $TARGETARCH"
 RUN wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-`uname -m`.tar.gz" | tar --strip-components=1 -xz -C /usr/local

build/docker/builder/gpu/ubuntu20.04/Dockerfile

@@ -11,7 +11,7 @@
 
 FROM nvidia/cuda:11.8.0-devel-ubuntu20.04
 
-RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 && \
+RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 ninja-build && \
     wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
     apt-get update && apt-get install -y --no-install-recommends \
     g++ gcc gfortran git make ccache libssl-dev zlib1g-dev zip unzip \
@@ -20,7 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce
     apt-get remove --purge -y && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install conan==1.58.0
+RUN pip3 install conan==1.61.0
 
 # Instal openblas
 # RUN wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \

ci/jenkins/PublishImages.groovy

@@ -27,7 +27,7 @@ pipeline {
     }
 
     stages {
-        stage('Generat Image Tag') {
+        stage('Generate Image Tag') {
             steps {
                 script {
                     def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()

configs/advanced/etcd.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 # This is the configuration file for the etcd server.
-# Only standalone users with embeded etcd should change this file, others could just keep this file As Is.
+# Only standalone users with embedded etcd should change this file, others could just keep this file As Is.
 # All the etcd client should be added to milvus.yaml if necessary
 
 # Human-readable name for this member.

configs/milvus.yaml

@@ -111,7 +111,7 @@ mq:
 pulsar:
   address: localhost # Address of pulsar
   port: 6650 # Port of Pulsar
-  webport: 80 # Web port of pulsar, if you connect direcly without proxy, should use 8080
+  webport: 80 # Web port of pulsar, if you connect directly without proxy, should use 8080
   maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.
   tenant: public
   namespace: default
@@ -346,7 +346,7 @@ dataCoord:
     balanceInterval: 360 #The interval for the channelBalancer on datacoord to check balance status
   segment:
     maxSize: 512 # Maximum size of a segment in MB
-    diskSegmentMaxSize: 2048 # Maximun size of a segment in MB for collection which has Disk index
+    diskSegmentMaxSize: 2048 # Maximum size of a segment in MB for collection which has Disk index
     sealProportion: 0.23
     # The time of the assignment expiration in ms
     # Warning! this parameter is an expert variable and closely related to data integrity. Without specific
@@ -443,7 +443,7 @@ grpc:
   serverMaxSendSize: 536870912
   serverMaxRecvSize: 536870912
   client:
-    compressionEnabled: true
+    compressionEnabled: false
     dialTimeout: 200
     keepAliveTime: 10000
     keepAliveTimeout: 20000

docs/design_docs/20210604-datanode_flowgraph_recovery_design.md

@@ -74,7 +74,7 @@ Supposing we have segments `s1, s2, s3`, corresponding positions `p1, p2, p3`
 const filter_threshold = recovery_time
 // mp means msgPack
 for mp := seeking(p1) {
-    if mp.position.endtime < filter_threshod {
+    if mp.position.endtime < filter_threshold {
         if mp.position < p3 {
             filter s3
         }

docs/design_docs/20211217-milvus_create_collection.md

@@ -86,7 +86,7 @@ type createCollectionTask struct {
    }
    ```
 
-   - `PostExecute`, `CreateCollectonTask` does nothing at this phase, and return directly.
+   - `PostExecute`, `CreateCollectionTask` does nothing at this phase, and return directly.
 
 4. `RootCoord` would wrap the `CreateCollection` request into `CreateCollectionReqTask`, and then call function `executeTask`. `executeTask` would return until the `context` is done or `CreateCollectionReqTask.Execute` is returned.
 
@@ -104,7 +104,7 @@ type CreateCollectionReqTask struct {
 }
 ```
 
-5. `CreateCollectionReqTask.Execute` would alloc `CollecitonID` and default `PartitionID`, and set `Virtual Channel` and `Physical Channel`, which are used by `MsgStream`, then write the `Collection`'s meta into `metaTable`
+5. `CreateCollectionReqTask.Execute` would alloc `CollectionID` and default `PartitionID`, and set `Virtual Channel` and `Physical Channel`, which are used by `MsgStream`, then write the `Collection`'s meta into `metaTable`
 
 6. After `Collection`'s meta written into `metaTable`, `Milvus` would consider this collection has been created successfully.
 

docs/design_docs/20220105-proxy.md

@@ -127,7 +127,7 @@ future work.
 
 For DqRequest, request and result data are written to the stream. The request data will be written to DqRequestChannel,
 and the result data will be written to DqResultChannel. Proxy will write the request of the collection into the
-DqRequestChannel, and the DqReqeustChannel will be jointly subscribed by a group of query nodes. When all query nodes
+DqRequestChannel, and the DqRequestChannel will be jointly subscribed by a group of query nodes. When all query nodes
 receive the DqRequest, they will write the query results into the DqResultChannel corresponding to the collection. As
 the consumer of the DqResultChannel, Proxy is responsible for collecting the query results and aggregating them,
 The result is then returned to the client.

docs/design_docs/20220105-query_boolean_expr.md

@@ -31,7 +31,7 @@ ConstantExpr :=
   | UnaryArithOp ConstantExpr
 
 Constant :=
-    INTERGER
+    INTEGER
   | FLOAT_NUMBER
 
 UnaryArithOp :=
@@ -64,7 +64,7 @@ CmpOp :=
   | "=="
   | "!="
 
-INTERGER := 整数
+INTEGER := 整数
 FLOAT_NUM := 浮点数
 IDENTIFIER := 列名
 ```

docs/design_docs/20230918-datanode_remove_datacoord_dependency.md

@@ -61,7 +61,7 @@ The rules system shall follow is:
 
 {% note %}
 
-**Note:** Segments meta shall be updated *BEFORE* changing the channel checkpoint in case of datanode crashing during the prodedure. Under this premise, reconsuming from the old checkpoint shall recover all the data and duplidated entires will be discarded by segment checkpoints.
+**Note:** Segments meta shall be updated *BEFORE* changing the channel checkpoint in case of datanode crashing during the prodedure. Under this premise, reconsuming from the old checkpoint shall recover all the data and duplidated entries will be discarded by segment checkpoints.
 
 {% endnote %}
 
@@ -78,7 +78,7 @@ The winning option is to:
 
 **Note:** `Datacoord` reloads from metastore periodically.
 Optimization 1: reload channel checkpoint first, then reload segment meta if newly read revision is greater than in-memory one.
-Optimization 2: After `L0 segemnt` is implemented, datacoord shall refresh growing segments only.
+Optimization 2: After `L0 segment` is implemented, datacoord shall refresh growing segments only.
 
 {% endnote %}
 

docs/design_docs/segcore/segment_growing.md

@@ -2,13 +2,13 @@
 
 Growing segment has the following additional interfaces:
 
-1. `PreInsert(size) -> reseveredOffset`: serial interface, which reserves space for future insertion and returns the `reseveredOffset`.
+1. `PreInsert(size) -> reservedOffset`: serial interface, which reserves space for future insertion and returns the `reservedOffset`.
 
-2. `Insert(reseveredOffset, size, ...Data...)`: write `...Data...` into range `[reseveredOffset, reseveredOffset + size)`. This interface is allowed to be called concurrently.
+2. `Insert(reservedOffset, size, ...Data...)`: write `...Data...` into range `[reservedOffset, reservedOffset + size)`. This interface is allowed to be called concurrently.
 
    1. `...Data...` contains row_ids, timestamps two system attributes, and other columns
    2. data columns can be stored either row-based or column-based.
-   3. `PreDelete & Delete(reseveredOffset, row_ids, timestamps)` is a delete interface similar to insert interface.
+   3. `PreDelete & Delete(reservedOffset, row_ids, timestamps)` is a delete interface similar to insert interface.
 
 Growing segment stores data in the form of chunk. The number of rows in each chunk is restricted by configs.
 

docs/developer_guides/appendix_a_basic_components.md

@@ -107,7 +107,7 @@ type Session struct {
 }
 
 // NewSession is a helper to build Session object.
-// ServerID, ServerName, Address, Exclusive will be assigned after registeration.
+// ServerID, ServerName, Address, Exclusive will be assigned after registration.
 // metaRoot is a path in etcd to save session information.
 // etcdEndpoints is to init etcdCli when NewSession
 func NewSession(ctx context.Context, metaRoot string, etcdEndpoints []string) *Session {}

docs/developer_guides/chap04_message_stream.md

@@ -7,7 +7,7 @@
 ```go
 type Client interface {
 	CreateChannels(req CreateChannelRequest) (CreateChannelResponse, error)
-	DestoryChannels(req DestoryChannelRequest) error
+	DestroyChannels(req DestroyChannelRequest) error
 	DescribeChannels(req DescribeChannelRequest) (DescribeChannelResponse, error)
 }
 ```
@@ -32,10 +32,10 @@ type CreateChannelResponse struct {
 }
 ```
 
-- _DestoryChannels_
+- _DestroyChannels_
 
 ```go
-type DestoryChannelRequest struct {
+type DestroyChannelRequest struct {
 	ChannelNames []string
 }
 ```

docs/developer_guides/chap05_proxy.md

@@ -105,7 +105,7 @@ type MilvusService interface {
 	CreatePartition(ctx context.Context, request *milvuspb.CreatePartitionRequest) (*commonpb.Status, error)
 	DropPartition(ctx context.Context, request *milvuspb.DropPartitionRequest) (*commonpb.Status, error)
 	HasPartition(ctx context.Context, request *milvuspb.HasPartitionRequest) (*milvuspb.BoolResponse, error)
-	LoadPartitions(ctx context.Context, request *milvuspb.LoadPartitonRequest) (*commonpb.Status, error)
+	LoadPartitions(ctx context.Context, request *milvuspb.LoadPartitionRequest) (*commonpb.Status, error)
 	ReleasePartitions(ctx context.Context, request *milvuspb.ReleasePartitionRequest) (*commonpb.Status, error)
 	GetPartitionStatistics(ctx context.Context, request *milvuspb.PartitionStatsRequest) (*milvuspb.PartitionStatsResponse, error)
 	ShowPartitions(ctx context.Context, request *milvuspb.ShowPartitionRequest) (*milvuspb.ShowPartitionResponse, error)
@@ -225,7 +225,7 @@ type CollectionSchema struct {
 	Fields      []*FieldSchema
 }
 
-type LoadPartitonRequest struct {
+type LoadPartitionRequest struct {
 	Base         *commonpb.MsgBase
 	DbID         UniqueID
 	CollectionID UniqueID

docs/developer_guides/chap07_query_coordinator.md

@@ -134,7 +134,7 @@ type PartitionStatesResponse struct {
 - _LoadPartitions_
 
 ```go
-type LoadPartitonRequest struct {
+type LoadPartitionRequest struct {
 	Base         *commonpb.MsgBase
 	DbID         UniqueID
 	CollectionID UniqueID

docs/user_guides/tls_proxy.md

@@ -78,7 +78,7 @@ certs		= $dir/certs		# Where the issued certs are kept
 crl_dir		= $dir/crl		# Where the issued crl are kept
 database	= $dir/index.txt	# database index file.
 #unique_subject	= no			# Set to 'no' to allow creation of
-					# several ctificates with same subject.
+					# several certificates with same subject.
 new_certs_dir	= $dir/newcerts		# default place for new certs.
 
 certificate	= $dir/cacert.pem 	# The CA certificate
@@ -89,7 +89,7 @@ crl		= $dir/crl.pem 		# The current CRL
 private_key	= $dir/private/cakey.pem# The private key
 RANDFILE	= $dir/private/.rand	# private random number file
 
-x509_extensions	= usr_cert		# The extentions to add to the cert
+x509_extensions	= usr_cert		# The extensions to add to the cert
 
 # Comment out the following two lines for the "traditional"
 # (and highly broken) format.
@@ -141,7 +141,7 @@ default_bits		= 2048
 default_keyfile 	= privkey.pem
 distinguished_name	= req_distinguished_name
 attributes		= req_attributes
-x509_extensions	= v3_ca	# The extentions to add to the self signed cert
+x509_extensions	= v3_ca	# The extensions to add to the self signed cert
 
 # Passwords for private keys if not present they will be prompted for
 # input_password = secret

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/klauspost/compress v1.16.7
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2
 	github.com/milvus-io/milvus/pkg v0.0.1
 	github.com/minio/minio-go/v7 v7.0.61
 	github.com/prometheus/client_golang v1.14.0

go.sum

@@ -581,8 +581,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8 h1:GoGErEOhdWjwSfQilXso3eINqb11yEBDLtoBMNdlve0=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2 h1:tBcKiEUcX6i3MaFYvMJO1F7R6fIoeLFkg1kSGE1Tvpk=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231017063757-b4720fe2ec8f h1:NGI0nposnZ2S0K1W/pEWRNzDkaLraaI10aMgKCMnVYs=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231017063757-b4720fe2ec8f/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
 github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=

internal/core/conanfile.py

@@ -14,18 +14,6 @@ class MilvusConan(ConanFile):
         "lzo/2.10",
         "arrow/12.0.1",
         "openssl/3.1.2",
-        "s2n/1.3.31@milvus/dev",
-        "aws-c-common/0.8.2@milvus/dev",
-        "aws-c-compression/0.2.15@milvus/dev",
-        "aws-c-sdkutils/0.1.3@milvus/dev",
-        "aws-checksums/0.1.13@milvus/dev",
-        "aws-c-cal/0.5.20@milvus/dev",
-        "aws-c-io/0.10.20@milvus/dev",
-        "aws-c-http/0.6.13@milvus/dev",
-        "aws-c-auth/0.6.11@milvus/dev",
-        "aws-c-event-stream/0.2.7@milvus/dev",
-        "aws-c-s3/0.1.37@milvus/dev",
-        "aws-crt-cpp/0.17.23@milvus/dev",
         "aws-sdk-cpp/1.9.234",
         "googleapis/cci.20221108",
         "benchmark/1.7.0",