From 74041550c0aa98c76bbed7f7ee5f5d94b7abd53f Mon Sep 17 00:00:00 2001 From: Aditi Ahuja Date: Tue, 15 Oct 2024 18:13:42 +0530 Subject: [PATCH 1/4] AddMany instead of Add to a bitmap --- index/scorch/snapshot_index_vr.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/index/scorch/snapshot_index_vr.go b/index/scorch/snapshot_index_vr.go index 9d32fd6aa..4427f7845 100644 --- a/index/scorch/snapshot_index_vr.go +++ b/index/scorch/snapshot_index_vr.go @@ -64,11 +64,16 @@ type IndexSnapshotVectorReader struct { // (AND) operations. Eg. finding the eligible doc IDs present in a segment. func (i *IndexSnapshotVectorReader) getEligibleDocIDs() *roaring.Bitmap { res := roaring.NewBitmap() + internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs)) // converts the doc IDs to uint32 and returns for _, eligibleDocInternalID := range i.eligibleDocIDs { - internalDocID, _ := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID)) - res.Add(uint32(internalDocID)) + internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID)) + if err != nil { + continue + } + internalDocIDs = append(internalDocIDs, uint32(internalDocID)) } + res.AddMany(internalDocIDs) return res } From c9eb83407c7aaacc2487c1b059fb8bf7f2b94e8e Mon Sep 17 00:00:00 2001 From: Aditi Ahuja Date: Wed, 16 Oct 2024 15:41:01 +0530 Subject: [PATCH 2/4] use GetCardinality() --- index/scorch/optimize_knn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index/scorch/optimize_knn.go b/index/scorch/optimize_knn.go index 7f0f0907b..ca179574c 100644 --- a/index/scorch/optimize_knn.go +++ b/index/scorch/optimize_knn.go @@ -115,7 +115,7 @@ func (o *OptimizeVR) Finish() error { } eligibleLocalDocNums := make([]uint64, - eligibleVectorInternalIDs.Stats().Cardinality) + eligibleVectorInternalIDs.GetCardinality()) // get the (segment-)local document numbers for i, docNum := range eligibleVectorInternalIDs.ToArray() { localDocNum := o.snapshot.localDocNumFromGlobal(index, From a328124a7bd2beb18f1f4f2a2532c7e9b1091ead Mon Sep 17 00:00:00 2001 From: Aditi Ahuja Date: Thu, 17 Oct 2024 11:14:57 +0530 Subject: [PATCH 3/4] added condition for no eligible docs --- index/scorch/snapshot_index_vr.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/index/scorch/snapshot_index_vr.go b/index/scorch/snapshot_index_vr.go index 4427f7845..320364bc7 100644 --- a/index/scorch/snapshot_index_vr.go +++ b/index/scorch/snapshot_index_vr.go @@ -64,16 +64,18 @@ type IndexSnapshotVectorReader struct { // (AND) operations. Eg. finding the eligible doc IDs present in a segment. func (i *IndexSnapshotVectorReader) getEligibleDocIDs() *roaring.Bitmap { res := roaring.NewBitmap() - internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs)) - // converts the doc IDs to uint32 and returns - for _, eligibleDocInternalID := range i.eligibleDocIDs { - internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID)) - if err != nil { - continue + if len(i.eligibleDocIDs) > 0 { + internalDocIDs := make([]uint32, 0, len(i.eligibleDocIDs)) + // converts the doc IDs to uint32 and returns + for _, eligibleDocInternalID := range i.eligibleDocIDs { + internalDocID, err := docInternalToNumber(index.IndexInternalID(eligibleDocInternalID)) + if err != nil { + continue + } + internalDocIDs = append(internalDocIDs, uint32(internalDocID)) } - internalDocIDs = append(internalDocIDs, uint32(internalDocID)) + res.AddMany(internalDocIDs) } - res.AddMany(internalDocIDs) return res } From 0d3d0ab2e9fcc64b87f61792ecbb200ddc605870 Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Fri, 18 Oct 2024 10:02:13 -0600 Subject: [PATCH 4/4] Upgrade to zapx/v16@v16.1.7 and go-faiss@v1.0.23 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index eeb873bd3..e8b257aa9 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/bits-and-blooms/bitset v1.12.0 github.com/blevesearch/bleve_index_api v1.1.12 github.com/blevesearch/geo v0.1.20 - github.com/blevesearch/go-faiss v1.0.22 + github.com/blevesearch/go-faiss v1.0.23 github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/goleveldb v1.0.1 @@ -24,7 +24,7 @@ require ( github.com/blevesearch/zapx/v13 v13.3.10 github.com/blevesearch/zapx/v14 v14.3.10 github.com/blevesearch/zapx/v15 v15.3.13 - github.com/blevesearch/zapx/v16 v16.1.6 + github.com/blevesearch/zapx/v16 v16.1.7 github.com/couchbase/moss v0.2.0 github.com/golang/protobuf v1.3.2 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index dc5a3df79..46e13fe4c 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/blevesearch/bleve_index_api v1.1.12 h1:P4bw9/G/5rulOF7SJ9l4FsDoo7UFJ+ github.com/blevesearch/bleve_index_api v1.1.12/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= -github.com/blevesearch/go-faiss v1.0.22 h1:j6jwgCOy2a2EQUTOYxjBA59rMn5KPA0jbfYyHNgc2Ls= -github.com/blevesearch/go-faiss v1.0.22/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/blevesearch/go-faiss v1.0.23 h1:Wmc5AFwDLKGl2L6mjLX1Da3vCL0EKa2uHHSorcIS1Uc= +github.com/blevesearch/go-faiss v1.0.23/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:kDy+zgJFJJoJYBvdfBSiZYBbdsUL0XcjHYWezpQBGPA= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= @@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/blevesearch/zapx/v16 v16.1.6 h1:3g1xL9HQMRURaiLiA0lBi6uTjp6w7VQlFg0XjdHvYcs= -github.com/blevesearch/zapx/v16 v16.1.6/go.mod h1:I1CnZUekZFlJZ+cUCnmoJZTD49suuunjuH6BdOTtT9g= +github.com/blevesearch/zapx/v16 v16.1.7 h1:I07qV6l1rPda19zyof9Q2J9E8cjZ57pQhNY0+ePI5vM= +github.com/blevesearch/zapx/v16 v16.1.7/go.mod h1:JqQlOqlRVaYDkpLIl3JnKql8u4zKTNlVEa3nLsi0Gn8= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=