enhance: add mem size for index file and fallback to multiply with se…

…rialized size Signed-off-by: chyezh <[email protected]>
milvus-io · Dec 24, 2024 · 51bbb80 · 51bbb80
1 parent 5304707
commit 51bbb80
Show file tree

Hide file tree

Showing 18 changed files with 1,144 additions and 1,115 deletions.
diff --git a/cmd/tools/migration/meta/210_to_220.go b/cmd/tools/migration/meta/210_to_220.go
@@ -251,21 +251,21 @@ func combineToSegmentIndexesMeta220(segmentIndexes SegmentIndexesMeta210, indexB
 			}
 
 			segmentIndexModel := &model.SegmentIndex{
-				SegmentID:      segID,
-				CollectionID:   record.GetCollectionID(),
-				PartitionID:    record.GetPartitionID(),
-				NumRows:        buildMeta.GetReq().GetNumRows(),
-				IndexID:        indexID,
-				BuildID:        record.GetBuildID(),
-				NodeID:         buildMeta.GetNodeID(),
-				IndexVersion:   buildMeta.GetIndexVersion(),
-				IndexState:     buildMeta.GetState(),
-				FailReason:     buildMeta.GetFailReason(),
-				IsDeleted:      buildMeta.GetMarkDeleted(),
-				CreatedUTCTime: record.GetCreateTime(),
-				IndexFileKeys:  fileKeys,
-				IndexSize:      buildMeta.GetSerializeSize(),
-				WriteHandoff:   buildMeta.GetState() == commonpb.IndexState_Finished,
+				SegmentID:           segID,
+				CollectionID:        record.GetCollectionID(),
+				PartitionID:         record.GetPartitionID(),
+				NumRows:             buildMeta.GetReq().GetNumRows(),
+				IndexID:             indexID,
+				BuildID:             record.GetBuildID(),
+				NodeID:              buildMeta.GetNodeID(),
+				IndexVersion:        buildMeta.GetIndexVersion(),
+				IndexState:          buildMeta.GetState(),
+				FailReason:          buildMeta.GetFailReason(),
+				IsDeleted:           buildMeta.GetMarkDeleted(),
+				CreatedUTCTime:      record.GetCreateTime(),
+				IndexFileKeys:       fileKeys,
+				IndexSerializedSize: buildMeta.GetSerializeSize(),
+				WriteHandoff:        buildMeta.GetState() == commonpb.IndexState_Finished,
 			}
 			segmentIndexModels.AddRecord(segID, indexID, segmentIndexModel)
 		}

diff --git a/internal/datacoord/compaction_trigger_test.go b/internal/datacoord/compaction_trigger_test.go
@@ -303,59 +303,59 @@ func Test_compactionTrigger_force(t *testing.T) {
 						segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{
 							1: {
 								indexID: {
-									SegmentID:      1,
-									CollectionID:   2,
-									PartitionID:    1,
-									NumRows:        100,
-									IndexID:        indexID,
-									BuildID:        1,
-									NodeID:         0,
-									IndexVersion:   1,
-									IndexState:     commonpb.IndexState_Finished,
-									FailReason:     "",
-									IsDeleted:      false,
-									CreatedUTCTime: 0,
-									IndexFileKeys:  nil,
-									IndexSize:      0,
-									WriteHandoff:   false,
+									SegmentID:           1,
+									CollectionID:        2,
+									PartitionID:         1,
+									NumRows:             100,
+									IndexID:             indexID,
+									BuildID:             1,
+									NodeID:              0,
+									IndexVersion:        1,
+									IndexState:          commonpb.IndexState_Finished,
+									FailReason:          "",
+									IsDeleted:           false,
+									CreatedUTCTime:      0,
+									IndexFileKeys:       nil,
+									IndexSerializedSize: 0,
+									WriteHandoff:        false,
 								},
 							},
 							2: {
 								indexID: {
-									SegmentID:      2,
-									CollectionID:   2,
-									PartitionID:    1,
-									NumRows:        100,
-									IndexID:        indexID,
-									BuildID:        2,
-									NodeID:         0,
-									IndexVersion:   1,
-									IndexState:     commonpb.IndexState_Finished,
-									FailReason:     "",
-									IsDeleted:      false,
-									CreatedUTCTime: 0,
-									IndexFileKeys:  nil,
-									IndexSize:      0,
-									WriteHandoff:   false,
+									SegmentID:           2,
+									CollectionID:        2,
+									PartitionID:         1,
+									NumRows:             100,
+									IndexID:             indexID,
+									BuildID:             2,
+									NodeID:              0,
+									IndexVersion:        1,
+									IndexState:          commonpb.IndexState_Finished,
+									FailReason:          "",
+									IsDeleted:           false,
+									CreatedUTCTime:      0,
+									IndexFileKeys:       nil,
+									IndexSerializedSize: 0,
+									WriteHandoff:        false,
 								},
 							},
 							3: {
 								indexID: {
-									SegmentID:      3,
-									CollectionID:   1111,
-									PartitionID:    1,
-									NumRows:        100,
-									IndexID:        indexID,
-									BuildID:        3,
-									NodeID:         0,
-									IndexVersion:   1,
-									IndexState:     commonpb.IndexState_Finished,
-									FailReason:     "",
-									IsDeleted:      false,
-									CreatedUTCTime: 0,
-									IndexFileKeys:  nil,
-									IndexSize:      0,
-									WriteHandoff:   false,
+									SegmentID:           3,
+									CollectionID:        1111,
+									PartitionID:         1,
+									NumRows:             100,
+									IndexID:             indexID,
+									BuildID:             3,
+									NodeID:              0,
+									IndexVersion:        1,
+									IndexState:          commonpb.IndexState_Finished,
+									FailReason:          "",
+									IsDeleted:           false,
+									CreatedUTCTime:      0,
+									IndexFileKeys:       nil,
+									IndexSerializedSize: 0,
+									WriteHandoff:        false,
 								},
 							},
 						},