diff --git a/cmd/tools/migration/meta/210_to_220.go b/cmd/tools/migration/meta/210_to_220.go index 5f15cab2210a1..6d7b50e0ae225 100644 --- a/cmd/tools/migration/meta/210_to_220.go +++ b/cmd/tools/migration/meta/210_to_220.go @@ -251,21 +251,21 @@ func combineToSegmentIndexesMeta220(segmentIndexes SegmentIndexesMeta210, indexB } segmentIndexModel := &model.SegmentIndex{ - SegmentID: segID, - CollectionID: record.GetCollectionID(), - PartitionID: record.GetPartitionID(), - NumRows: buildMeta.GetReq().GetNumRows(), - IndexID: indexID, - BuildID: record.GetBuildID(), - NodeID: buildMeta.GetNodeID(), - IndexVersion: buildMeta.GetIndexVersion(), - IndexState: buildMeta.GetState(), - FailReason: buildMeta.GetFailReason(), - IsDeleted: buildMeta.GetMarkDeleted(), - CreatedUTCTime: record.GetCreateTime(), - IndexFileKeys: fileKeys, - IndexSize: buildMeta.GetSerializeSize(), - WriteHandoff: buildMeta.GetState() == commonpb.IndexState_Finished, + SegmentID: segID, + CollectionID: record.GetCollectionID(), + PartitionID: record.GetPartitionID(), + NumRows: buildMeta.GetReq().GetNumRows(), + IndexID: indexID, + BuildID: record.GetBuildID(), + NodeID: buildMeta.GetNodeID(), + IndexVersion: buildMeta.GetIndexVersion(), + IndexState: buildMeta.GetState(), + FailReason: buildMeta.GetFailReason(), + IsDeleted: buildMeta.GetMarkDeleted(), + CreatedUTCTime: record.GetCreateTime(), + IndexFileKeys: fileKeys, + IndexSerializedSize: buildMeta.GetSerializeSize(), + WriteHandoff: buildMeta.GetState() == commonpb.IndexState_Finished, } segmentIndexModels.AddRecord(segID, indexID, segmentIndexModel) } diff --git a/internal/datacoord/compaction_trigger_test.go b/internal/datacoord/compaction_trigger_test.go index b1b912be218ef..1e9d3a7f92d5f 100644 --- a/internal/datacoord/compaction_trigger_test.go +++ b/internal/datacoord/compaction_trigger_test.go @@ -303,59 +303,59 @@ func Test_compactionTrigger_force(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ 1: { indexID: { - SegmentID: 1, - CollectionID: 2, - PartitionID: 1, - NumRows: 100, - IndexID: indexID, - BuildID: 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: 1, + CollectionID: 2, + PartitionID: 1, + NumRows: 100, + IndexID: indexID, + BuildID: 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, 2: { indexID: { - SegmentID: 2, - CollectionID: 2, - PartitionID: 1, - NumRows: 100, - IndexID: indexID, - BuildID: 2, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: 2, + CollectionID: 2, + PartitionID: 1, + NumRows: 100, + IndexID: indexID, + BuildID: 2, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, 3: { indexID: { - SegmentID: 3, - CollectionID: 1111, - PartitionID: 1, - NumRows: 100, - IndexID: indexID, - BuildID: 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: 3, + CollectionID: 1111, + PartitionID: 1, + NumRows: 100, + IndexID: indexID, + BuildID: 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, }, diff --git a/internal/datacoord/garbage_collector_test.go b/internal/datacoord/garbage_collector_test.go index c2527c0e3b87f..d741cc3ba17cd 100644 --- a/internal/datacoord/garbage_collector_test.go +++ b/internal/datacoord/garbage_collector_test.go @@ -487,40 +487,40 @@ func createMetaForRecycleUnusedSegIndexes(catalog metastore.DataCoordCatalog) *m segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, }, @@ -532,37 +532,37 @@ func createMetaForRecycleUnusedSegIndexes(catalog metastore.DataCoordCatalog) *m } meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, }) meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, }) for id, segment := range segments { @@ -652,40 +652,40 @@ func createMetaTableForRecycleUnusedIndexFiles(catalog *datacoord.Catalog) *meta segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, }, @@ -710,38 +710,38 @@ func createMetaTableForRecycleUnusedIndexFiles(catalog *datacoord.Catalog) *meta }, } meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, + WriteHandoff: false, }) meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }) for id, segment := range segments { meta.segments.SetSegment(id, segment) @@ -1052,40 +1052,40 @@ func TestGarbageCollector_clearETCD(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 1024, + WriteHandoff: false, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: []string{"file3", "file4"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file3", "file4"}, + IndexSerializedSize: 1024, + WriteHandoff: false, }, }, }, @@ -1138,39 +1138,39 @@ func TestGarbageCollector_clearETCD(t *testing.T) { } m.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 1024, + WriteHandoff: false, }) m.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: []string{"file3", "file4"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file3", "file4"}, + IndexSerializedSize: 1024, + WriteHandoff: false, }) for id, segment := range segments { diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go index bc12f4f88da1c..21a7b63038657 100644 --- a/internal/datacoord/index_meta.go +++ b/internal/datacoord/index_meta.go @@ -74,7 +74,7 @@ func newIndexTaskStats(s *model.SegmentIndex) *metricsinfo.IndexTaskStats { BuildID: s.BuildID, IndexState: s.IndexState.String(), FailReason: s.FailReason, - IndexSize: s.IndexSize, + IndexSize: s.IndexMemSize, IndexVersion: s.IndexVersion, CreatedUTCTime: typeutil.TimestampToString(s.CreatedUTCTime), FinishedUTCTime: typeutil.TimestampToString(s.FinishedUTCTime), @@ -788,7 +788,8 @@ func (m *indexMeta) FinishTask(taskInfo *workerpb.IndexTaskInfo) error { segIdx.IndexState = taskInfo.GetState() segIdx.IndexFileKeys = common.CloneStringList(taskInfo.GetIndexFileKeys()) segIdx.FailReason = taskInfo.GetFailReason() - segIdx.IndexSize = taskInfo.GetSerializedSize() + segIdx.IndexSerializedSize = taskInfo.GetSerializedSize() + segIdx.IndexMemSize = taskInfo.GetMemSize() segIdx.CurrentIndexVersion = taskInfo.GetCurrentIndexVersion() segIdx.FinishedUTCTime = uint64(time.Now().Unix()) return m.alterSegmentIndexes([]*model.SegmentIndex{segIdx}) @@ -885,8 +886,8 @@ func (m *indexMeta) SetStoredIndexFileSizeMetric(collections map[UniqueID]*colle coll, ok := collections[segmentIdx.CollectionID] if ok { metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName, coll.Schema.GetName(), - fmt.Sprint(segmentIdx.CollectionID)).Add(float64(segmentIdx.IndexSize)) - total += segmentIdx.IndexSize + fmt.Sprint(segmentIdx.CollectionID)).Add(float64(segmentIdx.IndexSerializedSize)) + total += segmentIdx.IndexSerializedSize } } return total @@ -1135,7 +1136,7 @@ func (m *indexMeta) GetSegmentIndexedFields(collectionID UniqueID, segmentID Uni IndexFieldID: index.IndexID, IndexID: index.IndexID, BuildID: buildID, - IndexSize: int64(si.IndexSize), + IndexSize: int64(si.IndexSerializedSize), }) } } diff --git a/internal/datacoord/index_meta_test.go b/internal/datacoord/index_meta_test.go index 8e4ddfccacf76..75a76203d9259 100644 --- a/internal/datacoord/index_meta_test.go +++ b/internal/datacoord/index_meta_test.go @@ -517,20 +517,20 @@ func TestMeta_AddSegmentIndex(t *testing.T) { } segmentIndex := &model.SegmentIndex{ - SegmentID: 1, - CollectionID: 2, - PartitionID: 3, - NumRows: 10240, - IndexID: 4, - BuildID: 5, - NodeID: 6, - IndexVersion: 0, - IndexState: 0, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: 1, + CollectionID: 2, + PartitionID: 3, + NumRows: 10240, + IndexID: 4, + BuildID: 5, + NodeID: 6, + IndexVersion: 0, + IndexState: 0, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSerializedSize: 0, } t.Run("save meta fail", func(t *testing.T) { @@ -664,20 +664,20 @@ func TestMeta_GetSegmentIndexState(t *testing.T) { t.Run("unissued", func(t *testing.T) { m.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) state := m.GetSegmentIndexState(collID, segID, indexID) @@ -686,20 +686,20 @@ func TestMeta_GetSegmentIndexState(t *testing.T) { t.Run("finish", func(t *testing.T) { m.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 0, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 0, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) state := m.GetSegmentIndexState(collID, segID, indexID) @@ -734,20 +734,20 @@ func TestMeta_GetIndexedSegment(t *testing.T) { m.segmentIndexes = map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSerializedSize: 0, }, }, } @@ -770,20 +770,20 @@ func TestMeta_GetIndexedSegment(t *testing.T) { } m.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 10, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) t.Run("success", func(t *testing.T) { @@ -1092,20 +1092,20 @@ func TestMeta_GetIndexParams(t *testing.T) { func TestMeta_GetIndexJob(t *testing.T) { m := newSegmentIndexMeta(nil) m.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) t.Run("exist", func(t *testing.T) { @@ -1180,20 +1180,20 @@ func updateSegmentIndexMeta(t *testing.T) *indexMeta { indexBuildInfo := newSegmentIndexBuildInfo() indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) return &indexMeta{ @@ -1201,20 +1201,20 @@ func updateSegmentIndexMeta(t *testing.T) *indexMeta { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }, }, }, @@ -1521,26 +1521,26 @@ func TestIndexMeta_GetUnindexedSegments(t *testing.T) { func TestBuildIndexTaskStatsJSON(t *testing.T) { im := &indexMeta{segmentBuildInfo: newSegmentIndexBuildInfo()} si1 := &model.SegmentIndex{ - BuildID: 1, - CollectionID: 100, - SegmentID: 1000, - IndexID: 10, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IndexSize: 1024, - IndexVersion: 1, - CreatedUTCTime: uint64(time.Now().Unix()), + BuildID: 1, + CollectionID: 100, + SegmentID: 1000, + IndexID: 10, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IndexSerializedSize: 1024, + IndexVersion: 1, + CreatedUTCTime: uint64(time.Now().Unix()), } si2 := &model.SegmentIndex{ - BuildID: 2, - CollectionID: 101, - SegmentID: 1001, - IndexID: 11, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IndexSize: 2048, - IndexVersion: 1, - CreatedUTCTime: uint64(time.Now().Unix()), + BuildID: 2, + CollectionID: 101, + SegmentID: 1001, + IndexID: 11, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IndexSerializedSize: 2048, + IndexVersion: 1, + CreatedUTCTime: uint64(time.Now().Unix()), } actualJSON := im.TaskStatsJSON() @@ -1638,20 +1638,20 @@ func TestMeta_GetSegmentIndexStatus(t *testing.T) { m.segmentIndexes = map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 0, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 0, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSerializedSize: 0, }, }, segID + 1: {}, diff --git a/internal/datacoord/index_service.go b/internal/datacoord/index_service.go index 62a51200c812e..9cd6dc3caa7ac 100644 --- a/internal/datacoord/index_service.go +++ b/internal/datacoord/index_service.go @@ -932,7 +932,8 @@ func (s *Server) GetIndexInfos(ctx context.Context, req *indexpb.GetIndexInfoReq IndexName: s.meta.indexMeta.GetIndexNameByID(segIdx.CollectionID, segIdx.IndexID), IndexParams: indexParams, IndexFilePaths: indexFilePaths, - SerializedSize: segIdx.IndexSize, + SerializedSize: segIdx.IndexSerializedSize, + MemSize: segIdx.IndexMemSize, IndexVersion: segIdx.IndexVersion, NumRows: segIdx.NumRows, CurrentIndexVersion: segIdx.CurrentIndexVersion, diff --git a/internal/datacoord/index_service_test.go b/internal/datacoord/index_service_test.go index 4923f576da968..c622ce891b246 100644 --- a/internal/datacoord/index_service_test.go +++ b/internal/datacoord/index_service_test.go @@ -400,89 +400,89 @@ func TestServer_AlterIndex(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, segID - 1: { @@ -843,21 +843,21 @@ func TestServer_GetIndexState(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 3000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_IndexStateNone, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 3000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_IndexStateNone, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, }, @@ -968,21 +968,21 @@ func TestServer_GetSegmentIndexState(t *testing.T) { }, } s.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: 10, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1025, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: 10, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 1025, + WriteHandoff: false, }) s.meta.segments.SetSegment(segID, &SegmentInfo{ SegmentInfo: &datapb.SegmentInfo{ @@ -1005,21 +1005,21 @@ func TestServer_GetSegmentIndexState(t *testing.T) { t.Run("finish", func(t *testing.T) { s.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: 10, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1025, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: 10, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 1025, + WriteHandoff: false, }) resp, err := s.GetSegmentIndexState(ctx, req) assert.NoError(t, err) @@ -1127,21 +1127,21 @@ func TestServer_GetIndexBuildProgress(t *testing.T) { t.Run("finish", func(t *testing.T) { s.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: 10, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: 10, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: []string{"file1", "file2"}, + IndexSerializedSize: 0, + WriteHandoff: false, }) s.meta.segments = NewSegmentsInfo() s.meta.segments.SetSegment(segID, &SegmentInfo{ @@ -1389,89 +1389,89 @@ func TestServer_DescribeIndex(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, segID - 1: { @@ -1894,89 +1894,89 @@ func TestServer_GetIndexStatistics(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, }, @@ -2286,21 +2286,21 @@ func TestServer_GetIndexInfos(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSerializedSize: 0, + WriteHandoff: false, }, }, }, diff --git a/internal/datacoord/metrics_info_test.go b/internal/datacoord/metrics_info_test.go index c5d7c5250ca60..c3ff020e5f35c 100644 --- a/internal/datacoord/metrics_info_test.go +++ b/internal/datacoord/metrics_info_test.go @@ -706,20 +706,20 @@ func TestServer_getSegmentsJSON(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ 1000: { 10: &model.SegmentIndex{ - SegmentID: 1000, - CollectionID: 1, - PartitionID: 2, - NumRows: 10250, - IndexID: 10, - BuildID: 10000, - NodeID: 1, - IndexVersion: 0, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: 1000, + CollectionID: 1, + PartitionID: 2, + NumRows: 10250, + IndexID: 10, + BuildID: 10000, + NodeID: 1, + IndexVersion: 0, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSerializedSize: 0, }, }, }, diff --git a/internal/datacoord/server_test.go b/internal/datacoord/server_test.go index 83df41f7bb36f..94772d907e468 100644 --- a/internal/datacoord/server_test.go +++ b/internal/datacoord/server_test.go @@ -1633,20 +1633,20 @@ func TestGetRecoveryInfo(t *testing.T) { }) assert.NoError(t, err) svr.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: seg4.ID, - CollectionID: 0, - PartitionID: 0, - NumRows: 100, - IndexID: 0, - BuildID: 0, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: seg4.ID, + CollectionID: 0, + PartitionID: 0, + NumRows: 100, + IndexID: 0, + BuildID: 0, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) req := &datapb.GetRecoveryInfoRequest{ diff --git a/internal/datacoord/services_test.go b/internal/datacoord/services_test.go index 669d433819f90..1bf23903bd676 100644 --- a/internal/datacoord/services_test.go +++ b/internal/datacoord/services_test.go @@ -1258,20 +1258,20 @@ func TestGetRecoveryInfoV2(t *testing.T) { }) assert.NoError(t, err) svr.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: seg4.ID, - CollectionID: 0, - PartitionID: 0, - NumRows: 100, - IndexID: 0, - BuildID: 0, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: seg4.ID, + CollectionID: 0, + PartitionID: 0, + NumRows: 100, + IndexID: 0, + BuildID: 0, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) ch := &channelMeta{Name: "vchan1", CollectionID: 0} diff --git a/internal/datacoord/task_scheduler_test.go b/internal/datacoord/task_scheduler_test.go index ddd12afcf5a94..d42a986c40acf 100644 --- a/internal/datacoord/task_scheduler_test.go +++ b/internal/datacoord/task_scheduler_test.go @@ -58,190 +58,190 @@ var ( func createIndexMeta(catalog metastore.DataCoordCatalog) *indexMeta { indexBuildInfo := newSegmentIndexBuildInfo() indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 2, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 2, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: true, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 2, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 2, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: true, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 3, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 3, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 4, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 4, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 4, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 4, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 5, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 5, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 6, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 6, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 6, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 6, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 7, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 7, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "error", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 7, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 7, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "error", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 8, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 8, - NodeID: nodeID + 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 8, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 8, + NodeID: nodeID + 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 9, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 9, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 9, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 9, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 10, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 10, - NodeID: nodeID, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 10, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 10, + NodeID: nodeID, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }) return &indexMeta{ @@ -278,200 +278,200 @@ func createIndexMeta(catalog metastore.DataCoordCatalog) *indexMeta { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 2: { indexID: { - SegmentID: segID + 2, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 2, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: true, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 2, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 2, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: true, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 3: { indexID: { - SegmentID: segID + 3, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 3, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 4: { indexID: { - SegmentID: segID + 4, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 4, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 4, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 4, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 5: { indexID: { - SegmentID: segID + 5, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 5, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 6: { indexID: { - SegmentID: segID + 6, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 6, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 6, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 6, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 7: { indexID: { - SegmentID: segID + 7, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 7, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "error", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 7, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 7, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "error", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 8: { indexID: { - SegmentID: segID + 8, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 8, - NodeID: nodeID + 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 8, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 8, + NodeID: nodeID + 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 9: { indexID: { - SegmentID: segID + 9, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 9, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 9, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 9, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, segID + 10: { indexID: { - SegmentID: segID + 10, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 10, - NodeID: nodeID, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 10, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 10, + NodeID: nodeID, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSerializedSize: 1, }, }, }, @@ -1541,20 +1541,20 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: minNumberOfRowsToBuild, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: minNumberOfRowsToBuild, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }, }, }, @@ -1583,20 +1583,20 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() { } mt.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: minNumberOfRowsToBuild, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: minNumberOfRowsToBuild, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, }) cm := mocks.NewChunkManager(s.T()) cm.EXPECT().RootPath().Return("ut-index") diff --git a/internal/metastore/kv/datacoord/kv_catalog_test.go b/internal/metastore/kv/datacoord/kv_catalog_test.go index 992aeb0830aed..fbd3081acbd4a 100644 --- a/internal/metastore/kv/datacoord/kv_catalog_test.go +++ b/internal/metastore/kv/datacoord/kv_catalog_test.go @@ -1013,20 +1013,20 @@ func TestCatalog_DropIndex(t *testing.T) { func TestCatalog_CreateSegmentIndex(t *testing.T) { segIdx := &model.SegmentIndex{ - SegmentID: 1, - CollectionID: 2, - PartitionID: 3, - NumRows: 1024, - IndexID: 4, - BuildID: 5, - NodeID: 6, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IndexVersion: 0, - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: 1, + CollectionID: 2, + PartitionID: 3, + NumRows: 1024, + IndexID: 4, + BuildID: 5, + NodeID: 6, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IndexVersion: 0, + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, } t.Run("success", func(t *testing.T) { @@ -1113,20 +1113,20 @@ func TestCatalog_ListSegmentIndexes(t *testing.T) { func TestCatalog_AlterSegmentIndexes(t *testing.T) { segIdx := &model.SegmentIndex{ - SegmentID: 0, - CollectionID: 0, - PartitionID: 0, - NumRows: 0, - IndexID: 0, - BuildID: 0, - NodeID: 0, - IndexState: 0, - FailReason: "", - IndexVersion: 0, - IsDeleted: false, - CreatedUTCTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: 0, + CollectionID: 0, + PartitionID: 0, + NumRows: 0, + IndexID: 0, + BuildID: 0, + NodeID: 0, + IndexState: 0, + FailReason: "", + IndexVersion: 0, + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSerializedSize: 0, } t.Run("add", func(t *testing.T) { diff --git a/internal/metastore/model/segment_index.go b/internal/metastore/model/segment_index.go index 9f6dfd8dce537..b31bd282456aa 100644 --- a/internal/metastore/model/segment_index.go +++ b/internal/metastore/model/segment_index.go @@ -4,6 +4,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus/internal/proto/indexpb" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/util/paramtable" ) type SegmentIndex struct { @@ -20,7 +21,12 @@ type SegmentIndex struct { IsDeleted bool CreatedUTCTime uint64 IndexFileKeys []string - IndexSize uint64 + // The byte size of serialized index data at oos. (compressed) + IndexSerializedSize uint64 + // The byte size of index data in memory. (uncompressed) + // The IndexMemSize may not be stored at old milvus implementation, so it may be not accurate. + // (generated by the IndexSerializedSize multiplied with a configured compress-ratio). + IndexMemSize uint64 // deprecated WriteHandoff bool CurrentIndexVersion int32 @@ -32,7 +38,10 @@ func UnmarshalSegmentIndexModel(segIndex *indexpb.SegmentIndex) *SegmentIndex { if segIndex == nil { return nil } - + memSize := segIndex.MemSize + if memSize == 0 { + memSize = segIndex.SerializeSize * paramtable.Get().DataCoordCfg.IndexMemSizeEstimateMultiplier.GetAsUint64() + } return &SegmentIndex{ SegmentID: segIndex.SegmentID, CollectionID: segIndex.CollectionID, @@ -47,7 +56,8 @@ func UnmarshalSegmentIndexModel(segIndex *indexpb.SegmentIndex) *SegmentIndex { IsDeleted: segIndex.Deleted, CreatedUTCTime: segIndex.CreateTime, IndexFileKeys: common.CloneStringList(segIndex.IndexFileKeys), - IndexSize: segIndex.SerializeSize, + IndexSerializedSize: segIndex.SerializeSize, + IndexMemSize: memSize, WriteHandoff: segIndex.WriteHandoff, CurrentIndexVersion: segIndex.GetCurrentIndexVersion(), } @@ -72,7 +82,8 @@ func MarshalSegmentIndexModel(segIdx *SegmentIndex) *indexpb.SegmentIndex { IndexFileKeys: common.CloneStringList(segIdx.IndexFileKeys), Deleted: segIdx.IsDeleted, CreateTime: segIdx.CreatedUTCTime, - SerializeSize: segIdx.IndexSize, + SerializeSize: segIdx.IndexSerializedSize, + MemSize: segIdx.IndexMemSize, WriteHandoff: segIdx.WriteHandoff, CurrentIndexVersion: segIdx.CurrentIndexVersion, } @@ -93,7 +104,8 @@ func CloneSegmentIndex(segIndex *SegmentIndex) *SegmentIndex { IsDeleted: segIndex.IsDeleted, CreatedUTCTime: segIndex.CreatedUTCTime, IndexFileKeys: common.CloneStringList(segIndex.IndexFileKeys), - IndexSize: segIndex.IndexSize, + IndexSerializedSize: segIndex.IndexSerializedSize, + IndexMemSize: segIndex.IndexMemSize, WriteHandoff: segIndex.WriteHandoff, CurrentIndexVersion: segIndex.CurrentIndexVersion, } diff --git a/internal/metastore/model/segment_index_test.go b/internal/metastore/model/segment_index_test.go index 8ad49de8f4442..8d09a969a0f31 100644 --- a/internal/metastore/model/segment_index_test.go +++ b/internal/metastore/model/segment_index_test.go @@ -31,20 +31,20 @@ var ( } indexModel2 = &SegmentIndex{ - CollectionID: colID, - PartitionID: partID, - SegmentID: segmentID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IndexVersion: 0, - IsDeleted: false, - CreatedUTCTime: 1, - IndexFileKeys: nil, - IndexSize: 0, + CollectionID: colID, + PartitionID: partID, + SegmentID: segmentID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IndexVersion: 0, + IsDeleted: false, + CreatedUTCTime: 1, + IndexFileKeys: nil, + IndexSerializedSize: 0, } ) diff --git a/internal/proto/index_coord.proto b/internal/proto/index_coord.proto index adcd0aed7bd37..367c529ae48fe 100644 --- a/internal/proto/index_coord.proto +++ b/internal/proto/index_coord.proto @@ -76,10 +76,11 @@ message SegmentIndex { repeated string index_file_keys = 11; bool deleted = 12; uint64 create_time = 13; - uint64 serialize_size = 14; + uint64 serialize_size = 14; // the total size of index file at oos. (may be compress) bool write_handoff = 15; int32 current_index_version = 16; int64 index_store_version = 17; + uint64 mem_size = 18; // the total size of index file at local (disk or memory) before loading by knowhere. (not compress) } message RegisterNodeRequest { @@ -159,6 +160,7 @@ message IndexFilePathInfo { int64 index_version = 9; int64 num_rows = 10; int32 current_index_version = 11; + uint64 mem_size = 12; } message SegmentInfo { diff --git a/internal/proto/worker.proto b/internal/proto/worker.proto index 6f8f72a0a441f..ac40c5ec00f5c 100644 --- a/internal/proto/worker.proto +++ b/internal/proto/worker.proto @@ -169,6 +169,7 @@ message IndexTaskInfo { string fail_reason = 5; int32 current_index_version = 6; int64 index_store_version = 7; + uint64 mem_size = 8; } message IndexJobResults { diff --git a/internal/querycoordv2/meta/coordinator_broker.go b/internal/querycoordv2/meta/coordinator_broker.go index cc84db836bbf3..cc2053c665139 100644 --- a/internal/querycoordv2/meta/coordinator_broker.go +++ b/internal/querycoordv2/meta/coordinator_broker.go @@ -356,7 +356,7 @@ func (broker *CoordinatorBroker) GetIndexInfo(ctx context.Context, collectionID BuildID: info.GetBuildID(), IndexParams: info.GetIndexParams(), IndexFilePaths: info.GetIndexFilePaths(), - IndexSize: int64(info.GetSerializedSize()), + IndexSize: int64(info.GetMemSize()), IndexVersion: info.GetIndexVersion(), NumRows: info.GetNumRows(), CurrentIndexVersion: info.GetCurrentIndexVersion(), diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 3f8723482311c..68d139b659a23 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -3324,6 +3324,9 @@ type dataCoordConfig struct { ChannelCheckpointMaxLag ParamItem `refreshable:"true"` SyncSegmentsInterval ParamItem `refreshable:"false"` + // Index related configuration + IndexMemSizeEstimateMultiplier ParamItem `refreshable:"true"` + // Clustering Compaction ClusteringCompactionEnable ParamItem `refreshable:"true"` ClusteringCompactionAutoEnable ParamItem `refreshable:"true"` @@ -3807,6 +3810,15 @@ During compaction, the size of segment # of rows is able to exceed segment max # } p.LevelZeroCompactionTriggerDeltalogMaxNum.Init(base.mgr) + p.IndexMemSizeEstimateMultiplier = ParamItem{ + Key: "dataCoord.index.memSizeEstimateMultiplier", + Version: "2.4.19", + DefaultValue: "2", + Doc: "When the memory size is not setup by index procedure, multiplier to estimate the memory size of index data", + Export: true, + } + p.IndexMemSizeEstimateMultiplier.Init(base.mgr) + p.ClusteringCompactionEnable = ParamItem{ Key: "dataCoord.compaction.clustering.enable", Version: "2.4.7",