diff --git a/cmd/tools/migration/meta/210_to_220.go b/cmd/tools/migration/meta/210_to_220.go index 1fceb29d0e083..5f15cab2210a1 100644 --- a/cmd/tools/migration/meta/210_to_220.go +++ b/cmd/tools/migration/meta/210_to_220.go @@ -251,21 +251,21 @@ func combineToSegmentIndexesMeta220(segmentIndexes SegmentIndexesMeta210, indexB } segmentIndexModel := &model.SegmentIndex{ - SegmentID: segID, - CollectionID: record.GetCollectionID(), - PartitionID: record.GetPartitionID(), - NumRows: buildMeta.GetReq().GetNumRows(), - IndexID: indexID, - BuildID: record.GetBuildID(), - NodeID: buildMeta.GetNodeID(), - IndexVersion: buildMeta.GetIndexVersion(), - IndexState: buildMeta.GetState(), - FailReason: buildMeta.GetFailReason(), - IsDeleted: buildMeta.GetMarkDeleted(), - CreateTime: record.GetCreateTime(), - IndexFileKeys: fileKeys, - IndexSize: buildMeta.GetSerializeSize(), - WriteHandoff: buildMeta.GetState() == commonpb.IndexState_Finished, + SegmentID: segID, + CollectionID: record.GetCollectionID(), + PartitionID: record.GetPartitionID(), + NumRows: buildMeta.GetReq().GetNumRows(), + IndexID: indexID, + BuildID: record.GetBuildID(), + NodeID: buildMeta.GetNodeID(), + IndexVersion: buildMeta.GetIndexVersion(), + IndexState: buildMeta.GetState(), + FailReason: buildMeta.GetFailReason(), + IsDeleted: buildMeta.GetMarkDeleted(), + CreatedUTCTime: record.GetCreateTime(), + IndexFileKeys: fileKeys, + IndexSize: buildMeta.GetSerializeSize(), + WriteHandoff: buildMeta.GetState() == commonpb.IndexState_Finished, } segmentIndexModels.AddRecord(segID, indexID, segmentIndexModel) } diff --git a/go.mod b/go.mod index 7db5c5cf0dc1d..b5489c8c3fcf5 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/klauspost/compress v1.17.9 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d - github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241108105827-266fb751b620 + github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241111062829-6de3d96f664f github.com/minio/minio-go/v7 v7.0.73 github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/prometheus/client_golang v1.14.0 diff --git a/go.sum b/go.sum index 9d2a841976107..3986e672b5f32 100644 --- a/go.sum +++ b/go.sum @@ -628,8 +628,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241108105827-266fb751b620 h1:0IWUDtDloift7cQHalhdjuVkL/3qSeiXFqR7MofZBkg= -github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241108105827-266fb751b620/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241111062829-6de3d96f664f h1:yLxT8NH0ixUOJMqJuk0xvGf0cKsr+N2xibyTat256PI= +github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241111062829-6de3d96f664f/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE= github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= diff --git a/internal/datacoord/compaction_task_meta.go b/internal/datacoord/compaction_task_meta.go index 6af1e13fafdb8..779d1be0482bd 100644 --- a/internal/datacoord/compaction_task_meta.go +++ b/internal/datacoord/compaction_task_meta.go @@ -19,10 +19,12 @@ package datacoord import ( "context" "encoding/json" + "strconv" "sync" "time" "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/samber/lo" "go.uber.org/zap" "google.golang.org/protobuf/proto" @@ -31,20 +33,25 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/timerecord" + "github.com/milvus-io/milvus/pkg/util/typeutil" ) func newCompactionTaskStats(task *datapb.CompactionTask) *metricsinfo.CompactionTask { return &metricsinfo.CompactionTask{ - PlanID: task.PlanID, - CollectionID: task.CollectionID, - Type: task.Type.String(), - State: task.State.String(), - FailReason: task.FailReason, - StartTime: task.StartTime, - EndTime: task.EndTime, - TotalRows: task.TotalRows, - InputSegments: task.InputSegments, - ResultSegments: task.ResultSegments, + PlanID: task.PlanID, + CollectionID: task.CollectionID, + Type: task.Type.String(), + State: task.State.String(), + FailReason: task.FailReason, + StartTime: typeutil.TimestampToString(uint64(task.StartTime)), + EndTime: typeutil.TimestampToString(uint64(task.EndTime)), + TotalRows: task.TotalRows, + InputSegments: lo.Map(task.InputSegments, func(t int64, i int) string { + return strconv.FormatInt(t, 10) + }), + ResultSegments: lo.Map(task.ResultSegments, func(t int64, i int) string { + return strconv.FormatInt(t, 10) + }), } } diff --git a/internal/datacoord/compaction_trigger_test.go b/internal/datacoord/compaction_trigger_test.go index ee3f60aeb10d2..1c94fec2686e4 100644 --- a/internal/datacoord/compaction_trigger_test.go +++ b/internal/datacoord/compaction_trigger_test.go @@ -308,59 +308,59 @@ func Test_compactionTrigger_force(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ 1: { indexID: { - SegmentID: 1, - CollectionID: 2, - PartitionID: 1, - NumRows: 100, - IndexID: indexID, - BuildID: 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: 1, + CollectionID: 2, + PartitionID: 1, + NumRows: 100, + IndexID: indexID, + BuildID: 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, 2: { indexID: { - SegmentID: 2, - CollectionID: 2, - PartitionID: 1, - NumRows: 100, - IndexID: indexID, - BuildID: 2, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: 2, + CollectionID: 2, + PartitionID: 1, + NumRows: 100, + IndexID: indexID, + BuildID: 2, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, 3: { indexID: { - SegmentID: 3, - CollectionID: 1111, - PartitionID: 1, - NumRows: 100, - IndexID: indexID, - BuildID: 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: 3, + CollectionID: 1111, + PartitionID: 1, + NumRows: 100, + IndexID: indexID, + BuildID: 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, }, diff --git a/internal/datacoord/garbage_collector_test.go b/internal/datacoord/garbage_collector_test.go index bd444bd682076..ee6e112649a7e 100644 --- a/internal/datacoord/garbage_collector_test.go +++ b/internal/datacoord/garbage_collector_test.go @@ -487,40 +487,40 @@ func createMetaForRecycleUnusedSegIndexes(catalog metastore.DataCoordCatalog) *m segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, + WriteHandoff: false, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, + WriteHandoff: false, }, }, }, @@ -532,37 +532,37 @@ func createMetaForRecycleUnusedSegIndexes(catalog metastore.DataCoordCatalog) *m } meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, }) meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, }) for id, segment := range segments { @@ -652,40 +652,40 @@ func createMetaTableForRecycleUnusedIndexFiles(catalog *datacoord.Catalog) *meta segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, + WriteHandoff: false, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, }, @@ -710,38 +710,38 @@ func createMetaTableForRecycleUnusedIndexFiles(catalog *datacoord.Catalog) *meta }, } meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, + WriteHandoff: false, }) meta.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }) for id, segment := range segments { meta.segments.SetSegment(id, segment) @@ -1052,40 +1052,40 @@ func TestGarbageCollector_clearETCD(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 1024, + WriteHandoff: false, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: []string{"file3", "file4"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file3", "file4"}, + IndexSize: 1024, + WriteHandoff: false, }, }, }, @@ -1138,39 +1138,39 @@ func TestGarbageCollector_clearETCD(t *testing.T) { } m.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 1024, + WriteHandoff: false, }) m.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 5000, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: []string{"file3", "file4"}, - IndexSize: 1024, - WriteHandoff: false, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 5000, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: []string{"file3", "file4"}, + IndexSize: 1024, + WriteHandoff: false, }) for id, segment := range segments { diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go index e31f12ca2ff30..aa7d16381e35d 100644 --- a/internal/datacoord/index_meta.go +++ b/internal/datacoord/index_meta.go @@ -43,6 +43,7 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/indexparams" + "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/timerecord" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -63,29 +64,17 @@ type indexMeta struct { segmentIndexes map[UniqueID]map[UniqueID]*model.SegmentIndex } -type indexTaskStats struct { - IndexID UniqueID `json:"index_id,omitempty"` - CollectionID UniqueID `json:"collection_id,omitempty"` - SegmentID UniqueID `json:"segment_id,omitempty"` - BuildID UniqueID `json:"build_id,omitempty"` - IndexState string `json:"index_state,omitempty"` - FailReason string `json:"fail_reason,omitempty"` - IndexSize uint64 `json:"index_size,omitempty"` - IndexVersion int64 `json:"index_version,omitempty"` - CreateTime uint64 `json:"create_time,omitempty"` -} - -func newIndexTaskStats(s *model.SegmentIndex) *indexTaskStats { - return &indexTaskStats{ - IndexID: s.IndexID, - CollectionID: s.CollectionID, - SegmentID: s.SegmentID, - BuildID: s.BuildID, - IndexState: s.IndexState.String(), - FailReason: s.FailReason, - IndexSize: s.IndexSize, - IndexVersion: s.IndexVersion, - CreateTime: s.CreateTime, +func newIndexTaskStats(s *model.SegmentIndex) *metricsinfo.IndexTaskStats { + return &metricsinfo.IndexTaskStats{ + IndexID: s.IndexID, + CollectionID: s.CollectionID, + SegmentID: s.SegmentID, + BuildID: s.BuildID, + IndexState: s.IndexState.String(), + FailReason: s.FailReason, + IndexSize: s.IndexSize, + IndexVersion: s.IndexVersion, + CreatedUTCTime: typeutil.TimestampToString(s.CreatedUTCTime), } } @@ -94,7 +83,7 @@ type segmentBuildInfo struct { // buildID -> segmentIndex buildID2SegmentIndex map[UniqueID]*model.SegmentIndex // taskStats records the task stats of the segment - taskStats *expirable.LRU[UniqueID, *indexTaskStats] + taskStats *expirable.LRU[UniqueID, *metricsinfo.IndexTaskStats] } func newSegmentIndexBuildInfo() *segmentBuildInfo { @@ -102,7 +91,7 @@ func newSegmentIndexBuildInfo() *segmentBuildInfo { // build ID -> segment index buildID2SegmentIndex: make(map[UniqueID]*model.SegmentIndex), // build ID -> task stats - taskStats: expirable.NewLRU[UniqueID, *indexTaskStats](64, nil, time.Minute*30), + taskStats: expirable.NewLRU[UniqueID, *metricsinfo.IndexTaskStats](64, nil, time.Minute*30), } } @@ -124,7 +113,7 @@ func (m *segmentBuildInfo) List() map[UniqueID]*model.SegmentIndex { return m.buildID2SegmentIndex } -func (m *segmentBuildInfo) GetTaskStats() []*indexTaskStats { +func (m *segmentBuildInfo) GetTaskStats() []*metricsinfo.IndexTaskStats { return m.taskStats.Values() } diff --git a/internal/datacoord/index_meta_test.go b/internal/datacoord/index_meta_test.go index 9b4de1107600a..8290b1db5ad89 100644 --- a/internal/datacoord/index_meta_test.go +++ b/internal/datacoord/index_meta_test.go @@ -37,6 +37,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/indexpb" "github.com/milvus-io/milvus/internal/proto/workerpb" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/util/metricsinfo" ) func TestReloadFromKV(t *testing.T) { @@ -516,20 +517,20 @@ func TestMeta_AddSegmentIndex(t *testing.T) { } segmentIndex := &model.SegmentIndex{ - SegmentID: 1, - CollectionID: 2, - PartitionID: 3, - NumRows: 10240, - IndexID: 4, - BuildID: 5, - NodeID: 6, - IndexVersion: 0, - IndexState: 0, - FailReason: "", - IsDeleted: false, - CreateTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: 1, + CollectionID: 2, + PartitionID: 3, + NumRows: 10240, + IndexID: 4, + BuildID: 5, + NodeID: 6, + IndexVersion: 0, + IndexState: 0, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSize: 0, } t.Run("save meta fail", func(t *testing.T) { @@ -663,20 +664,20 @@ func TestMeta_GetSegmentIndexState(t *testing.T) { t.Run("unissued", func(t *testing.T) { m.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSize: 0, }) state := m.GetSegmentIndexState(collID, segID, indexID) @@ -685,20 +686,20 @@ func TestMeta_GetSegmentIndexState(t *testing.T) { t.Run("finish", func(t *testing.T) { m.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 0, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 12, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 0, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 12, + IndexFileKeys: nil, + IndexSize: 0, }) state := m.GetSegmentIndexState(collID, segID, indexID) @@ -733,20 +734,20 @@ func TestMeta_GetIndexedSegment(t *testing.T) { m.segmentIndexes = map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSize: 0, }, }, } @@ -769,20 +770,20 @@ func TestMeta_GetIndexedSegment(t *testing.T) { } m.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 10, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 10, + IndexFileKeys: nil, + IndexSize: 0, }) t.Run("success", func(t *testing.T) { @@ -1091,20 +1092,20 @@ func TestMeta_GetIndexParams(t *testing.T) { func TestMeta_GetIndexJob(t *testing.T) { m := newSegmentIndexMeta(nil) m.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }) t.Run("exist", func(t *testing.T) { @@ -1179,20 +1180,20 @@ func updateSegmentIndexMeta(t *testing.T) *indexMeta { indexBuildInfo := newSegmentIndexBuildInfo() indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }) return &indexMeta{ @@ -1200,20 +1201,20 @@ func updateSegmentIndexMeta(t *testing.T) *indexMeta { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }, }, }, @@ -1520,26 +1521,26 @@ func TestIndexMeta_GetUnindexedSegments(t *testing.T) { func TestBuildIndexTaskStatsJSON(t *testing.T) { im := &indexMeta{segmentBuildInfo: newSegmentIndexBuildInfo()} si1 := &model.SegmentIndex{ - BuildID: 1, - CollectionID: 100, - SegmentID: 1000, - IndexID: 10, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IndexSize: 1024, - IndexVersion: 1, - CreateTime: uint64(time.Now().Unix()), + BuildID: 1, + CollectionID: 100, + SegmentID: 1000, + IndexID: 10, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IndexSize: 1024, + IndexVersion: 1, + CreatedUTCTime: uint64(time.Now().Unix()), } si2 := &model.SegmentIndex{ - BuildID: 2, - CollectionID: 101, - SegmentID: 1001, - IndexID: 11, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IndexSize: 2048, - IndexVersion: 1, - CreateTime: uint64(time.Now().Unix()), + BuildID: 2, + CollectionID: 101, + SegmentID: 1001, + IndexID: 11, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IndexSize: 2048, + IndexVersion: 1, + CreatedUTCTime: uint64(time.Now().Unix()), } actualJSON := im.TaskStatsJSON() @@ -1553,7 +1554,7 @@ func TestBuildIndexTaskStatsJSON(t *testing.T) { assert.True(t, ok) assert.EqualValues(t, si1, ret1) - expectedTasks := []*indexTaskStats{ + expectedTasks := []*metricsinfo.IndexTaskStats{ newIndexTaskStats(si1), newIndexTaskStats(si2), } diff --git a/internal/datacoord/index_service.go b/internal/datacoord/index_service.go index 4b754f86791fd..c31559fbce3f7 100644 --- a/internal/datacoord/index_service.go +++ b/internal/datacoord/index_service.go @@ -64,14 +64,14 @@ func (s *Server) createIndexForSegment(segment *SegmentInfo, indexID UniqueID) e return err } segIndex := &model.SegmentIndex{ - SegmentID: segment.ID, - CollectionID: segment.CollectionID, - PartitionID: segment.PartitionID, - NumRows: segment.NumOfRows, - IndexID: indexID, - BuildID: buildID, - CreateTime: uint64(segment.ID), - WriteHandoff: false, + SegmentID: segment.ID, + CollectionID: segment.CollectionID, + PartitionID: segment.PartitionID, + NumRows: segment.NumOfRows, + IndexID: indexID, + BuildID: buildID, + CreatedUTCTime: uint64(time.Now().Unix()), + WriteHandoff: false, } if err = s.meta.indexMeta.AddSegmentIndex(segIndex); err != nil { return err diff --git a/internal/datacoord/index_service_test.go b/internal/datacoord/index_service_test.go index 435a54dbbc32e..f6cad9e7532ac 100644 --- a/internal/datacoord/index_service_test.go +++ b/internal/datacoord/index_service_test.go @@ -400,152 +400,152 @@ func TestServer_AlterIndex(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, segID - 1: { indexID: { - SegmentID: segID - 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - CreateTime: createTS, + SegmentID: segID - 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + CreatedUTCTime: createTS, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - CreateTime: createTS, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + CreatedUTCTime: createTS, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - CreateTime: createTS, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + CreatedUTCTime: createTS, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - CreateTime: createTS, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + CreatedUTCTime: createTS, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - CreateTime: createTS, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + CreatedUTCTime: createTS, }, }, }, @@ -808,21 +808,21 @@ func TestServer_GetIndexState(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 3000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_IndexStateNone, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 3000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_IndexStateNone, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, }, @@ -933,21 +933,21 @@ func TestServer_GetSegmentIndexState(t *testing.T) { }, } s.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: 10, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1025, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: 10, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 1025, + WriteHandoff: false, }) s.meta.segments.SetSegment(segID, &SegmentInfo{ SegmentInfo: &datapb.SegmentInfo{ @@ -970,21 +970,21 @@ func TestServer_GetSegmentIndexState(t *testing.T) { t.Run("finish", func(t *testing.T) { s.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: 10, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 1025, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: 10, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 1025, + WriteHandoff: false, }) resp, err := s.GetSegmentIndexState(ctx, req) assert.NoError(t, err) @@ -1092,21 +1092,21 @@ func TestServer_GetIndexBuildProgress(t *testing.T) { t.Run("finish", func(t *testing.T) { s.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10250, - IndexID: indexID, - BuildID: 10, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: []string{"file1", "file2"}, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10250, + IndexID: indexID, + BuildID: 10, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: []string{"file1", "file2"}, + IndexSize: 0, + WriteHandoff: false, }) s.meta.segments = NewSegmentsInfo() s.meta.segments.SetSegment(segID, &SegmentInfo{ @@ -1354,152 +1354,152 @@ func TestServer_DescribeIndex(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, segID - 1: { indexID: { - SegmentID: segID - 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - CreateTime: createTS, + SegmentID: segID - 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + CreatedUTCTime: createTS, }, indexID + 1: { - SegmentID: segID - 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - CreateTime: createTS, + SegmentID: segID - 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + CreatedUTCTime: createTS, }, indexID + 3: { - SegmentID: segID - 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - CreateTime: createTS, + SegmentID: segID - 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + CreatedUTCTime: createTS, }, indexID + 4: { - SegmentID: segID - 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - CreateTime: createTS, + SegmentID: segID - 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + CreatedUTCTime: createTS, }, indexID + 5: { - SegmentID: segID - 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - CreateTime: createTS, + SegmentID: segID - 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + CreatedUTCTime: createTS, }, }, }, @@ -1859,89 +1859,89 @@ func TestServer_GetIndexStatistics(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 1: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 1, - BuildID: buildID + 1, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 1, + BuildID: buildID + 1, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 3: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 3, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 3, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 4: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 4, - BuildID: buildID + 4, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "mock failed", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 4, + BuildID: buildID + 4, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "mock failed", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, indexID + 5: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID + 5, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID + 5, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, }, @@ -2251,21 +2251,21 @@ func TestServer_GetIndexInfos(t *testing.T) { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 10000, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: createTS, - IndexFileKeys: nil, - IndexSize: 0, - WriteHandoff: false, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 10000, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: createTS, + IndexFileKeys: nil, + IndexSize: 0, + WriteHandoff: false, }, }, }, diff --git a/internal/datacoord/metrics_info.go b/internal/datacoord/metrics_info.go index 60ca296679dcb..165a42bbee592 100644 --- a/internal/datacoord/metrics_info.go +++ b/internal/datacoord/metrics_info.go @@ -37,6 +37,7 @@ import ( "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" + "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" "github.com/milvus-io/milvus/pkg/util/uniquegenerator" ) @@ -91,7 +92,7 @@ func (s *Server) getChannelsJSON(ctx context.Context, req *milvuspb.GetMetricsRe channel2Checkpoints := s.meta.GetChannelCheckpoints() for _, channel := range channels { if cp, ok := channel2Checkpoints[channel.Name]; ok { - channel.CheckpointTS = typeutil.TimestampToString(cp.GetTimestamp()) + channel.CheckpointTS = tsoutil.PhysicalTimeFormat(cp.GetTimestamp()) } else { log.Warn("channel not found in meta cache", zap.String("channel", channel.Name)) } @@ -139,15 +140,11 @@ func (s *Server) getDistJSON(ctx context.Context, req *milvuspb.GetMetricsReques dmChannel := metrics.NewDMChannelFrom(chInfo.GetVchan()) dmChannel.NodeID = nodeID dmChannel.WatchState = chInfo.State.String() - dmChannel.StartWatchTS = chInfo.GetStartTs() + dmChannel.StartWatchTS = typeutil.TimestampToString(uint64(chInfo.GetStartTs())) channels = append(channels, dmChannel) } } - if len(segments) == 0 && len(channels) == 0 { - return "" - } - dist := &metricsinfo.DataCoordDist{ Segments: segments, DMChannels: channels, diff --git a/internal/datacoord/metrics_info_test.go b/internal/datacoord/metrics_info_test.go index 1151dc70b5d13..7ad368c2831a9 100644 --- a/internal/datacoord/metrics_info_test.go +++ b/internal/datacoord/metrics_info_test.go @@ -34,6 +34,7 @@ import ( "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" + "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -216,8 +217,8 @@ func TestGetSyncTaskMetrics(t *testing.T) { SegmentID: 1, BatchRows: 100, SegmentLevel: "L0", - TSFrom: 1000, - TSTo: 2000, + TSFrom: "t1", + TSTo: "t2", DeltaRowCount: 10, FlushSize: 1024, RunningTime: "2h", @@ -502,7 +503,7 @@ func TestGetChannelsJSON(t *testing.T) { Name: "channel1", CollectionID: 100, NodeID: 1, - CheckpointTS: typeutil.TimestampToString(1000), + CheckpointTS: tsoutil.PhysicalTimeFormat(1000), }, } channelsBytes, err = json.Marshal(channels) @@ -678,7 +679,7 @@ func TestGetDistJSON(t *testing.T) { cm.EXPECT().GetChannelWatchInfos().Return(map[int64]map[string]*datapb.ChannelWatchInfo{}) svr.channelManager = cm - expectedJSON := "" + expectedJSON := "{}" actualJSON := svr.getDistJSON(ctx, req) assert.Equal(t, expectedJSON, actualJSON) }) diff --git a/internal/datacoord/server_test.go b/internal/datacoord/server_test.go index d8595570887ab..e9b8369c9a738 100644 --- a/internal/datacoord/server_test.go +++ b/internal/datacoord/server_test.go @@ -1607,20 +1607,20 @@ func TestGetRecoveryInfo(t *testing.T) { }) assert.NoError(t, err) svr.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: seg4.ID, - CollectionID: 0, - PartitionID: 0, - NumRows: 100, - IndexID: 0, - BuildID: 0, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: seg4.ID, + CollectionID: 0, + PartitionID: 0, + NumRows: 100, + IndexID: 0, + BuildID: 0, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }) req := &datapb.GetRecoveryInfoRequest{ diff --git a/internal/datacoord/services_test.go b/internal/datacoord/services_test.go index d8b813e702aab..2e120012cdff9 100644 --- a/internal/datacoord/services_test.go +++ b/internal/datacoord/services_test.go @@ -1258,20 +1258,20 @@ func TestGetRecoveryInfoV2(t *testing.T) { }) assert.NoError(t, err) svr.meta.indexMeta.updateSegmentIndex(&model.SegmentIndex{ - SegmentID: seg4.ID, - CollectionID: 0, - PartitionID: 0, - NumRows: 100, - IndexID: 0, - BuildID: 0, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: seg4.ID, + CollectionID: 0, + PartitionID: 0, + NumRows: 100, + IndexID: 0, + BuildID: 0, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }) ch := &channelMeta{Name: "vchan1", CollectionID: 0} diff --git a/internal/datacoord/task_scheduler_test.go b/internal/datacoord/task_scheduler_test.go index b1c6b1404c7b2..c4f31d29b7f98 100644 --- a/internal/datacoord/task_scheduler_test.go +++ b/internal/datacoord/task_scheduler_test.go @@ -58,190 +58,190 @@ var ( func createIndexMeta(catalog metastore.DataCoordCatalog) *indexMeta { indexBuildInfo := newSegmentIndexBuildInfo() indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 2, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 2, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: true, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 2, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 2, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: true, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 3, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 3, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 4, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 4, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 4, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 4, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 5, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 5, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 6, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 6, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 6, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 6, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 7, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 7, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "error", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 7, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 7, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "error", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 8, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 8, - NodeID: nodeID + 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 8, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 8, + NodeID: nodeID + 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 9, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 9, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 9, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 9, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) indexBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID + 10, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 10, - NodeID: nodeID, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 10, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 10, + NodeID: nodeID, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }) return &indexMeta{ @@ -278,200 +278,200 @@ func createIndexMeta(catalog metastore.DataCoordCatalog) *indexMeta { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: 1025, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: 1025, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 1: { indexID: { - SegmentID: segID + 1, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 1, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 1, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 1, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 2: { indexID: { - SegmentID: segID + 2, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 2, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: true, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 2, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 2, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: true, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 3: { indexID: { - SegmentID: segID + 3, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 3, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 3, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 3, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 4: { indexID: { - SegmentID: segID + 4, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 4, - NodeID: nodeID, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 4, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 4, + NodeID: nodeID, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 5: { indexID: { - SegmentID: segID + 5, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 5, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 5, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 5, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 6: { indexID: { - SegmentID: segID + 6, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 6, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Finished, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 6, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 6, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Finished, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 7: { indexID: { - SegmentID: segID + 7, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 7, - NodeID: 0, - IndexVersion: 1, - IndexState: commonpb.IndexState_Failed, - FailReason: "error", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 7, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 7, + NodeID: 0, + IndexVersion: 1, + IndexState: commonpb.IndexState_Failed, + FailReason: "error", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 8: { indexID: { - SegmentID: segID + 8, - CollectionID: collID, - PartitionID: partID, - NumRows: 1026, - IndexID: indexID, - BuildID: buildID + 8, - NodeID: nodeID + 1, - IndexVersion: 1, - IndexState: commonpb.IndexState_InProgress, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 8, + CollectionID: collID, + PartitionID: partID, + NumRows: 1026, + IndexID: indexID, + BuildID: buildID + 8, + NodeID: nodeID + 1, + IndexVersion: 1, + IndexState: commonpb.IndexState_InProgress, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 9: { indexID: { - SegmentID: segID + 9, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 9, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 9, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 9, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, segID + 10: { indexID: { - SegmentID: segID + 10, - CollectionID: collID, - PartitionID: partID, - NumRows: 500, - IndexID: indexID, - BuildID: buildID + 10, - NodeID: nodeID, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 1111, - IndexFileKeys: nil, - IndexSize: 1, + SegmentID: segID + 10, + CollectionID: collID, + PartitionID: partID, + NumRows: 500, + IndexID: indexID, + BuildID: buildID + 10, + NodeID: nodeID, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 1111, + IndexFileKeys: nil, + IndexSize: 1, }, }, }, @@ -1541,20 +1541,20 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() { segmentIndexes: map[UniqueID]map[UniqueID]*model.SegmentIndex{ segID: { indexID: { - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: minNumberOfRowsToBuild, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: minNumberOfRowsToBuild, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }, }, }, @@ -1583,20 +1583,20 @@ func (s *taskSchedulerSuite) Test_indexTaskWithMvOptionalScalarField() { } mt.indexMeta.segmentBuildInfo.Add(&model.SegmentIndex{ - SegmentID: segID, - CollectionID: collID, - PartitionID: partID, - NumRows: minNumberOfRowsToBuild, - IndexID: indexID, - BuildID: buildID, - NodeID: 0, - IndexVersion: 0, - IndexState: commonpb.IndexState_Unissued, - FailReason: "", - IsDeleted: false, - CreateTime: 0, - IndexFileKeys: nil, - IndexSize: 0, + SegmentID: segID, + CollectionID: collID, + PartitionID: partID, + NumRows: minNumberOfRowsToBuild, + IndexID: indexID, + BuildID: buildID, + NodeID: 0, + IndexVersion: 0, + IndexState: commonpb.IndexState_Unissued, + FailReason: "", + IsDeleted: false, + CreatedUTCTime: 0, + IndexFileKeys: nil, + IndexSize: 0, }) cm := mocks.NewChunkManager(s.T()) cm.EXPECT().RootPath().Return("ut-index") diff --git a/internal/distributed/proxy/httpserver/handler.go b/internal/distributed/proxy/httpserver/handler.go index 7bb8b5b20a557..6b781181d9d6f 100644 --- a/internal/distributed/proxy/httpserver/handler.go +++ b/internal/distributed/proxy/httpserver/handler.go @@ -27,9 +27,6 @@ func (h *Handlers) RegisterRoutesTo(router gin.IRouter) { router.GET("/health", wrapHandler(h.handleGetHealth)) router.POST("/dummy", wrapHandler(h.handleDummy)) - router.GET("/databases", wrapHandler(h.handleListDatabases)) - router.GET("/database", wrapHandler(h.handleDescribeDatabases)) - router.POST("/collection", wrapHandler(h.handleCreateCollection)) router.DELETE("/collection", wrapHandler(h.handleDropCollection)) router.GET("/collection/existence", wrapHandler(h.handleHasCollection)) @@ -99,24 +96,6 @@ func (h *Handlers) handleDummy(c *gin.Context) (interface{}, error) { return h.proxy.Dummy(c, &req) } -func (h *Handlers) handleListDatabases(c *gin.Context) (interface{}, error) { - req := milvuspb.ListDatabasesRequest{} - err := shouldBind(c, &req) - if err != nil { - return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err) - } - return h.proxy.ListDatabases(c, &req) -} - -func (h *Handlers) handleDescribeDatabases(c *gin.Context) (interface{}, error) { - req := milvuspb.DescribeDatabaseRequest{} - err := shouldBind(c, &req) - if err != nil { - return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err) - } - return h.proxy.DescribeDatabase(c, &req) -} - func (h *Handlers) handleCreateCollection(c *gin.Context) (interface{}, error) { wrappedReq := WrappedCreateCollectionRequest{} err := shouldBind(c, &wrappedReq) diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index 6c4bd02a3e10f..a4ac8ae1e10fb 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -456,13 +456,6 @@ func (s *Server) init() error { } } - if HTTPParams.Enabled.GetAsBool() { - registerHTTPHandlerOnce.Do(func() { - log.Info("register Proxy http server") - s.registerHTTPServer() - }) - } - if s.rootCoordClient == nil { var err error log.Debug("create RootCoord client for Proxy") @@ -529,6 +522,13 @@ func (s *Server) init() error { s.proxy.SetQueryCoordClient(s.queryCoordClient) log.Debug("set QueryCoord client for Proxy done") + if HTTPParams.Enabled.GetAsBool() { + registerHTTPHandlerOnce.Do(func() { + log.Info("register Proxy http server") + s.registerHTTPServer() + }) + } + log.Debug(fmt.Sprintf("update Proxy's state to %s", commonpb.StateCode_Initializing.String())) s.proxy.UpdateStateCode(commonpb.StateCode_Initializing) diff --git a/internal/flushcommon/pipeline/flow_graph_manager.go b/internal/flushcommon/pipeline/flow_graph_manager.go index 1cf36a177417f..468a033c74b56 100644 --- a/internal/flushcommon/pipeline/flow_graph_manager.go +++ b/internal/flushcommon/pipeline/flow_graph_manager.go @@ -28,6 +28,7 @@ import ( "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" + "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -127,7 +128,7 @@ func (fm *fgManagerImpl) GetChannelsJSON() string { channels = append(channels, &metricsinfo.Channel{ Name: ch, WatchState: ds.fg.Status(), - LatestTimeTick: typeutil.TimestampToString(latestTimeTick), + LatestTimeTick: tsoutil.PhysicalTimeFormat(latestTimeTick), NodeID: paramtable.GetNodeID(), CollectionID: ds.metacache.Collection(), }) diff --git a/internal/flushcommon/pipeline/flow_graph_manager_test.go b/internal/flushcommon/pipeline/flow_graph_manager_test.go index 3cb27e74459f7..3f69ccd5400cb 100644 --- a/internal/flushcommon/pipeline/flow_graph_manager_test.go +++ b/internal/flushcommon/pipeline/flow_graph_manager_test.go @@ -43,6 +43,7 @@ import ( "github.com/milvus-io/milvus/pkg/mq/msgstream" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" + "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -182,7 +183,7 @@ func TestGetChannelsJSON(t *testing.T) { { Name: "fake-ch-_1", WatchState: "Healthy", - LatestTimeTick: typeutil.TimestampToString(0), + LatestTimeTick: tsoutil.PhysicalTimeFormat(0), NodeID: paramtable.GetNodeID(), CollectionID: 1, }, diff --git a/internal/flushcommon/syncmgr/task.go b/internal/flushcommon/syncmgr/task.go index 9ebcbaae38291..1f0e4f8c3059c 100644 --- a/internal/flushcommon/syncmgr/task.go +++ b/internal/flushcommon/syncmgr/task.go @@ -42,6 +42,7 @@ import ( "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/retry" "github.com/milvus-io/milvus/pkg/util/timerecord" + "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -422,8 +423,8 @@ func (t *SyncTask) MarshalJSON() ([]byte, error) { SegmentID: t.segmentID, BatchRows: t.batchRows, SegmentLevel: t.level.String(), - TSFrom: t.tsFrom, - TSTo: t.tsTo, + TSFrom: tsoutil.PhysicalTimeFormat(t.tsFrom), + TSTo: tsoutil.PhysicalTimeFormat(t.tsTo), DeltaRowCount: t.deltaRowCount, FlushSize: t.flushedSize, RunningTime: t.execTime.String(), diff --git a/internal/flushcommon/syncmgr/task_test.go b/internal/flushcommon/syncmgr/task_test.go index bb2c334e785d9..29e103b437c15 100644 --- a/internal/flushcommon/syncmgr/task_test.go +++ b/internal/flushcommon/syncmgr/task_test.go @@ -400,8 +400,8 @@ func (s *SyncTaskSuite) TestSyncTask_MarshalJSON() { SegmentID: t.segmentID, BatchRows: t.batchRows, SegmentLevel: t.level.String(), - TSFrom: t.tsFrom, - TSTo: t.tsTo, + TSFrom: tsoutil.PhysicalTimeFormat(t.tsFrom), + TSTo: tsoutil.PhysicalTimeFormat(t.tsTo), DeltaRowCount: t.deltaRowCount, FlushSize: t.flushedSize, RunningTime: t.execTime.String(), diff --git a/internal/http/router.go b/internal/http/router.go index b0e4463f04236..4ce447d69c4ab 100644 --- a/internal/http/router.go +++ b/internal/http/router.go @@ -109,4 +109,14 @@ const ( DNSegmentsPath = "/_dn/segments" // DNChannelsPath is the path to get channels in DataNode. DNChannelsPath = "/_dn/channels" + + // DatabaseListPath is the path to get all databases. + DatabaseListPath = "/_db/list" + // DatabaseDescPath is the path to get database description. + DatabaseDescPath = "/_db/desc" + + // CollectionListPath is the path to get all collections. + CollectionListPath = "/_collection/list" + // CollectionDescPath is the path to get collection description. + CollectionDescPath = "/_collection/desc" ) diff --git a/internal/http/webui/collections.html b/internal/http/webui/collections.html index e6a9904ff76c2..21fda61d92779 100644 --- a/internal/http/webui/collections.html +++ b/internal/http/webui/collections.html @@ -99,7 +99,7 @@

// handleError(error); // }); - fetchData(MILVUS_URI + "/databases", databases) + fetchData(MILVUS_URI + "/_db/list", listDatabaseData) .then(data => { databaseData = data; renderDatabases(startPage, paginationSize) diff --git a/internal/http/webui/static/js/common.js b/internal/http/webui/static/js/common.js index 2757aabc73b3d..7a7b471a83059 100644 --- a/internal/http/webui/static/js/common.js +++ b/internal/http/webui/static/js/common.js @@ -31,21 +31,11 @@ const handleError = (error) => { // window.location.href = `5xx.html?error=${errorMessage}`; }; -const fetchData = (url, localData, kvParams) => { +const fetchData = (url, localData) => { if (DEBUG_MODE) { return new Promise((resolve) => { resolve(JSON.parse(localData)); }); - } else if (kvParams && kvParams.length !== 0) { - return fetch(url, { - method: 'POST', - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json', - }, - mode: 'no-cors', - body: JSON.stringify(kvParams) - }).then(response => response.json()) } else { return fetch(url).then(response => { return response.json(); @@ -62,18 +52,4 @@ function getQueryParams() { params[decodeURIComponent(key)] = decodeURIComponent(value || ''); }); return params; -} - -function formatTimestamp(timestamp) { - const date = new Date(timestamp); // Convert timestamp to a Date object - // Format the date components - const year = date.getFullYear(); - const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are zero-indexed - const day = ('0' + date.getDate()).slice(-2); - const hours = ('0' + date.getHours()).slice(-2); - const minutes = ('0' + date.getMinutes()).slice(-2); - const seconds = ('0' + date.getSeconds()).slice(-2); - - // Return formatted date string - return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}` } \ No newline at end of file diff --git a/internal/http/webui/static/js/mockdata.js b/internal/http/webui/static/js/mockdata.js index 1e21d99672484..88e047384c682 100644 --- a/internal/http/webui/static/js/mockdata.js +++ b/internal/http/webui/static/js/mockdata.js @@ -445,38 +445,67 @@ const mconfigs = ` } `; -const collections =` +const listCollectionData =` { - "status": { - "error_code": "Success", - "reason": "" - }, "collection_names": [ - "collection1", - "collection2", - "collection3", - "collection4", - "collection5", - "collection6", - "collection7", - "collection8", - "collection9", - "collection10" + "collection_1", + "collection_2", + "collection_3", + "collection_4", + "collection_5", + "collection_6", + "collection_7", + "collection_8", + "collection_9", + "collection_10" ], "collection_ids": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 - ], - "created_timestamps": [ - 1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400 + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" ], "created_utc_timestamps": [ - 1633036800, 1633123200, 1633209600, 1633296000, 1633382400, 1633468800, 1633555200, 1633641600, 1633728000, 1633814400 + "2021-10-01 00:00:00", + "2021-10-02 00:00:00", + "2021-10-03 00:00:00", + "2021-10-04 00:00:00", + "2021-10-05 00:00:00", + "2021-10-06 00:00:00", + "2021-10-07 00:00:00", + "2021-10-08 00:00:00", + "2021-10-09 00:00:00", + "2021-10-10 00:00:00" ], "inMemory_percentages": [ - 100, 90, 80, 70, 60, 50, 40, 30, 20, 10 + 100, + 90, + 80, + 70, + 60, + 50, + 40, + 30, + 20, + 10 ], "query_service_available": [ - true, false, false, false, false, false, false, false, false, false + true, + true, + true, + true, + true, + false, + false, + false, + false, + false ] } ` @@ -556,63 +585,57 @@ const collectionRequest = ` ] ` -const describeCollectionResp = ` +const describeCollectionData = ` { - "status": { - "error_code": 0, - "reason": "Success" - }, - "schema": { - "name": "example_collection", - "description": "This is an example collection schema", - "fields": [ - { - "name": "field1", - "data_type": "INT64", - "is_primary_key": true, - "auto_id": false - }, - { - "name": "field2", - "data_type": "FLOAT", - "is_primary_key": false, - "auto_id": false - } - ] - }, - "collectionID": 12345, - "virtual_channel_names": ["vchan1", "vchan2"], - "physical_channel_names": ["pchan1", "pchan2"], - "created_timestamp": 1633036800, - "created_utc_timestamp": 1633036800, + "collection_id": "1", + "collection_name": "example_collection", + "created_time": "2021-10-01 00:00:00", "shards_num": 2, + "consistency_level": "Strong", "aliases": ["alias1", "alias2"], - "start_positions": [ + "properties": { + "property_key": "property_value" + }, + "db_name": "example_db", + "num_partitions": 3, + "virtual_channel_names": ["v_channel1", "v_channel2"], + "physical_channel_names": ["p_channel1", "p_channel2"], + "partition_infos": [ { - "key": "start_key", - "data": "start_data" + "partition_name": "partition1", + "partition_id": "1", + "created_utc_timestamp": "2021-10-01 00:00:00" } ], - "consistency_level": 0, - "collection_name": "example_collection", - "properties": [ + "enable_dynamic_field": true, + "fields": [ { - "key": "property_key", - "value": "property_value" + "field_id": "1", + "name": "field1", + "is_primary_key": true, + "description": "description1", + "data_type": "int64", + "type_params": { + "param_key": "param_value" + }, + "index_params": { + "index_key": "index_value" + }, + "auto_id": false, + "element_type": "element_type1", + "default_value": "default_value1", + "is_dynamic": false, + "is_partition_key": false, + "is_clustering_key": false, + "nullable": true, + "is_function_output": false } - ], - "db_name": "example_db", - "num_partitions": 1, - "db_id": 1 + ] } ` -const databases = ` +const listDatabaseData = ` { - "status": { - "error_code": "Success", - "reason": "" - }, "db_names": [ "database_1", "database_2", @@ -625,33 +648,38 @@ const databases = ` "database_9", "database_10" ], - "created_timestamp": [ - 1633036800, - 1633123200, - 1633209600, - 1633296000, - 1633382400, - 1633468800, - 1633555200, - 1633641600, - 1633728000, - 1633814400 - ], "db_ids": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "created_timestamps": [ + "2021-10-01 00:00:00", + "2021-10-02 00:00:00", + "2021-10-03 00:00:00", + "2021-10-04 00:00:00", + "2021-10-05 00:00:00", + "2021-10-06 00:00:00", + "2021-10-07 00:00:00", + "2021-10-08 00:00:00", + "2021-10-09 00:00:00", + "2021-10-10 00:00:00" ] } ` -const describeDatabaseResp = ` +const describeDatabaseData = ` { - "status": { - "error_code": 0, - "reason": "Success" - }, "db_name": "example_db", - "dbID": 1, - "created_timestamp": 1633036800, + "db_id": 1, + "created_timestamp": "2021-10-01 00:00:00", "properties": [ { "key": "property_key", @@ -679,7 +707,7 @@ const qcCurrentTargets = ` "is_sorted": true, "node_id": 1, "is_invisible": false, - "loaded_timestamp": 1633072800, + "loaded_timestamp": "2021-10-01 00:00:00", "index": [ { "field_id": 1, @@ -701,20 +729,17 @@ const qcCurrentTargets = ` "collection_id": 1, "channel_name": "channel1", "unflushed_segment_ids": [ - 1 + "1" ], "flushed_segment_ids": [ - 2 + "2" ], "dropped_segment_ids": [ - 3 + "3" ], "level_zero_segment_ids": [ - 4 - ], - "partition_stats_versions": { - "1": 1 - } + "4" + ] } ] } @@ -774,7 +799,7 @@ const qcDist = ` "is_sorted": true, "node_id": 1, "is_invisible": false, - "loaded_timestamp": 1633072800, + "loaded_timestamp": "2021-10-01 00:00:00", "index": [ { "field_id": 1, @@ -798,15 +823,15 @@ const qcDist = ` "version": 1, "collection_id": 1, "channel_name": "channel1", - "unflushed_segment_ids": [1], - "flushed_segment_ids": [2], - "dropped_segment_ids": [3], - "level_zero_segment_ids": [4], + "unflushed_segment_ids": ["1"], + "flushed_segment_ids": ["2"], + "dropped_segment_ids": ["3"], + "level_zero_segment_ids": ["4"], "partition_stats_versions": { "1": 1 }, "watch_state": "Healthy", - "start_watch_ts": 1633072800 + "start_watch_ts": "2021-10-01 00:00:00" } ], "leader_views": [ @@ -830,7 +855,7 @@ const qcDist = ` "is_sorted": true, "node_id": 1, "is_invisible": false, - "loaded_timestamp": 1633072800, + "loaded_timestamp": "2021-10-01 00:00:00", "index": [ { "field_id": 1, @@ -958,7 +983,7 @@ const qnSegments = ` "is_sorted": true, "node_id": 1, "is_invisible": false, - "loaded_timestamp": 1620000000, + "loaded_timestamp": "2021-10-01 00:00:00", "index": [ { "field_id": 1, @@ -985,7 +1010,7 @@ const qnSegments = ` "is_sorted": true, "node_id": 2, "is_invisible": false, - "loaded_timestamp": 1620000001, + "loaded_timestamp": "2021-10-01 00:00:00", "index": [ { "field_id": 2, @@ -1059,22 +1084,22 @@ const dc_dist = ` "version": 1, "collection_id": 100, "channel_name": "channel1", - "unflushed_segment_ids": [1, 2, 3], - "flushed_segment_ids": [4, 5, 6], - "dropped_segment_ids": [7, 8, 9], + "unflushed_segment_ids": ["1", "2", "3"], + "flushed_segment_ids": ["4", "5", "6"], + "dropped_segment_ids": ["7", "8", "9"], "watch_state": "success", - "start_watch_ts": 123456789 + "start_watch_ts": "2023-10-01 12:05:00" }, { "node_id": 1, "version": 1, "collection_id": 100, "channel_name": "channel3", - "unflushed_segment_ids": [1, 2, 3], - "flushed_segment_ids": [4, 5, 6], - "dropped_segment_ids": [7, 8, 9], + "unflushed_segment_ids": ["1", "2", "3"], + "flushed_segment_ids": ["4", "5", "6"], + "dropped_segment_ids": ["7", "8", "9"], "watch_state": "to_watch", - "start_watch_ts": 123456789 + "start_watch_ts": "2023-10-01 12:05:00" } ] } @@ -1090,7 +1115,7 @@ const dc_build_index_task = ` "index_state": "Finished", "index_size": 1024, "index_version": 1, - "create_time": 1633036800 + "create_time": "2023-10-01 12:05:00" }, { "index_id": 2, @@ -1101,7 +1126,7 @@ const dc_build_index_task = ` "fail_reason": "Disk full", "index_size": 2048, "index_version": 2, - "create_time": 1633123200 + "create_time": "2023-10-01 12:05:00" } ]` @@ -1113,11 +1138,11 @@ const dc_compaction_task = ` "type": "Merge", "state": "Completed", "fail_reason": "", - "start_time": 1620000000, - "end_time": 1620003600, + "start_time": "2023-10-01 12:05:00", + "end_time": "2023-10-01 12:06:00", "total_rows": 10000, - "input_segments": [1, 2, 3], - "result_segments": [4] + "input_segments": ["1", "2", "3"], + "result_segments": ["4"] }, { "plan_id": 2, @@ -1125,10 +1150,10 @@ const dc_compaction_task = ` "type": "Merge", "state": "Failed", "fail_reason": "Disk full", - "start_time": 1620007200, - "end_time": 1620010800, + "start_time": "2023-10-01 12:05:00", + "end_time": "2023-10-01 12:06:00", "total_rows": 20000, - "input_segments": [5, 6, 7], + "input_segments": ["5", "6", "7"], "result_segments": [] } ]` @@ -1139,8 +1164,8 @@ const dn_sync_task = ` "segment_id": 1, "batch_rows": 1000, "segment_level": "L1", - "ts_from": 1633036800, - "ts_to": 1633040400, + "ts_from": "2023-10-01 12:05:00", + "ts_to": "2023-10-01 12:06:00", "delta_row_count": 10, "flush_size": 1024, "running_time": "100000000", @@ -1150,8 +1175,8 @@ const dn_sync_task = ` "segment_id": 2, "batch_rows": 2000, "segment_level": "L2", - "ts_from": 1633123200, - "ts_to": 1633126800, + "ts_from": "2023-10-01 12:05:00", + "ts_to": "2023-10-01 12:06:00", "delta_row_count": 20, "flush_size": 2048, "running_time": "200000000", diff --git a/internal/http/webui/static/js/render.js b/internal/http/webui/static/js/render.js index a7e55df81eb37..71cb032e35da6 100644 --- a/internal/http/webui/static/js/render.js +++ b/internal/http/webui/static/js/render.js @@ -126,7 +126,7 @@ function renderDatabases(currentPage, rowsPerPage) { tableHTML += ''; tableHTML += `${databaseData.db_names[i]}`; tableHTML += `${databaseData.db_ids? databaseData.db_ids[i] : 0}`; - tableHTML += `${databaseData.created_timestamp? formatTimestamp(databaseData.created_timestamp[i]) : ''}`; + tableHTML += `${databaseData.created_timestamps[i]}`; tableHTML += ''; // Hidden row for displaying collection details as JSON @@ -165,7 +165,7 @@ function renderDatabases(currentPage, rowsPerPage) { } function describeDatabase(databaseName, rowIndex, type) { - fetchData(`${MILVUS_URI}/database?db_name=${databaseName}`, describeDatabaseResp) + fetchData(`${MILVUS_URI}/_db/desc?db_name=${databaseName}`, describeDatabaseData) .then(data => { // Format data as JSON and insert into the designated row const jsonFormattedData = JSON.stringify(data, null, 2); @@ -182,7 +182,7 @@ function describeDatabase(databaseName, rowIndex, type) { } function describeCollection(databaseName, collectionName, rowIndex, type) { - fetchData(`${MILVUS_URI}/collection?db_name${databaseName}&&collection_name=${collectionName}`, describeCollectionResp) + fetchData(`${MILVUS_URI}/_collection/desc?db_name=${databaseName}&&collection_name=${collectionName}`, describeCollectionData) .then(data => { // Format data as JSON and insert into the designated row const jsonFormattedData = JSON.stringify(data, null, 2); @@ -199,7 +199,7 @@ function describeCollection(databaseName, collectionName, rowIndex, type) { } function fetchCollections(databaseName) { - fetchData(MILVUS_URI + `/collections?db_name=${databaseName}`, collections ) + fetchData(MILVUS_URI + `/_collection/list?db_name=${databaseName}`, listCollectionData ) .then(data => { collectionsData = data; renderCollections(databaseName, startPage, paginationSize) @@ -212,7 +212,7 @@ function fetchCollections(databaseName) { let collectionsData = null; // Global variable to store fetched data function renderCollections(databaseName, currentPage, rowsPerPage) { let data = collectionsData; - if (!data) { + if (!data || !data.collection_names) { console.error('No collections data available'); return; } @@ -229,12 +229,11 @@ function renderCollections(databaseName, currentPage, rowsPerPage) { const start = currentPage * rowsPerPage; const end = start + rowsPerPage; const totalCount = data.collection_names.length; - console.log(data) for (let i = start; i < end && i < totalCount; i++) { tableHTML += ''; tableHTML += `${data.collection_names[i]}`; tableHTML += `${data.collection_ids[i]}`; - tableHTML += `${formatTimestamp(data.created_utc_timestamps[i])}`; + tableHTML += `${data.created_utc_timestamps[i]}`; tableHTML += `${data.inMemory_percentages? data.inMemory_percentages[i]: 'unknown'}`; tableHTML += `${data.query_service_available? data.query_service_available[i] ? 'Yes' : 'No' : 'No'}`; tableHTML += ''; @@ -509,7 +508,7 @@ function renderDependencies(data) { const tr = ` ${key === 'metastore'? 'metastore [' + row['meta_type'] + ']' : 'mq [' + row['mq_type'] + ']'} - ${row['health_status']? 'Healthy' : 'Unhealthy:' + row['unhealthy_reason']} + ${row['health_status']? 'Healthy' : row['unhealthy_reason']} ${row['members_health']? row['members_health'].map(member => `