From 18d73c85b6169362f918740a7c54f1dc9a032fcc Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 9 Dec 2024 11:19:30 +0545 Subject: [PATCH] feat: add last updated future/past to metrics --- scrapers/kubernetes/informers.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scrapers/kubernetes/informers.go b/scrapers/kubernetes/informers.go index b7cc7c5b..b7dcb443 100644 --- a/scrapers/kubernetes/informers.go +++ b/scrapers/kubernetes/informers.go @@ -123,9 +123,14 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1 ctx.Logger.V(4).Infof("added: %s %s %s", u.GetUID(), u.GetKind(), u.GetName()) } - if u.GetCreationTimestamp().Time.After(start) { - ctx.Counter("kubernetes_informer_events", "type", "add", "kind", u.GetKind(), "scraper_id", ctx.ScraperID()).Add(1) + ctx.Counter("kubernetes_informer_events", + "type", "add", + "kind", u.GetKind(), + "scraper_id", ctx.ScraperID(), + "valid_timestamp", lo.Ternary(u.GetCreationTimestamp().Time.After(start), "true", "false"), + ).Add(1) + if u.GetCreationTimestamp().Time.After(start) { ctx.Histogram("informer_receive_lag", informerLagBuckets, "scraper", ctx.ScraperID(), "kind", watchResource.Kind, @@ -145,14 +150,13 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1 return } - ctx.Counter("kubernetes_informer_events", "type", "update", "kind", u.GetKind(), "scraper_id", ctx.ScraperID()).Add(1) - if ctx.Properties().On(false, "scraper.log.items") { ctx.Logger.V(3).Infof("updated: %s %s %s", u.GetUID(), u.GetKind(), u.GetName()) } lastUpdatedTime := health.GetLastUpdatedTime(u) - if lastUpdatedTime != nil && lastUpdatedTime.After(u.GetCreationTimestamp().Time) && lastUpdatedTime.Before(time.Now()) { + lastUpdatedInPast := lastUpdatedTime != nil && lastUpdatedTime.After(u.GetCreationTimestamp().Time) && lastUpdatedTime.Before(time.Now()) + if lastUpdatedInPast { ctx.Histogram("informer_receive_lag", informerLagBuckets, "scraper", ctx.ScraperID(), "kind", watchResource.Kind, @@ -160,6 +164,13 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1 ).Record(time.Duration(time.Since(*lastUpdatedTime).Milliseconds())) } + ctx.Counter("kubernetes_informer_events", + "type", "update", + "kind", u.GetKind(), + "scraper_id", ctx.ScraperID(), + "valid_timestamp", lo.Ternary(lastUpdatedInPast, "true", "false"), + ).Add(1) + queue.Enqueue(NewQueueItem(u, QueueItemOperationUpdate)) }, DeleteFunc: func(obj any) { @@ -177,8 +188,6 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1 return } - ctx.Counter("kubernetes_informer_events", "type", "delete", "kind", u.GetKind(), "scraper_id", ctx.ScraperID()).Add(1) - if ctx.Properties().On(false, "scraper.log.items") { ctx.Logger.V(3).Infof("deleted: %s %s %s", u.GetUID(), u.GetKind(), u.GetName()) } @@ -191,6 +200,13 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1 ).Record(time.Duration(time.Since(u.GetDeletionTimestamp().Time).Milliseconds())) } + ctx.Counter("kubernetes_informer_events", + "type", "delete", + "kind", u.GetKind(), + "scraper_id", ctx.ScraperID(), + "valid_timestamp", lo.Ternary(u.GetDeletionTimestamp() != nil, "true", "false"), + ).Add(1) + queue.Enqueue(NewQueueItem(u, QueueItemOperationDelete)) }, })