feat: add last updated future/past to metrics

flanksource · Dec 9, 2024 · 18d73c8 · 18d73c8
1 parent 1597e08
commit 18d73c8
Showing 1 changed file with 23 additions and 7 deletions.
diff --git a/scrapers/kubernetes/informers.go b/scrapers/kubernetes/informers.go
@@ -123,9 +123,14 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1
 				ctx.Logger.V(4).Infof("added: %s %s %s", u.GetUID(), u.GetKind(), u.GetName())
 			}
 
-			if u.GetCreationTimestamp().Time.After(start) {
-				ctx.Counter("kubernetes_informer_events", "type", "add", "kind", u.GetKind(), "scraper_id", ctx.ScraperID()).Add(1)
+			ctx.Counter("kubernetes_informer_events",
+				"type", "add",
+				"kind", u.GetKind(),
+				"scraper_id", ctx.ScraperID(),
+				"valid_timestamp", lo.Ternary(u.GetCreationTimestamp().Time.After(start), "true", "false"),
+			).Add(1)
 
+			if u.GetCreationTimestamp().Time.After(start) {
 				ctx.Histogram("informer_receive_lag", informerLagBuckets,
 					"scraper", ctx.ScraperID(),
 					"kind", watchResource.Kind,
@@ -145,21 +150,27 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1
 				return
 			}
 
-			ctx.Counter("kubernetes_informer_events", "type", "update", "kind", u.GetKind(), "scraper_id", ctx.ScraperID()).Add(1)
-
 			if ctx.Properties().On(false, "scraper.log.items") {
 				ctx.Logger.V(3).Infof("updated: %s %s %s", u.GetUID(), u.GetKind(), u.GetName())
 			}
 
 			lastUpdatedTime := health.GetLastUpdatedTime(u)
-			if lastUpdatedTime != nil && lastUpdatedTime.After(u.GetCreationTimestamp().Time) && lastUpdatedTime.Before(time.Now()) {
+			lastUpdatedInPast := lastUpdatedTime != nil && lastUpdatedTime.After(u.GetCreationTimestamp().Time) && lastUpdatedTime.Before(time.Now())
+			if lastUpdatedInPast {
 				ctx.Histogram("informer_receive_lag", informerLagBuckets,
 					"scraper", ctx.ScraperID(),
 					"kind", watchResource.Kind,
 					"operation", "update",
 				).Record(time.Duration(time.Since(*lastUpdatedTime).Milliseconds()))
 			}
 
+			ctx.Counter("kubernetes_informer_events",
+				"type", "update",
+				"kind", u.GetKind(),
+				"scraper_id", ctx.ScraperID(),
+				"valid_timestamp", lo.Ternary(lastUpdatedInPast, "true", "false"),
+			).Add(1)
+
 			queue.Enqueue(NewQueueItem(u, QueueItemOperationUpdate))
 		},
 		DeleteFunc: func(obj any) {
@@ -177,8 +188,6 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1
 				return
 			}
 
-			ctx.Counter("kubernetes_informer_events", "type", "delete", "kind", u.GetKind(), "scraper_id", ctx.ScraperID()).Add(1)
-
 			if ctx.Properties().On(false, "scraper.log.items") {
 				ctx.Logger.V(3).Infof("deleted: %s %s %s", u.GetUID(), u.GetKind(), u.GetName())
 			}
@@ -191,6 +200,13 @@ func (t *SharedInformerManager) Register(ctx api.ScrapeContext, watchResource v1
 				).Record(time.Duration(time.Since(u.GetDeletionTimestamp().Time).Milliseconds()))
 			}
 
+			ctx.Counter("kubernetes_informer_events",
+				"type", "delete",
+				"kind", u.GetKind(),
+				"scraper_id", ctx.ScraperID(),
+				"valid_timestamp", lo.Ternary(u.GetDeletionTimestamp() != nil, "true", "false"),
+			).Add(1)
+
 			queue.Enqueue(NewQueueItem(u, QueueItemOperationDelete))
 		},
 	})