add counter metrics

mercari · Aug 23, 2024 · d018a7d · d018a7d
1 parent f2cae86
commit d018a7d
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 4 deletions.
diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go
@@ -417,8 +417,6 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet
 	}
 
 	if recordMetrics {
-		metrics.NetHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas) - float64(recommendMin))
-		metrics.NetHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas - recommendMax))
 		metrics.ProposedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(recommendMin))
 		metrics.ProposedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(recommendMax))
 	}
@@ -448,6 +446,16 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet
 	hpa.Spec.MinReplicas = &minToActuallyApply
 	if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics {
 		// We don't want to record applied* metric when UpdateMode is Off.
+		netChangeMaxReplicas := float64(hpa.Spec.MaxReplicas - recommendMax)
+		netChangeMinReplicas := float64(*hpa.Spec.MinReplicas) - float64(recommendMin)
+		if netChangeMaxReplicas > 0 || netChangeMinReplicas < 0 {
+			metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
+		}
+		if netChangeMaxReplicas < 0 || netChangeMinReplicas > 0 {
+			metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
+		}
+		metrics.NetHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicas)
+		metrics.NetHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicas)
 		metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas))
 		metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas))
 	}

diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
@@ -46,6 +46,16 @@ var (
 		Help: "memory request (byte) that tortoises actually applys",
 	}, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"})
 
+	DecreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "decrease_apply_counter",
+		Help: "counter for number of resource decreases applied by tortoise",
+	}, []string{"tortoise_name", "namespace"})
+
+	IncreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "increase_apply_counter",
+		Help: "counter for number of resource increases applied by tortoise",
+	}, []string{"tortoise_name", "namespace"})
+
 	NetHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "net_hpa_minreplicas",
 		Help: "net hpa minReplicas that tortoises actually applys to hpa",
@@ -107,6 +117,12 @@ func init() {
 		AppliedHPAMinReplicas,
 		AppliedCPURequest,
 		AppliedMemoryRequest,
+		IncreaseApplyCounter,
+		DecreaseApplyCounter
+		NetHPAMaxReplicas,
+		NetHPAMinReplicas,
+		NetCPURequest,
+		NetMemoryRequest,
 		ProposedHPATargetUtilization,
 		ProposedHPAMinReplicas,
 		ProposedHPAMaxReplicas,

diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go
@@ -768,14 +768,21 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T
 		// only record metrics once in every reconcile loop.
 		for resourcename, value := range r.Resource {
 			oldRequest := oldRequestMap[r.ContainerName][resourcename]
+			netChange := float64(oldRequest.MilliValue() - value.MilliValue())
 			if resourcename == corev1.ResourceCPU {
 				// We don't want to record applied* metric when UpdateMode is Off.
 				metrics.AppliedCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.MilliValue()))
-				metrics.NetCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(oldRequest.MilliValue() - value.MilliValue()))
+				metrics.NetCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(netChange)
 			}
 			if resourcename == corev1.ResourceMemory {
 				metrics.AppliedMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.Value()))
-				metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(oldRequest.MilliValue() - value.MilliValue()))
+				metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(netChange))
+			}
+			if netChange > 0 {
+				metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
+			}
+			if netChange < 0 {
+				metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1)
 			}
 		}
 	}