From 88eeb92f6fdc5afe7dc576b90f9560667230f8f4 Mon Sep 17 00:00:00 2001
From: Kensei Nakada <handbomusic@gmail.com>
Date: Thu, 12 Oct 2023 17:57:54 +0900
Subject: [PATCH] add applied_hpa_maxreplicas and applied_hpa_minreplicas
 (#180)

---
 pkg/hpa/service.go     | 38 ++++++++++++++++++++------------------
 pkg/metrics/metrics.go | 18 ++++++++++++++----
 2 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go
index 7c22499a..9bfd1237 100644
--- a/pkg/hpa/service.go
+++ b/pkg/hpa/service.go
@@ -382,40 +382,42 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet
 		tortoise = c.RecordHPATargetUtilizationUpdate(tortoise, now)
 	}
 
-	max, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MaxReplicas, now)
+	recommendMax, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MaxReplicas, now)
 	if err != nil {
 		return nil, tortoise, fmt.Errorf("get maxReplicas recommendation: %w", err)
 	}
-	hpa.Spec.MaxReplicas = max
+	// We always set the maxReplicas to the the recommended one.
+	hpa.Spec.MaxReplicas = recommendMax
 
-	var min int32
+	recommendMin, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MinReplicas, now)
+	if err != nil {
+		return nil, tortoise, fmt.Errorf("get minReplicas recommendation: %w", err)
+	}
+	metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(recommendMin))
+	metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(recommendMax))
+
+	// the minReplicas to be applied is not always the same as the recommended one.
+	var minToActuallyApply int32
 	switch tortoise.Status.TortoisePhase {
 	case autoscalingv1beta2.TortoisePhaseEmergency:
 		// when emergency mode, we set the same value on minReplicas.
-		min = max
+		minToActuallyApply = recommendMax
 	case autoscalingv1beta2.TortoisePhaseBackToNormal:
-		idealMin, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MinReplicas, now)
-		if err != nil {
-			return nil, tortoise, fmt.Errorf("get minReplicas recommendation: %w", err)
-		}
 		currentMin := *hpa.Spec.MinReplicas
 		reduced := int32(math.Trunc(float64(currentMin) * c.replicaReductionFactor))
-		if idealMin > reduced {
-			min = idealMin
+		if recommendMin > reduced {
+			minToActuallyApply = recommendMin
 			// BackToNormal is finished
 			tortoise.Status.TortoisePhase = autoscalingv1beta2.TortoisePhaseWorking
 		} else {
-			min = reduced
+			minToActuallyApply = reduced
 		}
 	default:
-		min, err = GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MinReplicas, now)
-		if err != nil {
-			return nil, tortoise, fmt.Errorf("get minReplicas recommendation: %w", err)
-		}
+		minToActuallyApply = recommendMin
 	}
-	hpa.Spec.MinReplicas = &min
-	metrics.ProposedHPAMinReplicass.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas))
-	metrics.ProposedHPAMaxReplicass.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas))
+	hpa.Spec.MinReplicas = &minToActuallyApply
+	metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas))
+	metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas))
 
 	return hpa, tortoise, nil
 }
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index eea02e09..bf7cc752 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -11,17 +11,27 @@ var (
 		Help: "hpa utilization target values that tortoises actually applys to hpa",
 	}, []string{"tortoise_name", "namespace", "container_name", "resource_name", "hpa_name"})
 
+	AppliedHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "applied_hpa_minreplicas",
+		Help: "hpa minReplicas that tortoises actually applys to hpa",
+	}, []string{"tortoise_name", "namespace", "hpa_name"})
+
+	AppliedHPAMaxReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "applied_hpa_maxreplicas",
+		Help: "hpa maxReplicas that tortoises actually applys to hpa",
+	}, []string{"tortoise_name", "namespace", "hpa_name"})
+
 	ProposedHPATargetUtilization = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "proposed_hpa_utilization_target",
 		Help: "recommended hpa utilization target values that tortoises propose",
 	}, []string{"tortoise_name", "namespace", "container_name", "resource_name", "hpa_name"})
 
-	ProposedHPAMinReplicass = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+	ProposedHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "proposed_hpa_minreplicas",
 		Help: "recommended hpa minReplicas that tortoises propose",
 	}, []string{"tortoise_name", "namespace", "hpa_name"})
 
-	ProposedHPAMaxReplicass = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+	ProposedHPAMaxReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "proposed_hpa_maxreplicas",
 		Help: "recommended hpa maxReplicas that tortoises propose",
 	}, []string{"tortoise_name", "namespace", "hpa_name"})
@@ -42,8 +52,8 @@ func init() {
 	metrics.Registry.MustRegister(
 		AppliedHPATargetUtilization,
 		ProposedHPATargetUtilization,
-		ProposedHPAMinReplicass,
-		ProposedHPAMaxReplicass,
+		ProposedHPAMinReplicas,
+		ProposedHPAMaxReplicas,
 		ProposedCPURequest,
 		ProposedMemoryRequest,
 	)