automatic emergency mode trigger when kube metrics unavailable for hpa

mercari · Nov 26, 2024 · ba4351e · ba4351e
1 parent 3725788
commit ba4351e
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 0 deletions.
diff --git a/internal/controller/tortoise_controller.go b/internal/controller/tortoise_controller.go
@@ -185,6 +185,17 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 		return ctrl.Result{}, err
 	}
 	tortoise = tortoiseService.UpdateTortoiseAutoscalingPolicyInStatus(tortoise, hpa, now)
+	scalingActive := r.HpaService.checkHpaMetricStatus(ctx, hpa)
+	if scalingActive == false && tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff {
+		//switch to emergency mode
+		if tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeAuto {
+			tortoise.Spec.UpdateMode = autoscalingv1beta3.UpdateModeEmergency
+		}
+	}
+	if scalingActive == true && tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeEmergency {
+		tortoise.Spec.UpdateMode = autoscalingv1beta3.UpdateModeAuto
+	}
+
 	tortoise = r.TortoiseService.UpdateTortoisePhase(tortoise, now)
 	if tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseInitializing {
 		logger.Info("initializing tortoise", "tortoise", req.NamespacedName)

diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go
@@ -765,3 +765,18 @@ func (c *Service) excludeExternalMetric(ctx context.Context, hpa *v2.HorizontalP
 
 	return newHPA
 }
+
+func (c *Service) checkHpaMetricStatus(ctx context.Context, currenthpa *v2.HorizontalPodAutoscaler, now time.Time) bool {
+	currenthpa = currenthpa.DeepCopy()
+	conditions := currenthpa.Status.Conditions
+
+	if conditions[1].Type == "ScalingActive" && conditions[1].Status == "True" {
+		//switch to Auto mode since metrics are back
+		return true
+	}
+
+	if conditions[1].Type == "ScalingActive" && conditions[1].Status == "False" && conditions[1].Reason == "FailedGetResourceMetric" {
+		//switch to Emergency mode since no metrics
+		return false
+	}
+}