diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml index c8d0e8d..d8aa7c4 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml @@ -10,7 +10,7 @@ status: message: "recommended size matches current size" - status: "True" type: ScalingActive - message: "the HPA was unable to compute the replica count" + message: "the HPA was able to compute the replica count" currentMetrics: - containerResource: container: app diff --git a/internal/controller/tortoise_controller_test.go b/internal/controller/tortoise_controller_test.go index 533a340..447081c 100644 --- a/internal/controller/tortoise_controller_test.go +++ b/internal/controller/tortoise_controller_test.go @@ -186,7 +186,7 @@ func updateResourcesInTestCaseFile(path string, resource resources) error { } if resource.hpa != nil { - err = writeToFile(filepath.Join(path, "hpa.yaml"), removeUnnecessaryFieldsFromHPA(resource.hpa)) + err = writeToFile(filepath.Join(path, "hpa.yaml"), resource.hpa) if err != nil { return err } @@ -195,11 +195,6 @@ func updateResourcesInTestCaseFile(path string, resource resources) error { return nil } -func removeUnnecessaryFieldsFromHPA(hpa *v2.HorizontalPodAutoscaler) *v2.HorizontalPodAutoscaler { - //hpa.Status = v2.HorizontalPodAutoscalerStatus{} - return hpa -} - func removeUnnecessaryFieldsFromDeployment(deployment *v1.Deployment) *v1.Deployment { // remove all default values @@ -496,6 +491,9 @@ var _ = Describe("Test TortoiseController", func() { It("HPA scalingactive condition false", func() { runTest(filepath.Join("testdata", "reconcile-automatic-emergency-mode-hpa-condition")) }) + It("HPA scalingactive no metrics", func() { + runTest(filepath.Join("testdata", "reconcile-automatic-emergency-mode-hpa-no-metrics")) + }) }) Context("DeletionPolicy is handled correctly", func() { It("[DeletionPolicy = DeleteAll] delete HPA and VPA when Tortoise is deleted", func() { diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index a416d73..ce4ab3e 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -799,6 +799,7 @@ func (c *Service) CheckHpaMetricStatus(ctx context.Context, currenthpa *v2.Horiz for _, condition := range conditions { if condition.Type == "ScalingActive" && condition.Status == "False" && condition.Reason == "FailedGetResourceMetric" { //switch to Emergency mode since no metrics + logger.Info("HPA failed to get resource metrics, switch to emergency mode") return false } } @@ -807,9 +808,11 @@ func (c *Service) CheckHpaMetricStatus(ctx context.Context, currenthpa *v2.Horiz if len(currentMetrics) > 0 { for _, currentMetric := range currentMetrics { if !currentMetric.ContainerResource.Current.Value.IsZero() { + //Can still get metrics for some containers, scale based on those return true } } + logger.Info("HPA all metrics return 0, switch to emergency mode") return false }