diff --git a/api/v1beta3/tortoise_types.go b/api/v1beta3/tortoise_types.go index 91d2810..ce164e1 100644 --- a/api/v1beta3/tortoise_types.go +++ b/api/v1beta3/tortoise_types.go @@ -256,8 +256,7 @@ const ( // TortoisePhaseBackToNormal means tortoise was in the emergency mode, and now it's coming back to the normal operation. // During TortoisePhaseBackToNormal, the number of replicas of workloads are gradually reduced to the usual value. // - Emergency → BackToNormal - TortoisePhaseBackToNormal TortoisePhase = "BackToNormal" - TortoisePhaseAutoEmergency TortoisePhase = "AutoEmergency" + TortoisePhaseBackToNormal TortoisePhase = "BackToNormal" ) type TargetsStatus struct { diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml index 510c2b8..9ec7057 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml @@ -18,23 +18,23 @@ spec: value: 100 selectPolicy: Max stabilizationWindowSeconds: 0 - maxReplicas: 20 + maxReplicas: 15 metrics: - containerResource: container: app name: cpu target: - averageUtilization: 55 + averageUtilization: 30 type: Utilization type: ContainerResource - containerResource: container: istio-proxy name: cpu target: - averageUtilization: 55 + averageUtilization: 30 type: Utilization type: ContainerResource - minReplicas: 20 + minReplicas: 15 scaleTargetRef: apiVersion: apps/v1 kind: Deployment diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml index f81d0ff..85478b7 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml @@ -55,20 +55,13 @@ status: containerResourceRequests: - containerName: app resource: - cpu: "10" + cpu: "6" memory: 3Gi - containerName: istio-proxy resource: cpu: "4" memory: 3Gi tortoiseConditions: - - lastTransitionTime: "2023-01-01T00:00:00Z" - lastUpdateTime: "2023-01-01T00:00:00Z" - message: the current number of replicas is not bigger than the preferred max - replica number - reason: ScaledUpBasedOnPreferredMaxReplicas - status: "False" - type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" message: HPA target utilization is updated @@ -107,25 +100,25 @@ status: - from: 0 timezone: Local to: 24 - updatedAt: "2023-01-01T00:00:00Z" - value: 20 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 minReplicas: - from: 0 timezone: Local to: 24 - updatedAt: "2023-01-01T00:00:00Z" - value: 5 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 targetUtilizations: - containerName: app targetUtilization: - cpu: 90 + cpu: 30 - containerName: istio-proxy targetUtilization: - cpu: 55 + cpu: 30 vertical: containerResourceRecommendation: - RecommendedResource: - cpu: "10" + cpu: "6" memory: 3Gi containerName: app - RecommendedResource: @@ -140,4 +133,4 @@ status: verticalPodAutoscalers: - name: tortoise-monitor-mercari role: Monitor - tortoisePhase: AutoEmergency + tortoisePhase: Emergency diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml index aa8c9b0..2757b8f 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml @@ -19,36 +19,50 @@ status: memory: Vertical containerName: istio-proxy conditions: + containerResourceRequests: + - containerName: "app" + resource: + cpu: "10" + memory: 10Gi + - containerName: "istio-proxy" + resource: + cpu: "4" + memory: 4Gi + tortoiseConditions: + - message: "HPA target utilization is updated" + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated containerRecommendationFromVPA: - containerName: app maxRecommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null recommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null - containerName: istio-proxy maxRecommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null recommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null recommendations: horizontal: @@ -56,23 +70,31 @@ status: - from: 0 timezone: Local to: 24 - updatedAt: "2023-10-06T01:15:46Z" + updatedAt: "2023-10-06T01:15:47Z" value: 15 minReplicas: - from: 0 timezone: Local to: 24 - updatedAt: "2023-10-06T01:15:46Z" + updatedAt: "2023-10-06T01:15:47Z" value: 3 targetUtilizations: - containerName: app targetUtilization: - cpu: 50 + cpu: 30 - containerName: istio-proxy targetUtilization: - cpu: 50 + cpu: 30 vertical: - containerResourceRecommendation: null + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy targets: horizontalPodAutoscaler: tortoise-hpa-mercari verticalPodAutoscalers: diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml index c0b903b..45fa2a6 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml @@ -18,23 +18,23 @@ spec: value: 100 selectPolicy: Max stabilizationWindowSeconds: 0 - maxReplicas: 20 + maxReplicas: 15 metrics: - containerResource: container: app name: cpu target: - averageUtilization: 55 + averageUtilization: 30 type: Utilization type: ContainerResource - containerResource: container: istio-proxy name: cpu target: - averageUtilization: 55 + averageUtilization: 30 type: Utilization type: ContainerResource - minReplicas: 20 + minReplicas: 15 scaleTargetRef: apiVersion: apps/v1 kind: Deployment diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml index f81d0ff..85478b7 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml @@ -55,20 +55,13 @@ status: containerResourceRequests: - containerName: app resource: - cpu: "10" + cpu: "6" memory: 3Gi - containerName: istio-proxy resource: cpu: "4" memory: 3Gi tortoiseConditions: - - lastTransitionTime: "2023-01-01T00:00:00Z" - lastUpdateTime: "2023-01-01T00:00:00Z" - message: the current number of replicas is not bigger than the preferred max - replica number - reason: ScaledUpBasedOnPreferredMaxReplicas - status: "False" - type: ScaledUpBasedOnPreferredMaxReplicas - lastTransitionTime: "2023-01-01T00:00:00Z" lastUpdateTime: "2023-01-01T00:00:00Z" message: HPA target utilization is updated @@ -107,25 +100,25 @@ status: - from: 0 timezone: Local to: 24 - updatedAt: "2023-01-01T00:00:00Z" - value: 20 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 minReplicas: - from: 0 timezone: Local to: 24 - updatedAt: "2023-01-01T00:00:00Z" - value: 5 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 targetUtilizations: - containerName: app targetUtilization: - cpu: 90 + cpu: 30 - containerName: istio-proxy targetUtilization: - cpu: 55 + cpu: 30 vertical: containerResourceRecommendation: - RecommendedResource: - cpu: "10" + cpu: "6" memory: 3Gi containerName: app - RecommendedResource: @@ -140,4 +133,4 @@ status: verticalPodAutoscalers: - name: tortoise-monitor-mercari role: Monitor - tortoisePhase: AutoEmergency + tortoisePhase: Emergency diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml index d8aa7c4..2d42fb8 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml @@ -58,4 +58,3 @@ spec: apiVersion: apps/v1 kind: Deployment name: mercari-app - diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml index aa8c9b0..2757b8f 100644 --- a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml @@ -19,36 +19,50 @@ status: memory: Vertical containerName: istio-proxy conditions: + containerResourceRequests: + - containerName: "app" + resource: + cpu: "10" + memory: 10Gi + - containerName: "istio-proxy" + resource: + cpu: "4" + memory: 4Gi + tortoiseConditions: + - message: "HPA target utilization is updated" + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated containerRecommendationFromVPA: - containerName: app maxRecommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null recommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null - containerName: istio-proxy maxRecommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null recommendation: cpu: - quantity: "0" + quantity: "3" updatedAt: null memory: - quantity: "0" + quantity: 3Gi updatedAt: null recommendations: horizontal: @@ -56,23 +70,31 @@ status: - from: 0 timezone: Local to: 24 - updatedAt: "2023-10-06T01:15:46Z" + updatedAt: "2023-10-06T01:15:47Z" value: 15 minReplicas: - from: 0 timezone: Local to: 24 - updatedAt: "2023-10-06T01:15:46Z" + updatedAt: "2023-10-06T01:15:47Z" value: 3 targetUtilizations: - containerName: app targetUtilization: - cpu: 50 + cpu: 30 - containerName: istio-proxy targetUtilization: - cpu: 50 + cpu: 30 vertical: - containerResourceRecommendation: null + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy targets: horizontalPodAutoscaler: tortoise-hpa-mercari verticalPodAutoscalers: diff --git a/internal/controller/tortoise_controller.go b/internal/controller/tortoise_controller.go index 9dceda9..5d6a80a 100644 --- a/internal/controller/tortoise_controller.go +++ b/internal/controller/tortoise_controller.go @@ -241,10 +241,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ } scalingActive := r.HpaService.CheckHpaMetricStatus(ctx, hpa) if scalingActive == false && tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeAuto && tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseWorking { - tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseAutoEmergency - } - if scalingActive == true && tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeAuto && tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseAutoEmergency { - tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseBackToNormal + tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseEmergency } tortoise = r.TortoiseService.UpdateContainerRecommendationFromVPA(tortoise, monitorvpa, now) diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index ce4ab3e..06ee77f 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -428,9 +428,6 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet case autoscalingv1beta3.TortoisePhaseEmergency: // when emergency mode, we set the same value on minReplicas. minToActuallyApply = recommendMax - case autoscalingv1beta3.TortoisePhaseAutoEmergency: - // when emergency mode, we set the same value on minReplicas. - minToActuallyApply = recommendMax case autoscalingv1beta3.TortoisePhaseBackToNormal: // gradually reduce the minReplicas. currentMin := *hpa.Spec.MinReplicas