Merge branch 'main' of github.com:mercari/tortoise into scheduledscaling

api/autoscaling/v2/webhook_suite_test.go

@@ -164,7 +164,7 @@ var _ = BeforeSuite(func() {
 	eventRecorder := mgr.GetEventRecorderFor("tortoise-controller")
 	tortoiseService, err := tortoise.New(mgr.GetClient(), eventRecorder, config.RangeOfMinMaxReplicasRecommendationHours, config.TimeZone, config.TortoiseUpdateInterval, config.GatheringDataPeriodType)
 	Expect(err).NotTo(HaveOccurred())
-	hpaService, err := hpa.New(mgr.GetClient(), eventRecorder, config.ReplicaReductionFactor, config.MaximumTargetResourceUtilization, 100, time.Hour, 1000, 10000, "")
+	hpaService, err := hpa.New(mgr.GetClient(), eventRecorder, config.ReplicaReductionFactor, config.MaximumTargetResourceUtilization, 100, time.Hour, 1000, 10000, 3, "")
 	Expect(err).NotTo(HaveOccurred())
 
 	hpaWebhook := New(tortoiseService, hpaService)

api/v1beta3/tortoise_types.go

@@ -217,20 +217,34 @@ type TortoisePhase string
 const (
 	// TortoisePhaseInitializing means tortoise is just created and initializing some components (HPA and VPA),
 	// and wait for those components to be ready.
+	// Possible flow: (none) → Initializing
 	TortoisePhaseInitializing TortoisePhase = "Initializing"
-	// TortoisePhaseGatheringData means tortoise is now gathering data and cannot make the accurate recommendations.
+	// TortoisePhaseGatheringData means tortoise is now gathering data for MinReplicas/MaxReplicas
+	// and cannot make the accurate recommendations.
+	// Possible flow: Initializing → GatheringData
 	TortoisePhaseGatheringData TortoisePhase = "GatheringData"
 	// TortoisePhaseWorking means tortoise is making the recommendations,
 	// and applying the recommendation values.
+	// Possible flow:
+	//  - GatheringData → Working (when all the data is ready)
+	//  - PartlyWorking → Working (when all the data is ready)
+	//  - BackToNormal → Working (minReplica goes back to the normal number)
 	TortoisePhaseWorking TortoisePhase = "Working"
 	// TortoisePhasePartlyWorking means tortoise has maxReplicas and minReplicas recommendations ready,
 	// and applying the recommendation values.
 	// But, some of the resources are not scaled due to some reasons. (probably still gathering data)
+	// Possible flow:
+	//  - GatheringData → PartlyWorking (only some of resources are ready)
+	//  - Working → PartlyWorking (autoscaling policy is changed)
 	TortoisePhasePartlyWorking TortoisePhase = "PartlyWorking"
 	// TortoisePhaseEmergency means tortoise is in the emergency mode.
+	//
+	// Possible flow:
+	//  - Working → Emergency
 	TortoisePhaseEmergency TortoisePhase = "Emergency"
 	// TortoisePhaseBackToNormal means tortoise was in the emergency mode, and now it's coming back to the normal operation.
 	// During TortoisePhaseBackToNormal, the number of replicas of workloads are gradually reduced to the usual value.
+	//  - Emergency → BackToNormal
 	TortoisePhaseBackToNormal TortoisePhase = "BackToNormal"
 )
 

controllers/testdata/deletion-no-delete/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/deletion-policy-all/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/hpa.yaml

@@ -1,6 +1,4 @@
 metadata:
-  annotations:
-    tortoise.autoscaling.mercari.com/managed-by-tortoise: "true"
   name: tortoise-hpa-mercari
   namespace: default
 spec:
@@ -18,7 +16,7 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 10000
   metrics:
   - containerResource:
       container: app
@@ -27,7 +25,7 @@ spec:
         averageUtilization: 70
         type: Utilization
     type: ContainerResource
-  minReplicas: 5
+  minReplicas: 3
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment

controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml

@@ -68,12 +68,6 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: HPA target utilization is updated
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: The recommendation is provided

controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/mutable-autoscalingpolicy-remove-horizontal-2/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-istio-enabled-pod-working/before/deployment.yaml

@@ -24,5 +24,4 @@ spec:
           requests:
             cpu: "10"
             memory: 10Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/before/deployment.yaml

@@ -21,5 +21,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-multiple-containers-pod-working/before/deployment.yaml

@@ -25,5 +25,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-backtonormal/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-dryrun/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-emergency-started/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/deployment.yaml

@@ -8,6 +8,8 @@ spec:
   strategy: {}
   template:
     metadata:
+      annotations:
+        kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z"
       creationTimestamp: null
       labels:
         app: mercari

controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml

@@ -36,7 +36,7 @@ status:
     - containerName: app
       resource:
         cpu: "4"
-        memory: 4Gi
+        memory: 3Gi
     tortoiseConditions:
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
@@ -45,6 +45,11 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: The recommendation is provided
+      status: "True"
+      type: VerticalRecommendationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       status: "False"
@@ -90,4 +95,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: GatheringData
+  tortoisePhase: PartlyWorking

controllers/testdata/reconcile-for-the-single-container-pod-gathering-data/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml

@@ -18,7 +18,7 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 100
   metrics:
   - external:
       metric:
@@ -34,7 +34,7 @@ spec:
         averageUtilization: 50
         type: Utilization
     type: ContainerResource
-  minReplicas: 5
+  minReplicas: 1
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment

controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml

@@ -46,12 +46,6 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: HPA target utilization is updated
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: The recommendation is provided
@@ -102,4 +96,4 @@ status:
     verticalPodAutoscalers:
     - name: tortoise-monitor-mercari
       role: Monitor
-  tortoisePhase: Working
+  tortoisePhase: PartlyWorking

controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/vpa-Monitor.yaml

@@ -1,6 +1,4 @@
 metadata:
-  annotations:
-    tortoise.autoscaling.mercari.com/managed-by-tortoise: "true"
   name: tortoise-monitor-mercari
   namespace: default
 spec:

controllers/testdata/reconcile-for-the-single-container-pod-initializing/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml

@@ -18,7 +18,7 @@ spec:
         value: 100
       selectPolicy: Max
       stabilizationWindowSeconds: 0
-  maxReplicas: 20
+  maxReplicas: 100
   metrics:
   - containerResource:
       container: app
@@ -27,7 +27,7 @@ spec:
         averageUtilization: 50
         type: Utilization
     type: ContainerResource
-  minReplicas: 5
+  minReplicas: 1
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment

controllers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml

@@ -45,12 +45,6 @@ status:
       reason: ScaledUpBasedOnPreferredMaxReplicas
       status: "False"
       type: ScaledUpBasedOnPreferredMaxReplicas
-    - lastTransitionTime: "2023-01-01T00:00:00Z"
-      lastUpdateTime: "2023-01-01T00:00:00Z"
-      message: HPA target utilization is updated
-      reason: HPATargetUtilizationUpdated
-      status: "True"
-      type: HPATargetUtilizationUpdated
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: The recommendation is provided

controllers/testdata/reconcile-for-the-single-container-pod-partly-working/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/testdata/reconcile-for-the-single-container-pod-working/before/deployment.yaml

@@ -19,5 +19,4 @@ spec:
           requests:
             cpu: "4"
             memory: 4Gi
-status:
   replicas: 10

controllers/tortoise_controller.go

@@ -166,7 +166,13 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 		logger.Error(err, "failed to get deployment", "tortoise", req.NamespacedName)
 		return ctrl.Result{}, err
 	}
-	currentReplicaNum := dm.Status.Replicas
+	if dm.Spec.Replicas == nil {
+		logger.Error(nil, "the deployment doesn't have the number of replicas and tortoise cannot calculate the recommendation", "tortoise", req.NamespacedName, "deployment", klog.KObj(dm))
+		return ctrl.Result{}, nil
+
+	}
+
+	currentDesiredReplicaNum := *dm.Spec.Replicas // Use the desired replica number.
 
 	if tortoise.Spec.UpdateMode == autoscalingv1beta3.UpdateModeOff /* When Off, ContainerResourceRequests should be reset */ ||
 		tortoise.Status.Conditions.ContainerResourceRequests == nil /* The first reconciliation */ {
@@ -191,7 +197,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 	if tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseInitializing {
 		logger.Info("initializing tortoise", "tortoise", req.NamespacedName)
 		// need to initialize HPA and VPA.
-		if err := r.initializeVPAAndHPA(ctx, tortoise, currentReplicaNum, now); err != nil {
+		if err := r.initializeVPAAndHPA(ctx, tortoise, currentDesiredReplicaNum, now); err != nil {
 			return ctrl.Result{}, fmt.Errorf("initialize VPA and HPA: %w", err)
 		}
 
@@ -204,7 +210,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 		return ctrl.Result{}, fmt.Errorf("add finalizer: %w", err)
 	}
 
-	tortoise, err = r.HpaService.UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx, tortoise, hpa, currentReplicaNum, now)
+	tortoise, err = r.HpaService.UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx, tortoise, hpa, currentDesiredReplicaNum, now)
 	if err != nil {
 		logger.Error(err, "update HPA spec from Tortoise autoscaling policy", "tortoise", req.NamespacedName)
 		return ctrl.Result{}, err
@@ -243,7 +249,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 
 	tortoise = r.TortoiseService.UpdateContainerRecommendationFromVPA(tortoise, monitorvpa, now)
 
-	tortoise, err = r.RecommenderService.UpdateRecommendations(ctx, tortoise, hpa, currentReplicaNum, now)
+	tortoise, err = r.RecommenderService.UpdateRecommendations(ctx, tortoise, hpa, currentDesiredReplicaNum, now)
 	if err != nil {
 		logger.Error(err, "update recommendation in tortoise", "tortoise", req.NamespacedName)
 		return ctrl.Result{}, err
@@ -266,7 +272,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 		return ctrl.Result{}, err
 	}
 
-	tortoise, err = r.TortoiseService.UpdateResourceRequest(ctx, tortoise, currentReplicaNum, now)
+	tortoise, err = r.TortoiseService.UpdateResourceRequest(ctx, tortoise, currentDesiredReplicaNum, now)
 	if err != nil {
 		logger.Error(err, "update VPA based on the recommendation in tortoise", "tortoise", req.NamespacedName)
 		return ctrl.Result{}, err

controllers/tortoise_controller_test.go

@@ -245,7 +245,7 @@ func startController(ctx context.Context) func() {
 	Expect(err).ShouldNot(HaveOccurred())
 	cli, err := vpa.New(mgr.GetConfig(), recorder)
 	Expect(err).ShouldNot(HaveOccurred())
-	hpaS, err := hpa.New(mgr.GetClient(), recorder, 0.95, 90, 25, time.Hour, 1000, 10000, ".*-exclude-metric")
+	hpaS, err := hpa.New(mgr.GetClient(), recorder, 0.95, 90, 25, time.Hour, 1000, 10000, 3, ".*-exclude-metric")
 	Expect(err).ShouldNot(HaveOccurred())
 	reconciler := &TortoiseReconciler{
 		Scheme:             scheme,