modify ContainerResourcePhases when tortoise is initialized or the au…

…toscaling policy changed
mercari · Oct 11, 2023 · 6790c0d · 6790c0d
1 parent 3b0d352
commit 6790c0d
Show file tree

Hide file tree

Showing 33 changed files with 585 additions and 74 deletions.
diff --git a/api/v1beta1/tortoise_types.go b/api/v1beta1/tortoise_types.go
@@ -154,14 +154,15 @@ type ContainerResourcePhases struct {
 	// ContainerName is the name of target container.
 	ContainerName string `json:"containerName" protobuf:"bytes,1,name=containerName"`
 	// ResourcePhases is the phase of each resource of this container.
-	ResourceePhases map[v1.ResourceName]ContainerResourcePhase `json:"resourcePhases" protobuf:"bytes,2,name=resourcePhases"`
+	ResourcePhases map[v1.ResourceName]ContainerResourcePhase `json:"resourcePhases" protobuf:"bytes,2,name=resourcePhases"`
 }
 
 type ContainerResourcePhase string
 
 const (
 	ContainerResourcePhaseGatheringData ContainerResourcePhase = "GatheringData"
 	ContainerResourcePhaseWorking       ContainerResourcePhase = "Working"
+	ContainerResourcePhaseOff           ContainerResourcePhase = "Off"
 )
 
 type TortoisePhase string

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/controllers/testdata/deletion-no-delete/before/tortoise.yaml b/controllers/testdata/deletion-no-delete/before/tortoise.yaml
@@ -84,7 +84,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/deletion-policy-all/before/tortoise.yaml b/controllers/testdata/deletion-policy-all/before/tortoise.yaml
@@ -84,7 +84,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml
@@ -92,11 +92,11 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working
     - containerName: "istio-proxy"
       resourcePhases: 
         cpu: Working 
-        memory: Working
+        memory: GatheringData
diff --git a/controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/tortoise.yaml
@@ -83,7 +83,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml
@@ -90,9 +90,9 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
-        cpu: Working 
+        cpu: GatheringData
         memory: Working
     - containerName: "istio-proxy"
       resourcePhases: 

diff --git a/...rollers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/before/tortoise.yaml b/...rollers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/before/tortoise.yaml
@@ -88,7 +88,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/after/tortoise.yaml
@@ -89,7 +89,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/before/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/before/tortoise.yaml
@@ -83,7 +83,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/after/tortoise.yaml
@@ -87,7 +87,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/before/tortoise.yaml
@@ -79,7 +79,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/...llers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/after/tortoise.yaml b/...llers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/after/tortoise.yaml
@@ -88,7 +88,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/...lers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/before/tortoise.yaml b/...lers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/before/tortoise.yaml
@@ -80,7 +80,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-emergency/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-emergency/after/tortoise.yaml
@@ -92,7 +92,7 @@ status:
       role: Monitor
   tortoisePhase: Emergency
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/...rollers/testdata/reconcile-for-the-multiple-containers-pod-emergency/before/tortoise.yaml b/...rollers/testdata/reconcile-for-the-multiple-containers-pod-emergency/before/tortoise.yaml
@@ -84,7 +84,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml
@@ -90,7 +90,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/tortoise.yaml
@@ -83,7 +83,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml
@@ -91,7 +91,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-working/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-working/before/tortoise.yaml
@@ -83,7 +83,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-dryrun/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-dryrun/after/tortoise.yaml
@@ -66,7 +66,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-dryrun/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-dryrun/before/tortoise.yaml
@@ -62,7 +62,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-emergency/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-emergency/after/tortoise.yaml
@@ -66,7 +66,7 @@ status:
       role: Monitor
   tortoisePhase: Emergency
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-emergency/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-emergency/before/tortoise.yaml
@@ -62,7 +62,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-working/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-working/after/tortoise.yaml
@@ -65,7 +65,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-working/before/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-working/before/tortoise.yaml
@@ -61,7 +61,7 @@ status:
       role: Monitor
   tortoisePhase: Working
   containerResourcePhases:
-    - containerName: "nginx"
+    - containerName: "app"
       resourcePhases:
         cpu: Working 
         memory: Working

diff --git a/controllers/tortoise_controller.go b/controllers/tortoise_controller.go
@@ -134,32 +134,13 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 		return ctrl.Result{RequeueAfter: r.Interval}, nil
 	}
 
-	// TODO: If HPA is modified, we need to change the tortoise phase.
-	// How to scale during that time?
-	_, err = r.HpaService.UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx, tortoise)
+	tortoise, err = r.HpaService.UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx, tortoise, dm)
 	if err != nil {
-		if !apierrors.IsNotFound(err) {
-			logger.Error(err, "update HPA spec from Tortoise autoscaling policy", "tortoise", req.NamespacedName)
-			return ctrl.Result{}, err
-		}
-
-		logger.V(4).Info("HPA doesn't exist for this tortoise. We may need to recreate hpa", "tortoise", req.NamespacedName)
-
-		// If not found, it's one of:
-		// - the user don't specify Horizontal in any autoscalingPolicy.
-		//   - In that case, we don't need to create an initial HPA.
-		// - the user didn't specify Horizontal in any autoscalingPolicy previously,
-		//   but just updated tortoise to have Horizontal in some.
-		//   - In that case, we need to create an initial HPA.
-		//
-		// In both cases, we just need to call `InitializeHPA` and it handles both cases.
-		tortoise, err = r.HpaService.InitializeHPA(ctx, tortoise, dm)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
+		logger.Error(err, "update HPA spec from Tortoise autoscaling policy", "tortoise", req.NamespacedName)
+		return ctrl.Result{}, err
 	}
 
-	vpa, ready, err := r.VpaService.GetTortoiseMonitorVPA(ctx, tortoise)
+	monitorvpa, ready, err := r.VpaService.GetTortoiseMonitorVPA(ctx, tortoise)
 	if err != nil {
 		logger.Error(err, "failed to get tortoise VPA", "tortoise", req.NamespacedName)
 		return ctrl.Result{}, err
@@ -175,14 +156,17 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 		return ctrl.Result{RequeueAfter: r.Interval}, nil
 	}
 
+	// VPA is ready, we mark all Vertical scaling resources as Running.
+	tortoise = vpa.MakeAllVerticalContainerResourcePhaseWorking(tortoise)
+
 	logger.V(4).Info("VPA created by tortoise is ready, proceeding to generate the recommendation", "tortoise", req.NamespacedName)
 	hpa, err := r.HpaService.GetHPAOnTortoise(ctx, tortoise)
 	if err != nil {
 		logger.Error(err, "failed to get HPA", "tortoise", req.NamespacedName)
 		return ctrl.Result{}, err
 	}
 
-	tortoise = r.TortoiseService.UpdateUpperRecommendation(tortoise, vpa)
+	tortoise = r.TortoiseService.UpdateUpperRecommendation(tortoise, monitorvpa)
 
 	tortoise, err = r.RecommenderService.UpdateRecommendations(ctx, tortoise, hpa, dm, now)
 	if err != nil {