Skip to content

Commit

Permalink
modify ContainerResourcePhases when tortoise is initialized or the au…
Browse files Browse the repository at this point in the history
…toscaling policy changed (#165)

* modify ContainerResourcePhases when tortoise is initialized or the autoscaling policy changed

* fix by lint
  • Loading branch information
sanposhiho authored Oct 11, 2023
1 parent 3b0d352 commit a15be18
Show file tree
Hide file tree
Showing 33 changed files with 586 additions and 74 deletions.
3 changes: 2 additions & 1 deletion api/v1beta1/tortoise_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,15 @@ type ContainerResourcePhases struct {
// ContainerName is the name of target container.
ContainerName string `json:"containerName" protobuf:"bytes,1,name=containerName"`
// ResourcePhases is the phase of each resource of this container.
ResourceePhases map[v1.ResourceName]ContainerResourcePhase `json:"resourcePhases" protobuf:"bytes,2,name=resourcePhases"`
ResourcePhases map[v1.ResourceName]ContainerResourcePhase `json:"resourcePhases" protobuf:"bytes,2,name=resourcePhases"`
}

type ContainerResourcePhase string

const (
ContainerResourcePhaseGatheringData ContainerResourcePhase = "GatheringData"
ContainerResourcePhaseWorking ContainerResourcePhase = "Working"
ContainerResourcePhaseOff ContainerResourcePhase = "Off"
)

type TortoisePhase string
Expand Down
4 changes: 2 additions & 2 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,11 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
- containerName: "istio-proxy"
resourcePhases:
cpu: Working
memory: Working
memory: GatheringData
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
cpu: GatheringData
memory: Working
- containerName: "istio-proxy"
resourcePhases:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ status:
role: Monitor
tortoisePhase: Emergency
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ status:
role: Monitor
tortoisePhase: Emergency
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ status:
role: Monitor
tortoisePhase: Working
containerResourcePhases:
- containerName: "nginx"
- containerName: "app"
resourcePhases:
cpu: Working
memory: Working
Expand Down
32 changes: 8 additions & 24 deletions controllers/tortoise_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,32 +134,13 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
return ctrl.Result{RequeueAfter: r.Interval}, nil
}

// TODO: If HPA is modified, we need to change the tortoise phase.
// How to scale during that time?
_, err = r.HpaService.UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx, tortoise)
tortoise, err = r.HpaService.UpdateHPASpecFromTortoiseAutoscalingPolicy(ctx, tortoise, dm)
if err != nil {
if !apierrors.IsNotFound(err) {
logger.Error(err, "update HPA spec from Tortoise autoscaling policy", "tortoise", req.NamespacedName)
return ctrl.Result{}, err
}

logger.V(4).Info("HPA doesn't exist for this tortoise. We may need to recreate hpa", "tortoise", req.NamespacedName)

// If not found, it's one of:
// - the user don't specify Horizontal in any autoscalingPolicy.
// - In that case, we don't need to create an initial HPA.
// - the user didn't specify Horizontal in any autoscalingPolicy previously,
// but just updated tortoise to have Horizontal in some.
// - In that case, we need to create an initial HPA.
//
// In both cases, we just need to call `InitializeHPA` and it handles both cases.
tortoise, err = r.HpaService.InitializeHPA(ctx, tortoise, dm)
if err != nil {
return ctrl.Result{}, err
}
logger.Error(err, "update HPA spec from Tortoise autoscaling policy", "tortoise", req.NamespacedName)
return ctrl.Result{}, err
}

vpa, ready, err := r.VpaService.GetTortoiseMonitorVPA(ctx, tortoise)
monitorvpa, ready, err := r.VpaService.GetTortoiseMonitorVPA(ctx, tortoise)
if err != nil {
logger.Error(err, "failed to get tortoise VPA", "tortoise", req.NamespacedName)
return ctrl.Result{}, err
Expand All @@ -175,14 +156,17 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
return ctrl.Result{RequeueAfter: r.Interval}, nil
}

// VPA is ready, we mark all Vertical scaling resources as Running.
tortoise = vpa.MakeAllVerticalContainerResourcePhaseWorking(tortoise)

logger.V(4).Info("VPA created by tortoise is ready, proceeding to generate the recommendation", "tortoise", req.NamespacedName)
hpa, err := r.HpaService.GetHPAOnTortoise(ctx, tortoise)
if err != nil {
logger.Error(err, "failed to get HPA", "tortoise", req.NamespacedName)
return ctrl.Result{}, err
}

tortoise = r.TortoiseService.UpdateUpperRecommendation(tortoise, vpa)
tortoise = r.TortoiseService.UpdateUpperRecommendation(tortoise, monitorvpa)

tortoise, err = r.RecommenderService.UpdateRecommendations(ctx, tortoise, hpa, dm, now)
if err != nil {
Expand Down
Loading

0 comments on commit a15be18

Please sign in to comment.