diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index 66b9a4b5136..3cf8bd34616 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -82,6 +82,18 @@ spec: {{- range .Values.flags.excludeLabelPropagationPrefixes }} - "--exclude-label-propagation-prefix={{ . }}" {{- end }} + {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingRunnerSet }} + - "--max-concurrent-reconciles-for-autoscaling-runner-set={{ . }}" + {{- end }} + {{- with .Values.flags.maxConcurrentReconcilesForEphemeralRunnerSet }} + - "--max-concurrent-reconciles-for-ephemeral-runner-set={{ . }}" + {{- end }} + {{- with .Values.flags.maxConcurrentReconcilesForEphemeralRunner }} + - "--max-concurrent-reconciles-for-ephemeral-runner={{ . }}" + {{- end }} + {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingListener }} + - "--max-concurrent-reconciles-for-autoscaling-listener={{ . }}" + {{- end }} command: - "/manager" {{- with .Values.metrics }} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index 8e74317e456..ce0d616d590 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -130,3 +130,9 @@ flags: ## Labels that match prefix specified in the list are excluded from propagation. # excludeLabelPropagationPrefixes: # - "argocd.argoproj.io/instance" + + ## Defines the maximum number of concurrent reconciles for each reconciler. + # maxConcurrentReconcilesForAutoscalingRunnerSet: 1 + # maxConcurrentReconcilesForEphemeralRunnerSet: 1 + # maxConcurrentReconcilesForEphemeralRunner: 1 + # maxConcurrentReconcilesForAutoscalingListener: 1 diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index f2de2216ac1..bcffd9228e7 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -55,6 +56,8 @@ type AutoscalingListenerReconciler struct { ListenerMetricsAddr string ListenerMetricsEndpoint string + MaxConcurrentReconciles int + ResourceBuilder } @@ -730,6 +733,7 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). + WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). Complete(r) } diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index 6746df3db33..84bd059d8ad 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -79,6 +80,7 @@ type AutoscalingRunnerSetReconciler struct { DefaultRunnerScaleSetListenerImagePullSecrets []string UpdateStrategy UpdateStrategy ActionsClient actions.MultiClient + MaxConcurrentReconciles int ResourceBuilder } @@ -763,6 +765,7 @@ func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) erro }, )). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). + WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). Complete(r) } diff --git a/controllers/actions.github.com/ephemeralrunner_controller.go b/controllers/actions.github.com/ephemeralrunner_controller.go index 36ea1146ba5..015461aa9b0 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller.go +++ b/controllers/actions.github.com/ephemeralrunner_controller.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -49,9 +50,10 @@ const ( // EphemeralRunnerReconciler reconciles a EphemeralRunner object type EphemeralRunnerReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - ActionsClient actions.MultiClient + Log logr.Logger + Scheme *runtime.Scheme + ActionsClient actions.MultiClient + MaxConcurrentReconciles int ResourceBuilder } @@ -828,6 +830,7 @@ func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&v1alpha1.EphemeralRunner{}). Owns(&corev1.Pod{}). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). + WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). Complete(r) } diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller.go b/controllers/actions.github.com/ephemeralrunnerset_controller.go index c1c2523ef73..8fdb9c2304a 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -53,6 +54,8 @@ type EphemeralRunnerSetReconciler struct { PublishMetrics bool + MaxConcurrentReconciles int + ResourceBuilder } @@ -575,6 +578,7 @@ func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error For(&v1alpha1.EphemeralRunnerSet{}). Owns(&v1alpha1.EphemeralRunner{}). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). + WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). Complete(r) } diff --git a/main.go b/main.go index d7edea6cb60..1a02548872f 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,11 @@ func main() { autoScalerImagePullSecrets stringSlice commonRunnerLabels commaSeparatedStringSlice + + maxConcurrentReconcilesForAutoscalingRunnerSet int + maxConcurrentReconcilesForEphemeralRunnerSet int + maxConcurrentReconcilesForEphemeralRunner int + maxConcurrentReconcilesForAutoscalingListener int ) var c github.Config err = envconfig.Process("github", &c) @@ -145,6 +150,10 @@ func main() { flag.BoolVar(&autoScalingRunnerSetOnly, "auto-scaling-runner-set-only", false, "Make controller only reconcile AutoRunnerScaleSet object.") flag.StringVar(&updateStrategy, "update-strategy", "immediate", `Resources reconciliation strategy on upgrade with running/pending jobs. Valid values are: "immediate", "eventual". Defaults to "immediate".`) flag.Var(&autoScalerImagePullSecrets, "auto-scaler-image-pull-secrets", "The default image-pull secret name for auto-scaler listener container.") + flag.IntVar(&maxConcurrentReconcilesForAutoscalingRunnerSet, "max-concurrent-reconciles-for-autoscaling-runner-set", 1, "The maximum number of concurrent reconciles for AutoscalingRunnerSet.") + flag.IntVar(&maxConcurrentReconcilesForEphemeralRunnerSet, "max-concurrent-reconciles-for-ephemeral-runner-set", 1, "The maximum number of concurrent reconciles for EphemeralRunnerSet.") + flag.IntVar(&maxConcurrentReconcilesForEphemeralRunner, "max-concurrent-reconciles-for-ephemeral-runner", 1, "The maximum number of concurrent reconciles for EphemeralRunner.") + flag.IntVar(&maxConcurrentReconcilesForAutoscalingListener, "max-concurrent-reconciles-for-autoscaling-listener", 1, "The maximum number of concurrent reconciles for AutoscalingListener.") flag.Parse() runnerPodDefaults.RunnerImagePullSecrets = runnerImagePullSecrets @@ -273,30 +282,33 @@ func main() { ActionsClient: actionsMultiClient, UpdateStrategy: actionsgithubcom.UpdateStrategy(updateStrategy), DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets, - ResourceBuilder: rb, + MaxConcurrentReconciles: maxConcurrentReconcilesForAutoscalingRunnerSet, + ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet") os.Exit(1) } if err = (&actionsgithubcom.EphemeralRunnerReconciler{ - Client: mgr.GetClient(), - Log: log.WithName("EphemeralRunner").WithValues("version", build.Version), - Scheme: mgr.GetScheme(), - ActionsClient: actionsMultiClient, - ResourceBuilder: rb, + Client: mgr.GetClient(), + Log: log.WithName("EphemeralRunner").WithValues("version", build.Version), + Scheme: mgr.GetScheme(), + ActionsClient: actionsMultiClient, + MaxConcurrentReconciles: maxConcurrentReconcilesForEphemeralRunner, + ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunner") os.Exit(1) } if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{ - Client: mgr.GetClient(), - Log: log.WithName("EphemeralRunnerSet").WithValues("version", build.Version), - Scheme: mgr.GetScheme(), - ActionsClient: actionsMultiClient, - PublishMetrics: metricsAddr != "0", - ResourceBuilder: rb, + Client: mgr.GetClient(), + Log: log.WithName("EphemeralRunnerSet").WithValues("version", build.Version), + Scheme: mgr.GetScheme(), + ActionsClient: actionsMultiClient, + PublishMetrics: metricsAddr != "0", + MaxConcurrentReconciles: maxConcurrentReconcilesForEphemeralRunnerSet, + ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet") os.Exit(1) @@ -308,6 +320,7 @@ func main() { Scheme: mgr.GetScheme(), ListenerMetricsAddr: listenerMetricsAddr, ListenerMetricsEndpoint: listenerMetricsEndpoint, + MaxConcurrentReconciles: maxConcurrentReconcilesForAutoscalingListener, ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "AutoscalingListener")