From fcd0e30450e18a551345aed611898b7e26f20729 Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Mon, 17 Jun 2024 20:45:31 +0200 Subject: [PATCH] feat: add argo rollouts resume support (#220) * add argo rollouts resume support * fix gofmt * fix linter * prevent second promotion * add rollback * apply review comments * fix health status * use status code * kick service reconciler on rollout promote --------- Co-authored-by: michaeljguarino --- cmd/agent/main.go | 45 ++++- go.mod | 11 +- go.sum | 22 +-- internal/controller/argorollout_controller.go | 160 ++++++++++++++++++ internal/utils/argorollout.go | 105 ++++++++++++ pkg/controller/service/health.go | 33 ++++ pkg/controller/service/reconciler_status.go | 11 +- pkg/controller/service/status_collector.go | 9 +- 8 files changed, 375 insertions(+), 21 deletions(-) create mode 100644 internal/controller/argorollout_controller.go create mode 100644 internal/utils/argorollout.go diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 84537d54..0cc37549 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,8 +1,13 @@ package main import ( + "net/http" "os" + "time" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" + rolloutv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + roclientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned" templatesv1 "github.com/open-policy-agent/frameworks/constraint/pkg/apis/templates/v1" constraintstatusv1beta1 "github.com/open-policy-agent/gatekeeper/v3/apis/status/v1beta1" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -10,6 +15,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -32,9 +39,15 @@ func init() { utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(constraintstatusv1beta1.AddToScheme(scheme)) utilruntime.Must(templatesv1.AddToScheme(scheme)) + utilruntime.Must(rolloutv1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } +const ( + httpClientTimout = time.Second * 5 + httpCacheExpiryTime = time.Second * 2 +) + func main() { opt := newOptions() config := ctrl.GetConfigOrDie() @@ -50,7 +63,21 @@ func main() { setupLog.Error(err, "unable to create manager") os.Exit(1) } - + rolloutsClient, err := roclientset.NewForConfig(config) + if err != nil { + setupLog.Error(err, "unable to create rollouts client") + os.Exit(1) + } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + setupLog.Error(err, "unable to create dynamic client") + os.Exit(1) + } + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + setupLog.Error(err, "unable to create kubernetes client") + os.Exit(1) + } setupLog.Info("starting agent") ctrlMgr, serviceReconciler, gateReconciler := runAgent(opt, config, ctx, mgr.GetClient()) @@ -70,6 +97,17 @@ func main() { ConsoleClient: ctrlMgr.GetClient(), Reader: mgr.GetCache(), } + argoRolloutController := &controller.ArgoRolloutReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ConsoleClient: ctrlMgr.GetClient(), + ConsoleURL: opt.consoleUrl, + HttpClient: &http.Client{Timeout: httpClientTimout}, + ArgoClientSet: rolloutsClient, + DynamicClient: dynamicClient, + SvcReconciler: serviceReconciler, + KubeClient: kubeClient, + } reconcileGroups := map[schema.GroupVersionKind]controller.SetupWithManager{ { @@ -87,6 +125,11 @@ func main() { Version: "v1beta1", Kind: "ConstraintPodStatus", }: constraintController.SetupWithManager, + { + Group: rolloutv1alpha1.SchemeGroupVersion.Group, + Version: rolloutv1alpha1.SchemeGroupVersion.Version, + Kind: rollouts.RolloutKind, + }: argoRolloutController.SetupWithManager, } if err = (&controller.CrdRegisterControllerReconciler{ diff --git a/go.mod b/go.mod index af76da4d..2d461195 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/Yamashou/gqlgenc v0.18.1 + github.com/argoproj/argo-rollouts v1.6.6 github.com/elastic/crd-ref-docs v0.0.12 github.com/evanphx/json-patch v5.7.0+incompatible github.com/fluxcd/flagger v1.35.0 @@ -19,7 +20,7 @@ require ( github.com/open-policy-agent/gatekeeper/v3 v3.15.1 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/pkg/errors v0.9.1 - github.com/pluralsh/console-client-go v0.5.18 + github.com/pluralsh/console-client-go v0.7.0 github.com/pluralsh/controller-reconcile-helper v0.0.4 github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 github.com/pluralsh/polly v0.1.10 @@ -87,7 +88,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect - github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -111,7 +112,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/btree v1.0.1 // indirect + github.com/google/btree v1.1.2 // indirect github.com/google/cel-go v0.17.7 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -122,7 +123,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect - github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -184,7 +185,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/sosodev/duration v1.2.0 // indirect github.com/spf13/afero v1.9.3 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cast v1.5.1 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.15.0 // indirect diff --git a/go.sum b/go.sum index 20483e8f..ee757816 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/argoproj/argo-rollouts v1.6.6 h1:JCJ0cGAwWkh2xCAHZ1OQmrobysRjCatmG9IZaLJpS1g= +github.com/argoproj/argo-rollouts v1.6.6/go.mod h1:X2kTiBaYCSounmw1kmONdIZTwJNzNQYC0SrXUgSw9UI= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -173,8 +175,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -289,8 +291,8 @@ github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.17.7 h1:6ebJFzu1xO2n7TLtN+UBqShGBhlD85bhvglh5DpcfqQ= github.com/google/cel-go v0.17.7/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -345,8 +347,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= @@ -526,8 +528,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pluralsh/console-client-go v0.5.18 h1:uwYsoGaggvi3uPZYL/+qdhvgl73sGBiuVUfQGAC/J4c= -github.com/pluralsh/console-client-go v0.5.18/go.mod h1:eyCiLA44YbXiYyJh8303jk5JdPkt9McgCo5kBjk4lKo= +github.com/pluralsh/console-client-go v0.7.0 h1:7BcvftmKhssYd8F06NGsWXKxs7O3K8gQDYrQebvbmHE= +github.com/pluralsh/console-client-go v0.7.0/go.mod h1:eyCiLA44YbXiYyJh8303jk5JdPkt9McgCo5kBjk4lKo= github.com/pluralsh/controller-reconcile-helper v0.0.4 h1:1o+7qYSyoeqKFjx+WgQTxDz4Q2VMpzprJIIKShxqG0E= github.com/pluralsh/controller-reconcile-helper v0.0.4/go.mod h1:AfY0gtteD6veBjmB6jiRx/aR4yevEf6K0M13/pGan/s= github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 h1:ab2PN+6if/Aq3/sJM0AVdy1SYuMAnq4g20VaKhTm/Bw= @@ -590,8 +592,8 @@ github.com/sosodev/duration v1.2.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERA github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= diff --git a/internal/controller/argorollout_controller.go b/internal/controller/argorollout_controller.go new file mode 100644 index 00000000..f286b051 --- /dev/null +++ b/internal/controller/argorollout_controller.go @@ -0,0 +1,160 @@ +package controller + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/typed/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd/abort" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" + rolloutv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + roclientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned" + console "github.com/pluralsh/console-client-go" + "github.com/pluralsh/deployment-operator/internal/utils" + "github.com/pluralsh/deployment-operator/pkg/client" + "github.com/pluralsh/deployment-operator/pkg/controller/service" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + inventoryAnnotationName = "config.k8s.io/owning-inventory" + closed = "closed" +) + +var requeueRollout = ctrl.Result{RequeueAfter: time.Second * 5} + +// ArgoRolloutReconciler reconciles a Argo Rollout custom resource. +type ArgoRolloutReconciler struct { + k8sClient.Client + Scheme *runtime.Scheme + ConsoleClient client.Client + ConsoleURL string + HttpClient *http.Client + ArgoClientSet roclientset.Interface + DynamicClient dynamic.Interface + KubeClient kubernetes.Interface + SvcReconciler *service.ServiceReconciler +} + +// Reconcile Argo Rollout custom resources to ensure that Console stays in sync with Kubernetes cluster. +func (r *ArgoRolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + // Read resource from Kubernetes cluster. + rollout := &rolloutv1alpha1.Rollout{} + if err := r.Get(ctx, req.NamespacedName, rollout); err != nil { + logger.Error(err, "unable to fetch rollout") + return ctrl.Result{}, k8sClient.IgnoreNotFound(err) + } + + if !rollout.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + serviceID, ok := rollout.Annotations[inventoryAnnotationName] + if !ok { + return ctrl.Result{}, nil + } + if serviceID == "" { + return ctrl.Result{}, fmt.Errorf("the service ID from the inventory annotation is empty") + } + service, err := r.ConsoleClient.GetService(serviceID) + if err != nil { + return ctrl.Result{}, err + } + consoleURL, err := sanitizeURL(r.ConsoleURL) + if err != nil { + return ctrl.Result{}, err + } + if rollout.Status.Phase == rolloutv1alpha1.RolloutPhasePaused { + // wait until the agent will change component status + if !hasPausedRolloutComponent(service) { + return requeueRollout, nil + } + + rolloutIf := r.ArgoClientSet.ArgoprojV1alpha1().Rollouts(rollout.Namespace) + promoteURL := fmt.Sprintf("%s/ext/v1/gate/%s", consoleURL, serviceID) + rollbackURL := fmt.Sprintf("%s/ext/v1/rollback/%s", consoleURL, serviceID) + + promoteResponse, err := r.get(promoteURL) + if err != nil { + return ctrl.Result{}, err + } + if promoteResponse == http.StatusOK { + return ctrl.Result{}, r.promote(ctx, rolloutIf, rollout, serviceID) + } + rollbackResponse, err := r.get(rollbackURL) + if err != nil { + return ctrl.Result{}, err + } + if rollbackResponse == http.StatusOK { + return ctrl.Result{}, r.rollback(rolloutIf, rollout) + } + return requeueRollout, nil + } + return ctrl.Result{}, nil +} + +func (r *ArgoRolloutReconciler) promote(ctx context.Context, rolloutIf clientset.RolloutInterface, rollout *rolloutv1alpha1.Rollout, svcId string) error { + if _, err := utils.PromoteRollout(ctx, rolloutIf, rollout.Name); err != nil { + return err + } + + if r.SvcReconciler != nil { + r.SvcReconciler.SvcQueue.AddRateLimited(svcId) + } + return nil +} + +func (r *ArgoRolloutReconciler) rollback(rolloutIf clientset.RolloutInterface, rollout *rolloutv1alpha1.Rollout) error { + if _, err := abort.AbortRollout(rolloutIf, rollout.Name); err != nil { + return err + } + return nil +} + +func hasPausedRolloutComponent(service *console.GetServiceDeploymentForAgent_ServiceDeployment) bool { + for _, component := range service.Components { + if component.Kind == rollouts.RolloutKind { + if component.State != nil && *component.State == console.ComponentStatePaused { + return true + } + } + } + return false +} + +func sanitizeURL(consoleURL string) (string, error) { + u, err := url.Parse(consoleURL) + if err != nil { + return "", err + } + return fmt.Sprintf("%s://%s", u.Scheme, u.Host), nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ArgoRolloutReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&rolloutv1alpha1.Rollout{}). + Complete(r) +} + +func (r *ArgoRolloutReconciler) get(url string) (int, error) { + // Make the HTTP request + resp, err := r.HttpClient.Get(url) + if err != nil { + return http.StatusInternalServerError, err + } + defer resp.Body.Close() + + return resp.StatusCode, nil +} diff --git a/internal/utils/argorollout.go b/internal/utils/argorollout.go new file mode 100644 index 00000000..8f7b3480 --- /dev/null +++ b/internal/utils/argorollout.go @@ -0,0 +1,105 @@ +package utils + +import ( + "context" + "fmt" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/typed/rollouts/v1alpha1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + unpausePatch = `{"spec":{"paused":false}}` + clearPauseConditionsPatch = `{"status":{"pauseConditions":null}}` + clearPauseConditionsAndControllerPausePatch = `{"status":{"pauseConditions":null, "controllerPause":false, "currentStepIndex":%d}}` + unpauseAndClearPauseConditionsPatch = `{"spec":{"paused":false},"status":{"pauseConditions":null}}` + clearPauseConditionsPatchWithStep = `{"status":{"pauseConditions":null, "currentStepIndex":%d}}` + unpauseAndClearPauseConditionsPatchWithStep = `{"spec":{"paused":false},"status":{"pauseConditions":null, "currentStepIndex":%d}}` +) + +// PromoteRollout promotes a rollout to the next step, or to end of all steps +func PromoteRollout(ctx context.Context, rolloutIf clientset.RolloutInterface, name string) (*v1alpha1.Rollout, error) { + ro, err := rolloutIf.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + specPatch, statusPatch, unifiedPatch := getPatches(ro) + if statusPatch != nil { + ro, err = rolloutIf.Patch(ctx, name, types.MergePatchType, statusPatch, metav1.PatchOptions{}, "status") + if err != nil { + // NOTE: in the future, we can simply return error here, if we wish to drop support for v0.9 + if !k8serrors.IsNotFound(err) { + return nil, err + } + // we got a NotFound error. status subresource is not being used, so perform unifiedPatch + specPatch = unifiedPatch + } + } + if specPatch != nil { + ro, err = rolloutIf.Patch(ctx, name, types.MergePatchType, specPatch, metav1.PatchOptions{}) + if err != nil { + return nil, err + } + } + return ro, nil +} + +func isInconclusive(rollout *v1alpha1.Rollout) bool { + return rollout.Spec.Strategy.Canary != nil && rollout.Status.Canary.CurrentStepAnalysisRunStatus != nil && rollout.Status.Canary.CurrentStepAnalysisRunStatus.Status == v1alpha1.AnalysisPhaseInconclusive +} + +func getPatches(rollout *v1alpha1.Rollout) ([]byte, []byte, []byte) { + var specPatch, statusPatch, unifiedPatch []byte + + unifiedPatch = []byte(unpauseAndClearPauseConditionsPatch) + if rollout.Spec.Paused { + specPatch = []byte(unpausePatch) + } + // in case if canary rollout in inconclusive state, we want to unset controller pause , clean pause conditions and increment step index + // so that rollout can proceed to next step + // without such patch, rollout will be stuck in inconclusive state in case if next step is pause step + switch { + case isInconclusive(rollout) && len(rollout.Status.PauseConditions) > 0 && rollout.Status.ControllerPause: + _, index := GetCurrentCanaryStep(rollout) + if index != nil { + if *index < int32(len(rollout.Spec.Strategy.Canary.Steps)) { + *index++ + } + statusPatch = []byte(fmt.Sprintf(clearPauseConditionsAndControllerPausePatch, *index)) + } + case len(rollout.Status.PauseConditions) > 0: + statusPatch = []byte(clearPauseConditionsPatch) + case rollout.Spec.Strategy.Canary != nil: + _, index := GetCurrentCanaryStep(rollout) + // At this point, the controller knows that the rollout is a canary with steps and GetCurrentCanaryStep returns 0 if + // the index is not set in the rollout + if index != nil { + if *index < int32(len(rollout.Spec.Strategy.Canary.Steps)) { + *index++ + } + statusPatch = []byte(fmt.Sprintf(clearPauseConditionsPatchWithStep, *index)) + unifiedPatch = []byte(fmt.Sprintf(unpauseAndClearPauseConditionsPatchWithStep, *index)) + } + } + return specPatch, statusPatch, unifiedPatch +} + +// GetCurrentCanaryStep returns the current canary step. If there are no steps or the rollout +// has already executed the last step, the func returns nil +func GetCurrentCanaryStep(rollout *v1alpha1.Rollout) (*v1alpha1.CanaryStep, *int32) { + if rollout.Spec.Strategy.Canary == nil || len(rollout.Spec.Strategy.Canary.Steps) == 0 { + return nil, nil + } + currentStepIndex := int32(0) + if rollout.Status.CurrentStepIndex != nil { + currentStepIndex = *rollout.Status.CurrentStepIndex + } + if len(rollout.Spec.Strategy.Canary.Steps) <= int(currentStepIndex) { + return nil, ¤tStepIndex + } + return &rollout.Spec.Strategy.Canary.Steps[currentStepIndex], ¤tStepIndex +} diff --git a/pkg/controller/service/health.go b/pkg/controller/service/health.go index 61a9048c..247072ea 100644 --- a/pkg/controller/service/health.go +++ b/pkg/controller/service/health.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + rolloutv1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" flaggerv1beta1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" "github.com/pluralsh/deployment-operator/pkg/lua" appsv1 "k8s.io/api/apps/v1" @@ -98,6 +99,38 @@ func getHPAHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { } } +func getArgoRolloutHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { + var argo rolloutv1alpha1.Rollout + var msg string + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &argo); err != nil { + return nil, err + } + switch argo.Status.Phase { + case rolloutv1alpha1.RolloutPhasePaused: + return &HealthStatus{ + Status: HealthStatusPaused, + Message: argo.Status.Message, + }, nil + + case rolloutv1alpha1.RolloutPhaseDegraded: + return &HealthStatus{ + Status: HealthStatusDegraded, + Message: argo.Status.Message, + }, nil + + case rolloutv1alpha1.RolloutPhaseHealthy: + return &HealthStatus{ + Status: HealthStatusHealthy, + Message: argo.Status.Message, + }, nil + default: + return &HealthStatus{ + Status: HealthStatusProgressing, + Message: msg, + }, nil + } +} + func getCanaryHealth(obj *unstructured.Unstructured) (*HealthStatus, error) { var canary flaggerv1beta1.Canary if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &canary); err != nil { diff --git a/pkg/controller/service/reconciler_status.go b/pkg/controller/service/reconciler_status.go index dff51bae..c04bc6bb 100644 --- a/pkg/controller/service/reconciler_status.go +++ b/pkg/controller/service/reconciler_status.go @@ -5,15 +5,14 @@ import ( "fmt" "strings" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" console "github.com/pluralsh/console-client-go" + "github.com/pluralsh/deployment-operator/pkg/manifests" "github.com/samber/lo" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/cli-utils/pkg/print/stats" - - "github.com/pluralsh/deployment-operator/pkg/manifests" - "sigs.k8s.io/cli-utils/pkg/apply/event" + "sigs.k8s.io/cli-utils/pkg/print/stats" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -98,6 +97,10 @@ func (s *ServiceReconciler) GetHealthCheckFunc(gvk schema.GroupVersionKind) func if gvk.Kind == CanaryKind { return getCanaryHealth } + case rollouts.Group: + if gvk.Kind == rollouts.RolloutKind { + return getArgoRolloutHealth + } case "autoscaling": if gvk.Kind == HorizontalPodAutoscalerKind { return getHPAHealth diff --git a/pkg/controller/service/status_collector.go b/pkg/controller/service/status_collector.go index 19ba2a4a..1d217e4b 100644 --- a/pkg/controller/service/status_collector.go +++ b/pkg/controller/service/status_collector.go @@ -108,13 +108,20 @@ func (sc *serviceComponentsStatusCollector) fromSyncResult(e event.StatusEvent, version = v } + synced := e.PollResourceInfo.Status == status.CurrentStatus + + if e.PollResourceInfo.Status == status.UnknownStatus { + if sc.reconciler.toStatus(e.Resource) != nil { + synced = *sc.reconciler.toStatus(e.Resource) == console.ComponentStateRunning + } + } return &console.ComponentAttributes{ Group: gvk.Group, Kind: gvk.Kind, Namespace: e.Resource.GetNamespace(), Name: e.Resource.GetName(), Version: version, - Synced: e.PollResourceInfo.Status == status.CurrentStatus, + Synced: synced, State: sc.reconciler.toStatus(e.Resource), } }