From 26ffaf70bcd80c9edd8f70ddf4dec7fcee4c9335 Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski <zreigz@gmail.com> Date: Mon, 20 Nov 2023 19:02:46 +0100 Subject: [PATCH] pre-install hooks --- pkg/hook/delete_policy.go | 7 +- pkg/hook/types.go | 10 +- pkg/manifests/template/template.go | 7 +- pkg/sync/hook.go | 147 ++++++++++++++++++++++++----- pkg/sync/loop.go | 42 +-------- pkg/sync/status.go | 20 +++- pkg/sync/utils.go | 81 ++++++++++++++++ 7 files changed, 243 insertions(+), 71 deletions(-) create mode 100644 pkg/sync/utils.go diff --git a/pkg/hook/delete_policy.go b/pkg/hook/delete_policy.go index fd3e902e..d9962542 100644 --- a/pkg/hook/delete_policy.go +++ b/pkg/hook/delete_policy.go @@ -2,6 +2,7 @@ package hook import ( resourceutil "github.com/pluralsh/deployment-operator/pkg/sync/resource" + "github.com/pluralsh/polly/containers" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -30,5 +31,9 @@ func DeletePolicies(obj *unstructured.Unstructured) []DeletePolicy { policies = append(policies, p) } } - return policies + if policies == nil { + policies = append(policies, BeforeHookCreation) + } + newSet := containers.ToSet[DeletePolicy](policies) + return newSet.List() } diff --git a/pkg/hook/types.go b/pkg/hook/types.go index 9aa2d5ce..5247899f 100644 --- a/pkg/hook/types.go +++ b/pkg/hook/types.go @@ -1,6 +1,7 @@ package hook import ( + "github.com/pluralsh/polly/containers" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -29,8 +30,9 @@ func NewHookType(t string) (Type, bool) { } type Hook struct { - Weight int - Types []Type - Kind schema.ObjectKind - Object *unstructured.Unstructured + Weight int + Types containers.Set[Type] + DeletePolicies containers.Set[DeletePolicy] + Kind schema.ObjectKind + Object *unstructured.Unstructured } diff --git a/pkg/manifests/template/template.go b/pkg/manifests/template/template.go index 970b06c9..b9269824 100644 --- a/pkg/manifests/template/template.go +++ b/pkg/manifests/template/template.go @@ -1,10 +1,11 @@ package template import ( - "github.com/pluralsh/deployment-operator/pkg/hook" "os" "path/filepath" + "github.com/pluralsh/deployment-operator/pkg/hook" + console "github.com/pluralsh/console-client-go" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/kubectl/pkg/cmd/util" @@ -16,14 +17,14 @@ const ( RendererHelm Renderer = "helm" RendererRaw Renderer = "raw" RenderKustomize Renderer = "kustomize" -ChartFileName = "Chart.yaml" + ChartFileName = "Chart.yaml" ) type Template interface { Render(svc *console.ServiceDeploymentExtended, utilFactory util.Factory) ([]*unstructured.Unstructured, error) } -func Render(dir string, svc *console.ServiceDeploymentExtended, utilFactory util.Factory) ([]*unstructured.Unstructured, error) { +func Render(dir string, svc *console.ServiceDeploymentExtended, utilFactory util.Factory) ([]*unstructured.Unstructured, []*unstructured.Unstructured, error) { renderer := RendererRaw _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { diff --git a/pkg/sync/hook.go b/pkg/sync/hook.go index 9ca71c86..281f85dc 100644 --- a/pkg/sync/hook.go +++ b/pkg/sync/hook.go @@ -1,30 +1,135 @@ package sync import ( - "sort" + "context" + "fmt" "github.com/pluralsh/deployment-operator/pkg/hook" + "github.com/pluralsh/polly/containers" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/cli-utils/pkg/inventory" ) -func GetHooks(obj []*unstructured.Unstructured) []hook.Hook { - hooks := make([]hook.Hook, 0) - for _, h := range obj { - hooks = append(hooks, hook.Hook{ - Weight: hook.Weight(h), - Types: hook.Types(h), - Kind: h.GetObjectKind(), - Object: h, - }) - } - sort.Slice(hooks, func(i, j int) bool { - kindI := hooks[i].Kind.GroupVersionKind() - kindJ := hooks[j].Kind.GroupVersionKind() - return kindI.Kind < kindJ.Kind - }) - sort.Slice(hooks, func(i, j int) bool { - return hooks[i].Weight < hooks[j].Weight - }) - - return hooks +func (engine *Engine) managePreInstallHooks(ctx context.Context, namespace, name, id string, hookObjects []*unstructured.Unstructured, objects []*unstructured.Unstructured) error { + installed, err := engine.isInstalled(id, objects) + if err != nil { + return err + } + if installed { + return nil + } + preInstallHooks := make([]hook.Hook, 0) + hooks := GetHooks(hookObjects) + for _, h := range hooks { + if h.Types.Has(hook.PreInstall) { + preInstallHooks = append(preInstallHooks, h) + } + } + + return engine.preInstallHooks(ctx, namespace, name, id, preInstallHooks) +} + +func (engine *Engine) preInstallHooks(ctx context.Context, namespace, name, id string, hooks []hook.Hook) error { + inv := inventory.WrapInventoryInfoObj(hookInventoryObjTemplate(id, hook.PreInstall)) + var manifests []*unstructured.Unstructured + hookSet := containers.Set[*unstructured.Unstructured]{} + deleteBefore := containers.Set[*unstructured.Unstructured]{} + deleteFailed := containers.Set[*unstructured.Unstructured]{} + deleteSucceeded := containers.Set[*unstructured.Unstructured]{} + for _, h := range hooks { + if h.DeletePolicies.Has(hook.BeforeHookCreation) { + deleteBefore.Add(h.Object) + } else if h.DeletePolicies.Has(hook.HookFailed) { + deleteFailed.Add(h.Object) + } else if h.DeletePolicies.Has(hook.HookSucceeded) { + deleteSucceeded.Add(h.Object) + } + hookSet.Add(h.Object) + } + dynamicClient, err := engine.utilFactory.DynamicClient() + if err != nil { + return err + } + client, err := engine.utilFactory.KubernetesClientSet() + if err != nil { + return err + } + invMap, err := client.CoreV1().ConfigMaps(inventoryFileNamespace).Get(ctx, getInvHookName(id, hook.PreInstall), metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } + if invMap != nil { + // delete previous resources + for _, r := range deleteBefore.List() { + gvk := r.GroupVersionKind() + gvr := schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + Resource: gvk.Kind, + } + if err := dynamicClient.Resource(gvr).Namespace(r.GetNamespace()).Delete(ctx, r.GetName(), metav1.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } + } + } + + manifests = hookSet.List() + if len(manifests) == 0 { + return nil + } + + ch := engine.applier.Run(ctx, inv, manifests, GetDefaultApplierOptions()) + statsCollector, statusCollector, err := GetStatusCollector(ch, false) + if err != nil { + return err + } + + if err := FormatSummary(namespace, name, *statsCollector); err != nil { + return err + } + + for _, v := range statusCollector.latestStatus { + if v.Resource == nil { + continue + } + + } + + return nil +} + +func (engine *Engine) isInstalled(id string, objects []*unstructured.Unstructured) (bool, error) { + inventoryName := GetInventoryName(id) + client, err := engine.utilFactory.KubernetesClientSet() + if err != nil { + return false, err + } + invConfigMap, err := client.CoreV1().ConfigMaps(inventoryFileNamespace).Get(context.Background(), inventoryName, metav1.GetOptions{}) + if err != nil { + return false, err + } + if apierrors.IsNotFound(err) { + return false, nil + } + if len(invConfigMap.Data) == len(objects) { + return true, nil + } + + return false, nil +} + +func hookInventoryObjTemplate(id string, t hook.Type) *unstructured.Unstructured { + name := getInvHookName(id, t) + return GenDefaultInventoryUnstructuredMap(inventoryFileNamespace, name, name) +} + +func getInvHookName(id string, t hook.Type) string { + return fmt.Sprintf("%s-hook-%s", t, id) } diff --git a/pkg/sync/loop.go b/pkg/sync/loop.go index e2870b7b..8772afb4 100644 --- a/pkg/sync/loop.go +++ b/pkg/sync/loop.go @@ -18,6 +18,8 @@ const ( // The field manager name for the ones agentk owns, see // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management fieldManager = "application/apply-patch" + + inventoryFileNamespace = "plrl-deploy-operator" ) func (engine *Engine) ControlLoop() { @@ -122,27 +124,8 @@ func (engine *Engine) processItem(item interface{}) error { log.Error(err, "failed to check namespace") return err } - ch := engine.applier.Run(ctx, inv, manifests, apply.ApplierOptions{ - ServerSideOptions: common.ServerSideOptions{ - // It's supported since Kubernetes 1.16, so there should be no reason not to use it. - // https://kubernetes.io/docs/reference/using-api/server-side-apply/ - ServerSideApply: true, - // GitOps repository is the source of truth and that's what we are applying, so overwrite any conflicts. - // https://kubernetes.io/docs/reference/using-api/server-side-apply/#conflicts - ForceConflicts: true, - // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management - FieldManager: fieldManager, - }, - ReconcileTimeout: 10 * time.Second, - // If we are not waiting for status, tell the applier to not - // emit the events. - EmitStatusEvents: true, - NoPrune: false, - DryRunStrategy: common.DryRunNone, - PrunePropagationPolicy: metav1.DeletePropagationBackground, - PruneTimeout: 20 * time.Second, - InventoryPolicy: inventory.PolicyAdoptAll, - }) + + ch := engine.applier.Run(ctx, inv, manifests, GetDefaultApplierOptions()) return engine.UpdateApplyStatus(id, svc.Name, svc.Namespace, ch, false, vcache) } @@ -172,20 +155,5 @@ func (engine *Engine) splitObjects(id string, objs []*unstructured.Unstructured) } func (engine *Engine) defaultInventoryObjTemplate(id string) (*unstructured.Unstructured, error) { - name := "inventory-" + id - namespace := "plrl-deploy-operator" - - return &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": map[string]interface{}{ - "name": name, - "namespace": namespace, - "labels": map[string]interface{}{ - common.InventoryLabel: id, - }, - }, - }, - }, nil + return GenDefaultInventoryUnstructuredMap(inventoryFileNamespace, GetInventoryName(id), id), nil } diff --git a/pkg/sync/status.go b/pkg/sync/status.go index b7be26f9..a99792f3 100644 --- a/pkg/sync/status.go +++ b/pkg/sync/status.go @@ -208,10 +208,9 @@ func FormatActionGroupEvent(age event.ActionGroupEvent) error { return nil } -func (engine *Engine) UpdateApplyStatus(id, name, namespace string, ch <-chan event.Event, printStatus bool, vcache map[manifests.GroupName]string) error { +func GetStatusCollector(ch <-chan event.Event, printStatus bool) (*stats.Stats, *StatusCollector, error) { var statsCollector stats.Stats var err error - components := []*console.ComponentAttributes{} statusCollector := &StatusCollector{ latestStatus: make(map[object.ObjMetadata]event.StatusEvent), } @@ -223,10 +222,10 @@ func (engine *Engine) UpdateApplyStatus(id, name, namespace string, ch <-chan ev if err := FormatActionGroupEvent( e.ActionGroupEvent, ); err != nil { - return err + return nil, nil, err } case event.ErrorType: - return e.ErrorEvent.Err + return nil, nil, e.ErrorEvent.Err case event.ApplyType: gk := e.ApplyEvent.Identifier.GroupKind name := e.ApplyEvent.Identifier.Name @@ -260,7 +259,18 @@ func (engine *Engine) UpdateApplyStatus(id, name, namespace string, ch <-chan ev } } - if err := FormatSummary(namespace, name, statsCollector); err != nil { + return &statsCollector, statusCollector, nil +} + +func (engine *Engine) UpdateApplyStatus(id, name, namespace string, ch <-chan event.Event, printStatus bool, vcache map[manifests.GroupName]string) error { + components := []*console.ComponentAttributes{} + + statsCollector, statusCollector, err := GetStatusCollector(ch, printStatus) + if err != nil { + return err + } + + if err := FormatSummary(namespace, name, *statsCollector); err != nil { return err } diff --git a/pkg/sync/utils.go b/pkg/sync/utils.go new file mode 100644 index 00000000..bb9a446d --- /dev/null +++ b/pkg/sync/utils.go @@ -0,0 +1,81 @@ +package sync + +import ( + "sort" + "time" + + "github.com/pluralsh/deployment-operator/pkg/hook" + "github.com/pluralsh/polly/containers" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/cli-utils/pkg/apply" + "sigs.k8s.io/cli-utils/pkg/common" + "sigs.k8s.io/cli-utils/pkg/inventory" +) + +func GenDefaultInventoryUnstructuredMap(namespace, name, id string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + "labels": map[string]interface{}{ + common.InventoryLabel: id, + }, + }, + }, + } +} + +func GetInventoryName(id string) string { + return "inventory-" + id +} + +func GetDefaultApplierOptions() apply.ApplierOptions { + return apply.ApplierOptions{ + ServerSideOptions: common.ServerSideOptions{ + // It's supported since Kubernetes 1.16, so there should be no reason not to use it. + // https://kubernetes.io/docs/reference/using-api/server-side-apply/ + ServerSideApply: true, + // GitOps repository is the source of truth and that's what we are applying, so overwrite any conflicts. + // https://kubernetes.io/docs/reference/using-api/server-side-apply/#conflicts + ForceConflicts: true, + // https://kubernetes.io/docs/reference/using-api/server-side-apply/#field-management + FieldManager: fieldManager, + }, + ReconcileTimeout: 10 * time.Second, + // If we are not waiting for status, tell the applier to not + // emit the events. + EmitStatusEvents: true, + NoPrune: false, + DryRunStrategy: common.DryRunNone, + PrunePropagationPolicy: metav1.DeletePropagationBackground, + PruneTimeout: 20 * time.Second, + InventoryPolicy: inventory.PolicyAdoptAll, + } +} + +func GetHooks(obj []*unstructured.Unstructured) []hook.Hook { + hooks := make([]hook.Hook, 0) + for _, h := range obj { + hooks = append(hooks, hook.Hook{ + Weight: hook.Weight(h), + Types: containers.ToSet(hook.Types(h)), + DeletePolicies: containers.ToSet(hook.DeletePolicies(h)), + Kind: h.GetObjectKind(), + Object: h, + }) + } + sort.Slice(hooks, func(i, j int) bool { + kindI := hooks[i].Kind.GroupVersionKind() + kindJ := hooks[j].Kind.GroupVersionKind() + return kindI.Kind < kindJ.Kind + }) + sort.Slice(hooks, func(i, j int) bool { + return hooks[i].Weight < hooks[j].Weight + }) + + return hooks +}