From ff7bbd6fa6bedd551797a2ed443e32e548d080f8 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Tue, 22 Aug 2023 17:17:00 -0400 Subject: [PATCH 1/2] Add OpenStackDataplaneDeployment CRD Adds a new OpenStackDataPlaneDeployment CRD, which will trigger the ansible execution(s) for the given set of OpenStackDataPlaneNodeSets. An OpenStackDataplaneDeployment can be thought of like a Kubernets Job. Once it's successful, it's considered complete. The ansible executions won't be retriggered by further reconciliations. As part of this change, the following are implemented: - The deployment logic and handling is removed from the OpenStackDataPlaneNodeSet controller and moved to the new OpenStackDataPlaneDeployment controller. - The DeployStrategy section is dropped from OpenStackDataPlaneNodeSet and is no longer used. The ansible related fields (tags, skipTags, limit) are now set directly on OpenStackDataPlaneDeployment. - dnsClusterAddresses and ctlplaneSearchDomain are now set as Status fields on OpenStackDataPlaneNodeSet so that the values can be retrieved when they are needed from the OpenStackDataPlaneDeployment controller. Signed-off-by: James Slagle --- Makefile | 1 + PROJECT | 9 + ...ack.org_openstackdataplanedeployments.yaml | 91 ++++++ ...nstack.org_openstackdataplanenodesets.yaml | 20 +- api/v1beta1/common.go | 20 -- api/v1beta1/conditions.go | 26 ++ .../openstackdataplanedeployment_types.go | 110 +++++++ .../openstackdataplanenodeset_types.go | 22 +- api/v1beta1/zz_generated.deepcopy.go | 122 +++++++- ...ack.org_openstackdataplanedeployments.yaml | 91 ++++++ ...nstack.org_openstackdataplanenodesets.yaml | 20 +- config/crd/kustomization.yaml | 3 + ...tion_in_openstackdataplanedeployments.yaml | 7 + ...hook_in_openstackdataplanedeployments.yaml | 16 + ...nstackdataplanedeployment_editor_role.yaml | 31 ++ ...nstackdataplanedeployment_viewer_role.yaml | 27 ++ config/rbac/role.yaml | 26 ++ ..._v1beta1_openstackdataplanedeployment.yaml | 7 + ...taplanedeployment_baremetal_with_ipam.yaml | 7 + ...ane_v1beta1_openstackdataplanenodeset.yaml | 2 - config/samples/kustomization.yaml | 1 + ...openstackdataplanedeployment_controller.go | 274 ++++++++++++++++++ .../openstackdataplanenodeset_controller.go | 50 +--- docs/openstack_dataplanedeloyment.md | 136 +++++++++ docs/openstack_dataplanenodeset.md | 17 +- docs/openstack_dataplaneservice.md | 14 - main.go | 12 + pkg/deployment/deployment.go | 48 ++- pkg/deployment/ipam.go | 6 +- tests/functional/base_test.go | 6 - ...enstackdataplanenodeset_controller_test.go | 10 - .../dataplane-create-test/00-assert.yaml | 4 - .../00-assert.yaml | 5 +- .../00-dataplane-create.yaml | 2 - .../01-assert.yaml | 55 +--- .../01-dataplane-deploy.yaml | 6 +- .../dataplane-extramounts/00-assert.yaml | 4 +- .../00-dataplane-create.yaml | 8 + .../dataplane-service-config/00-assert.yaml | 2 +- .../dataplane-service-config/00-create.yaml | 10 +- .../00-assert.yaml | 23 +- .../00-dataplane-create.yaml | 10 +- 42 files changed, 1076 insertions(+), 285 deletions(-) create mode 100644 api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml create mode 100644 api/v1beta1/openstackdataplanedeployment_types.go create mode 100644 config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml create mode 100644 config/crd/patches/cainjection_in_openstackdataplanedeployments.yaml create mode 100644 config/crd/patches/webhook_in_openstackdataplanedeployments.yaml create mode 100644 config/rbac/openstackdataplanedeployment_editor_role.yaml create mode 100644 config/rbac/openstackdataplanedeployment_viewer_role.yaml create mode 100644 config/samples/dataplane_v1beta1_openstackdataplanedeployment.yaml create mode 100644 config/samples/dataplane_v1beta1_openstackdataplanedeployment_baremetal_with_ipam.yaml create mode 100644 controllers/openstackdataplanedeployment_controller.go create mode 100644 docs/openstack_dataplanedeloyment.md diff --git a/Makefile b/Makefile index 063af5d49..011bc4d7b 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,7 @@ manifests: gowork controller-gen crd-to-markdown ## Generate WebhookConfiguratio rm -f api/bases/* && cp -a config/crd/bases api/ $(CRD_MARKDOWN) -f api/v1beta1/common.go -f api/v1beta1/openstackdataplanenodeset_types.go -n OpenStackDataPlaneNodeSet > docs/openstack_dataplanenodeset.md $(CRD_MARKDOWN) -f api/v1beta1/common.go -f api/v1beta1/openstackdataplaneservice_types.go -n OpenStackDataPlaneService > docs/openstack_dataplaneservice.md + $(CRD_MARKDOWN) -f api/v1beta1/common.go -f api/v1beta1/openstackdataplanedeployment_types.go -n OpenStackDataPlaneDeployment > docs/openstack_dataplanedeloyment.md .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. diff --git a/PROJECT b/PROJECT index 044530e77..e66db3494 100644 --- a/PROJECT +++ b/PROJECT @@ -33,4 +33,13 @@ resources: kind: OpenStackDataPlaneService path: github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1 version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: dataplane + kind: OpenStackDataPlaneDeployment + path: github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1 + version: v1beta1 version: "3" diff --git a/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml b/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml new file mode 100644 index 000000000..3fa3c613f --- /dev/null +++ b/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: openstackdataplanedeployments.dataplane.openstack.org +spec: + group: dataplane.openstack.org + names: + kind: OpenStackDataPlaneDeployment + listKind: OpenStackDataPlaneDeploymentList + plural: openstackdataplanedeployments + shortNames: + - osdpd + - osdpdeployment + - osdpdeployments + singular: openstackdataplanedeployment + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NodeSets + jsonPath: .spec.NodeSets + name: NodeSets + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + ansibleLimit: + type: string + ansibleSkipTags: + type: string + ansibleTags: + type: string + nodeSets: + items: + type: string + type: array + required: + - nodeSets + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + severity: + type: string + status: + type: string + type: + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + deployed: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml b/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml index cf10ee39a..2f75215a3 100644 --- a/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml +++ b/api/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml @@ -191,20 +191,6 @@ spec: - ctlplaneInterface - deploymentSSHSecret type: object - deployStrategy: - properties: - ansibleLimit: - type: string - ansibleSkipTags: - type: string - ansibleTags: - type: string - deploy: - default: true - type: boolean - required: - - deploy - type: object env: items: properties: @@ -1903,6 +1889,12 @@ spec: type: object status: properties: + CtlplaneSearchDomain: + type: string + DNSClusterAddresses: + items: + type: string + type: array conditions: items: properties: diff --git a/api/v1beta1/common.go b/api/v1beta1/common.go index e8b60f992..49bf20f4d 100644 --- a/api/v1beta1/common.go +++ b/api/v1beta1/common.go @@ -123,26 +123,6 @@ type NodeTemplate struct { NetworkData *corev1.SecretReference `json:"networkData,omitempty"` } -// DeployStrategySection for fields controlling the deployment -type DeployStrategySection struct { - // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} - // Deploy boolean to trigger ansible execution - // +kubebuilder:default=true - Deploy bool `json:"deploy"` - - // AnsibleTags for ansible execution - // +kubebuilder:validation:Optional - AnsibleTags string `json:"ansibleTags,omitempty"` - - // AnsibleLimit for ansible execution - // +kubebuilder:validation:Optional - AnsibleLimit string `json:"ansibleLimit,omitempty"` - - // AnsibleSkipTags for ansible execution - // +kubebuilder:validation:Optional - AnsibleSkipTags string `json:"ansibleSkipTags,omitempty"` -} - // NetworkConfigSection is a specification of the Network configuration details type NetworkConfigSection struct { diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 40943d91c..d4b4b4e58 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -85,4 +85,30 @@ const ( // InputReadyWaitingMessage not yet ready InputReadyWaitingMessage = "Waiting for input %s, not yet ready" + + // NodeSetDeploymentReadyCondition Status=True condition indicates if the + // NodeSet Deployment is finished and successful. + NodeSetDeploymentReadyCondition string = "%s Deployment ready" + + // NodeSetDeploymentReadyMessage ready + NodeSetDeploymentReadyMessage = "%s Deployment ready" + + // NodeSetDeploymentReadyWaitingMessage not yet ready + NodeSetDeploymentReadyWaitingMessage = "%s Deployment not yet ready" + + // NodeSetDeploymentErrorMessage error + NodeSetDeploymentErrorMessage = "%s Deployment error occurred %s" + + // NodeSetServiceDeploymentReadyCondition Status=True condition indicates if the + // NodeSet Deployment is finished and successful. + NodeSetServiceDeploymentReadyCondition string = "%s %s Deployment ready" + + // NodeSetServiceDeploymentReadyMessage ready + NodeSetServiceDeploymentReadyMessage = "%s %s Deployment ready" + + // NodeSetServiceDeploymentReadyWaitingMessage not yet ready + NodeSetServiceDeploymentReadyWaitingMessage = "%s %s Deployment not yet ready" + + // NodeSetServiceDeploymentErrorMessage error + NodeSetServiceDeploymentErrorMessage = "%s %s Deployment error occurred" ) diff --git a/api/v1beta1/openstackdataplanedeployment_types.go b/api/v1beta1/openstackdataplanedeployment_types.go new file mode 100644 index 000000000..63e90e6f0 --- /dev/null +++ b/api/v1beta1/openstackdataplanedeployment_types.go @@ -0,0 +1,110 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// OpenStackDataPlaneDeploymentSpec defines the desired state of OpenStackDataPlaneDeployment +type OpenStackDataPlaneDeploymentSpec struct { + + // +kubebuilder:validation:Required + // NodeSets is the list of NodeSets deployed + NodeSets []string `json:"nodeSets"` + + // AnsibleTags for ansible execution + // +kubebuilder:validation:Optional + AnsibleTags string `json:"ansibleTags,omitempty"` + + // AnsibleLimit for ansible execution + // +kubebuilder:validation:Optional + AnsibleLimit string `json:"ansibleLimit,omitempty"` + + // AnsibleSkipTags for ansible execution + // +kubebuilder:validation:Optional + AnsibleSkipTags string `json:"ansibleSkipTags,omitempty"` +} + +// OpenStackDataPlaneDeploymentStatus defines the observed state of OpenStackDataPlaneDeployment +type OpenStackDataPlaneDeploymentStatus struct { + // +operator-sdk:csv:customresourcedefinitions:type=status,xDescriptors={"urn:alm:descriptor:io.kubernetes.conditions"} + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // +operator-sdk:csv:customresourcedefinitions:type=status,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} + // Deployed + Deployed bool `json:"deployed,omitempty" optional:"true"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+operator-sdk:csv:customresourcedefinitions:displayName="OpenStack Data Plane Deployments" +//+kubebuilder:resource:shortName=osdpd;osdpdeployment;osdpdeployments +//+kubebuilder:printcolumn:name="NodeSets",type="string",JSONPath=".spec.NodeSets",description="NodeSets" +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +//+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" + +// OpenStackDataPlaneDeployment is the Schema for the openstackdataplanedeployments API +type OpenStackDataPlaneDeployment struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec OpenStackDataPlaneDeploymentSpec `json:"spec,omitempty"` + Status OpenStackDataPlaneDeploymentStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// OpenStackDataPlaneDeploymentList contains a list of OpenStackDataPlaneDeployment +type OpenStackDataPlaneDeploymentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []OpenStackDataPlaneDeployment `json:"items"` +} + +func init() { + SchemeBuilder.Register(&OpenStackDataPlaneDeployment{}, &OpenStackDataPlaneDeploymentList{}) +} + +// IsReady - returns true if the OpenStackDataPlaneDeployment is ready +func (instance OpenStackDataPlaneDeployment) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.ReadyCondition) +} + +// InitConditions - Initializes Status Conditons +func (instance *OpenStackDataPlaneDeployment) InitConditions() { + instance.Status.Conditions = condition.Conditions{} + + cl := condition.CreateList( + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.InitReason), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InitReason), + ) + + if instance.Spec.NodeSets != nil { + for _, nodeSet := range instance.Spec.NodeSets { + readyCondition := condition.Type(fmt.Sprintf(NodeSetDeploymentReadyCondition, nodeSet)) + cl = append(cl, *condition.UnknownCondition(readyCondition, condition.InitReason, condition.InitReason)) + } + } + + instance.Status.Conditions.Init(&cl) + instance.Status.Deployed = false +} diff --git a/api/v1beta1/openstackdataplanenodeset_types.go b/api/v1beta1/openstackdataplanenodeset_types.go index 830eec5b8..1ec371877 100644 --- a/api/v1beta1/openstackdataplanenodeset_types.go +++ b/api/v1beta1/openstackdataplanenodeset_types.go @@ -17,8 +17,6 @@ limitations under the License. package v1beta1 import ( - "fmt" - condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" baremetalv1 "github.com/openstack-k8s-operators/openstack-baremetal-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" @@ -52,10 +50,6 @@ type OpenStackDataPlaneNodeSetSpec struct { // Env is a list containing the environment variables to pass to the pod Env []corev1.EnvVar `json:"env,omitempty"` - // +kubebuilder:validation:Optional - // DeployStrategy section to control how the node is deployed - DeployStrategy DeployStrategySection `json:"deployStrategy,omitempty"` - // +kubebuilder:validation:Optional // NetworkAttachments is a list of NetworkAttachment resource names to pass to the ansibleee resource // which allows to connect the ansibleee runner to the given network @@ -92,6 +86,12 @@ type OpenStackDataPlaneNodeSetStatus struct { // +operator-sdk:csv:customresourcedefinitions:type=status,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} // Deployed Deployed bool `json:"deployed,omitempty" optional:"true"` + + // DNSClusterAddresses + DNSClusterAddresses []string `json:"DNSClusterAddresses,omitempty" optional:"true"` + + // CtlplaneSearchDomain + CtlplaneSearchDomain string `json:"CtlplaneSearchDomain,omitempty" optional:"true"` } //+kubebuilder:object:root=true @@ -130,13 +130,6 @@ func (instance *OpenStackDataPlaneNodeSet) InitConditions() { cl = append(cl, *condition.UnknownCondition(NodeSetBareMetalProvisionReadyCondition, condition.InitReason, condition.InitReason)) } - if instance.Spec.Services != nil && instance.Spec.DeployStrategy.Deploy { - for _, service := range instance.Spec.Services { - readyCondition := condition.Type(fmt.Sprintf(ServiceReadyCondition, service)) - cl = append(cl, *condition.UnknownCondition(readyCondition, condition.InitReason, condition.InitReason)) - } - } - instance.Status.Conditions.Init(&cl) instance.Status.Deployed = false } @@ -145,9 +138,6 @@ func (instance *OpenStackDataPlaneNodeSet) InitConditions() { func (instance OpenStackDataPlaneNodeSet) GetAnsibleEESpec() AnsibleEESpec { return AnsibleEESpec{ NetworkAttachments: instance.Spec.NetworkAttachments, - AnsibleTags: instance.Spec.DeployStrategy.AnsibleTags, - AnsibleLimit: instance.Spec.DeployStrategy.AnsibleLimit, - AnsibleSkipTags: instance.Spec.DeployStrategy.AnsibleSkipTags, ExtraMounts: instance.Spec.NodeTemplate.ExtraMounts, Env: instance.Spec.Env, } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 7f5591692..67ec54aa8 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -99,21 +99,6 @@ func (in *AnsibleOpts) DeepCopy() *AnsibleOpts { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DeployStrategySection) DeepCopyInto(out *DeployStrategySection) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeployStrategySection. -func (in *DeployStrategySection) DeepCopy() *DeployStrategySection { - if in == nil { - return nil - } - out := new(DeployStrategySection) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeService) DeepCopyInto(out *KubeService) { *out = *in @@ -226,6 +211,107 @@ func (in *NodeTemplate) DeepCopy() *NodeTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackDataPlaneDeployment) DeepCopyInto(out *OpenStackDataPlaneDeployment) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackDataPlaneDeployment. +func (in *OpenStackDataPlaneDeployment) DeepCopy() *OpenStackDataPlaneDeployment { + if in == nil { + return nil + } + out := new(OpenStackDataPlaneDeployment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenStackDataPlaneDeployment) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackDataPlaneDeploymentList) DeepCopyInto(out *OpenStackDataPlaneDeploymentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]OpenStackDataPlaneDeployment, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackDataPlaneDeploymentList. +func (in *OpenStackDataPlaneDeploymentList) DeepCopy() *OpenStackDataPlaneDeploymentList { + if in == nil { + return nil + } + out := new(OpenStackDataPlaneDeploymentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *OpenStackDataPlaneDeploymentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackDataPlaneDeploymentSpec) DeepCopyInto(out *OpenStackDataPlaneDeploymentSpec) { + *out = *in + if in.NodeSets != nil { + in, out := &in.NodeSets, &out.NodeSets + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackDataPlaneDeploymentSpec. +func (in *OpenStackDataPlaneDeploymentSpec) DeepCopy() *OpenStackDataPlaneDeploymentSpec { + if in == nil { + return nil + } + out := new(OpenStackDataPlaneDeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenStackDataPlaneDeploymentStatus) DeepCopyInto(out *OpenStackDataPlaneDeploymentStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackDataPlaneDeploymentStatus. +func (in *OpenStackDataPlaneDeploymentStatus) DeepCopy() *OpenStackDataPlaneDeploymentStatus { + if in == nil { + return nil + } + out := new(OpenStackDataPlaneDeploymentStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenStackDataPlaneNodeSet) DeepCopyInto(out *OpenStackDataPlaneNodeSet) { *out = *in @@ -304,7 +390,6 @@ func (in *OpenStackDataPlaneNodeSetSpec) DeepCopyInto(out *OpenStackDataPlaneNod (*in)[i].DeepCopyInto(&(*out)[i]) } } - out.DeployStrategy = in.DeployStrategy if in.NetworkAttachments != nil { in, out := &in.NetworkAttachments, &out.NetworkAttachments *out = make([]string, len(*in)) @@ -337,6 +422,11 @@ func (in *OpenStackDataPlaneNodeSetStatus) DeepCopyInto(out *OpenStackDataPlaneN (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DNSClusterAddresses != nil { + in, out := &in.DNSClusterAddresses, &out.DNSClusterAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackDataPlaneNodeSetStatus. diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml new file mode 100644 index 000000000..3fa3c613f --- /dev/null +++ b/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: openstackdataplanedeployments.dataplane.openstack.org +spec: + group: dataplane.openstack.org + names: + kind: OpenStackDataPlaneDeployment + listKind: OpenStackDataPlaneDeploymentList + plural: openstackdataplanedeployments + shortNames: + - osdpd + - osdpdeployment + - osdpdeployments + singular: openstackdataplanedeployment + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NodeSets + jsonPath: .spec.NodeSets + name: NodeSets + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + ansibleLimit: + type: string + ansibleSkipTags: + type: string + ansibleTags: + type: string + nodeSets: + items: + type: string + type: array + required: + - nodeSets + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + severity: + type: string + status: + type: string + type: + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + deployed: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml index cf10ee39a..2f75215a3 100644 --- a/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml +++ b/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml @@ -191,20 +191,6 @@ spec: - ctlplaneInterface - deploymentSSHSecret type: object - deployStrategy: - properties: - ansibleLimit: - type: string - ansibleSkipTags: - type: string - ansibleTags: - type: string - deploy: - default: true - type: boolean - required: - - deploy - type: object env: items: properties: @@ -1903,6 +1889,12 @@ spec: type: object status: properties: + CtlplaneSearchDomain: + type: string + DNSClusterAddresses: + items: + type: string + type: array conditions: items: properties: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d9837f7ad..f292bb167 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/dataplane.openstack.org_openstackdataplanenodesets.yaml - bases/dataplane.openstack.org_openstackdataplaneservices.yaml +- bases/dataplane.openstack.org_openstackdataplanedeployments.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -13,6 +14,7 @@ patchesStrategicMerge: #- patches/webhook_in_openstackdataplaneroles.yaml #- patches/webhook_in_openstackdataplanenodes.yaml #- patches/webhook_in_openstackdataplaneservices.yaml +#- patches/webhook_in_openstackdataplanedeployments.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -21,6 +23,7 @@ patchesStrategicMerge: #- patches/cainjection_in_openstackdataplaneroles.yaml #- patches/cainjection_in_openstackdataplanenodes.yaml #- patches/cainjection_in_openstackdataplaneservices.yaml +#- patches/cainjection_in_openstackdataplanedeployments.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_openstackdataplanedeployments.yaml b/config/crd/patches/cainjection_in_openstackdataplanedeployments.yaml new file mode 100644 index 000000000..0411aef54 --- /dev/null +++ b/config/crd/patches/cainjection_in_openstackdataplanedeployments.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: openstackdataplanedeployments.dataplane.openstack.org diff --git a/config/crd/patches/webhook_in_openstackdataplanedeployments.yaml b/config/crd/patches/webhook_in_openstackdataplanedeployments.yaml new file mode 100644 index 000000000..d12cf7ec8 --- /dev/null +++ b/config/crd/patches/webhook_in_openstackdataplanedeployments.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: openstackdataplanedeployments.dataplane.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/openstackdataplanedeployment_editor_role.yaml b/config/rbac/openstackdataplanedeployment_editor_role.yaml new file mode 100644 index 000000000..2a699d9f7 --- /dev/null +++ b/config/rbac/openstackdataplanedeployment_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit openstackdataplanedeployments. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: openstackdataplanedeployment-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: dataplane-operator + app.kubernetes.io/part-of: dataplane-operator + app.kubernetes.io/managed-by: kustomize + name: openstackdataplanedeployment-editor-role +rules: +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments/status + verbs: + - get diff --git a/config/rbac/openstackdataplanedeployment_viewer_role.yaml b/config/rbac/openstackdataplanedeployment_viewer_role.yaml new file mode 100644 index 000000000..7dfe913b6 --- /dev/null +++ b/config/rbac/openstackdataplanedeployment_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view openstackdataplanedeployments. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: openstackdataplanedeployment-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: dataplane-operator + app.kubernetes.io/part-of: dataplane-operator + app.kubernetes.io/managed-by: kustomize + name: openstackdataplanedeployment-viewer-role +rules: +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments + verbs: + - get + - list + - watch +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 05df34e70..de33a0149 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -77,6 +77,32 @@ rules: - patch - update - watch +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments/finalizers + verbs: + - update +- apiGroups: + - dataplane.openstack.org + resources: + - openstackdataplanedeployments/status + verbs: + - get + - patch + - update - apiGroups: - dataplane.openstack.org resources: diff --git a/config/samples/dataplane_v1beta1_openstackdataplanedeployment.yaml b/config/samples/dataplane_v1beta1_openstackdataplanedeployment.yaml new file mode 100644 index 000000000..3f7c3921d --- /dev/null +++ b/config/samples/dataplane_v1beta1_openstackdataplanedeployment.yaml @@ -0,0 +1,7 @@ +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: openstack-edpm +spec: + nodeSets: + - openstack-edpm diff --git a/config/samples/dataplane_v1beta1_openstackdataplanedeployment_baremetal_with_ipam.yaml b/config/samples/dataplane_v1beta1_openstackdataplanedeployment_baremetal_with_ipam.yaml new file mode 100644 index 000000000..a88134791 --- /dev/null +++ b/config/samples/dataplane_v1beta1_openstackdataplanedeployment_baremetal_with_ipam.yaml @@ -0,0 +1,7 @@ +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: openstack-edpm-ipam +spec: + nodeSets: + - openstack-edpm-ipam diff --git a/config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml b/config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml index 3e2464673..e64a23acb 100644 --- a/config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml +++ b/config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml @@ -19,8 +19,6 @@ spec: - libvirt - nova preProvisioned: true - deployStrategy: - deploy: true nodes: edpm-compute-0: hostName: edpm-compute-0 diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index dfdf1cdd7..5833b432c 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -4,4 +4,5 @@ resources: - dataplane_v1beta1_openstackdataplanenodeset_ceph.yaml - dataplane_v1beta1_openstackdataplanenodeset_customnetworks.yaml - dataplane_v1beta1_openstackdataplaneservice.yaml + - dataplane_v1beta1_openstackdataplanedeployment.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go new file mode 100644 index 000000000..1f3e988cd --- /dev/null +++ b/controllers/openstackdataplanedeployment_controller.go @@ -0,0 +1,274 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/go-logr/logr" + dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1" + "github.com/openstack-k8s-operators/dataplane-operator/pkg/deployment" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + "github.com/openstack-k8s-operators/openstack-ansibleee-operator/api/v1alpha1" +) + +// OpenStackDataPlaneDeploymentReconciler reconciles a OpenStackDataPlaneDeployment object +type OpenStackDataPlaneDeploymentReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme + Log logr.Logger +} + +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplanedeployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplanedeployments/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplanedeployments/finalizers,verbs=update +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplanenodesets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplanenodesets/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplanenodesets/finalizers,verbs=update +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplaneservices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplaneservices/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=dataplane.openstack.org,resources=openstackdataplaneservices/finalizers,verbs=update +//+kubebuilder:rbac:groups=ansibleee.openstack.org,resources=openstackansibleees,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=baremetal.openstack.org,resources=openstackbaremetalsets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +//+kubebuilder:rbac:groups=network.openstack.org,resources=ipsets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=network.openstack.org,resources=ipsets/status,verbs=get +//+kubebuilder:rbac:groups=network.openstack.org,resources=ipsets/finalizers,verbs=update +//+kubebuilder:rbac:groups=network.openstack.org,resources=netconfigs,verbs=get;list;watch +//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsmasqs,verbs=get;list;watch +//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsmasqs/status,verbs=get +//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsdata,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsdata/status,verbs=get +//+kubebuilder:rbac:groups=network.openstack.org,resources=dnsdata/finalizers,verbs=update +//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + + logger := log.FromContext(ctx) + logger.Info("Reconciling Deployment") + + // Fetch the OpenStackDataPlaneDeployment instance + instance := &dataplanev1.OpenStackDataPlaneDeployment{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, _ := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + logger, + ) + if err != nil { + return ctrl.Result{}, err + } + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { // update the Ready condition based on the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + // something is not ready so reset the Ready condition + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + // and recalculate it based on the state of the rest of the conditions + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + logger.Error(err, "Error updating instance status conditions") + _err = err + return + } + }() + + // If the deploy is already done, return immediately. + if instance.Status.Deployed { + logger.Info("Already deployed", "instance.Status.Deployed", instance.Status.Deployed) + return ctrl.Result{}, nil + } + + // Initialize Status + if instance.Status.Conditions == nil { + instance.InitConditions() + // Register overall status immediately to have an early feedback e.g. + // in the cli + return ctrl.Result{}, nil + } + + // All setup tasks done + // Mark SetupReadyCondition=True + instance.Status.Conditions.MarkTrue(dataplanev1.SetupReadyCondition, condition.ReadyMessage) + + // Ensure NodeSets + nodeSets := dataplanev1.OpenStackDataPlaneNodeSetList{} + for _, nodeSet := range instance.Spec.NodeSets { + + // Fetch the OpenStackDataPlaneNodeSet instance + nodeSetInstance := &dataplanev1.OpenStackDataPlaneNodeSet{} + err := r.Client.Get( + ctx, + types.NamespacedName{ + Namespace: instance.GetNamespace(), + Name: nodeSet}, + nodeSetInstance) + if err != nil { + // NodeSet not found, force a requeue + if k8s_errors.IsNotFound(err) { + logger.Info("NodeSet not found", "NodeSet", nodeSet) + return ctrl.Result{RequeueAfter: time.Second * 15}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + nodeSets.Items = append(nodeSets.Items, *nodeSetInstance) + } + + // Check that all nodeSets are SetupReady + for _, nodeSet := range nodeSets.Items { + if !nodeSet.Status.Conditions.IsTrue(dataplanev1.SetupReadyCondition) { + logger.Info("NodeSet SetupReadyCondition is not True", "NodeSet", nodeSet.Name) + return ctrl.Result{RequeueAfter: time.Second * 15}, nil + } + } + + // All nodeSets successfully fetched. + // Mark InputReadyCondition=True + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage) + + // Deploy each nodeSet + // The loop starts and checks NodeSet deployments sequentially. However, after they + // are started, they are running in parallel, since the loop does not wait + // for the first started NodeSet to finish before starting the next. + shouldRequeue := false + haveError := false + for _, nodeSet := range nodeSets.Items { + + logger.Info(fmt.Sprintf("Deploying NodeSet: %s", nodeSet.Name)) + logger.Info("Set Status.Deployed to false", "instance", instance) + instance.Status.Deployed = false + logger.Info("Set DeploymentReadyCondition false", "instance", instance) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, condition.RequestedReason, + condition.SeverityInfo, condition.DeploymentReadyRunningMessage)) + ansibleEESpec := nodeSet.GetAnsibleEESpec() + ansibleEESpec.AnsibleTags = instance.Spec.AnsibleTags + ansibleEESpec.AnsibleSkipTags = instance.Spec.AnsibleSkipTags + ansibleEESpec.AnsibleLimit = instance.Spec.AnsibleLimit + + nodeSetSecretInv := fmt.Sprintf("dataplanenodeset-%s", nodeSet.Name) + + if nodeSet.Status.DNSClusterAddresses != nil && nodeSet.Status.CtlplaneSearchDomain != "" { + ansibleEESpec.DNSConfig = &corev1.PodDNSConfig{ + Nameservers: nodeSet.Status.DNSClusterAddresses, + Searches: []string{nodeSet.Status.CtlplaneSearchDomain}, + } + } + + deployResult, err := deployment.Deploy( + ctx, helper, &nodeSet, instance, + nodeSetSecretInv, &instance.Status, ansibleEESpec, + nodeSet.Spec.Services) + + if err != nil { + util.LogErrorForObject(helper, err, fmt.Sprintf("OpenStackDeployment error for NodeSet %s", nodeSet.Name), instance) + haveError = true + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + dataplanev1.DataPlaneNodeSetErrorMessage, + err.Error())) + } + + if deployResult != nil { + shouldRequeue = true + } else { + logger.Info("OpenStackDeployment succeeded for NodeSet", "NodeSet", nodeSet.Name) + logger.Info("Set NodeSetDeploymentReadyCondition true", "nodeSet", nodeSet.Name) + instance.Status.Conditions.Set( + condition.TrueCondition( + condition.Type(fmt.Sprintf(dataplanev1.NodeSetDeploymentReadyCondition, nodeSet.Name)), + condition.DeploymentReadyMessage)) + + logger.Info("Set NodeSet DeploymentReadyCondition true", "nodeSet", nodeSet.Name) + _, err = controllerutil.CreateOrPatch(ctx, helper.GetClient(), &nodeSet, func() error { + nodeSet.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + return nil + }) + if err != nil { + util.LogErrorForObject(helper, err, "Unable to set Status on NodeSet", &nodeSet) + return ctrl.Result{}, err + } + } + } + + if haveError { + return ctrl.Result{}, err + } + + if shouldRequeue { + logger.Info("Not all NodeSets done for OpenStackDeployment") + return ctrl.Result{}, nil + } + + logger.Info("Set status deploy true", "instance", instance) + instance.Status.Deployed = true + logger.Info("Set DeploymentReadyCondition true", "instance", instance) + instance.Status.Conditions.Set(condition.TrueCondition(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage)) + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *OpenStackDataPlaneDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&dataplanev1.OpenStackDataPlaneDeployment{}). + Owns(&v1alpha1.OpenStackAnsibleEE{}). + Complete(r) +} diff --git a/controllers/openstackdataplanenodeset_controller.go b/controllers/openstackdataplanenodeset_controller.go index ef0f750da..dabf81850 100644 --- a/controllers/openstackdataplanenodeset_controller.go +++ b/controllers/openstackdataplanenodeset_controller.go @@ -174,6 +174,8 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req if err != nil || !isReady { return ctrl.Result{}, err } + instance.Status.DNSClusterAddresses = dnsClusterAddresses + instance.Status.CtlplaneSearchDomain = ctlplaneSearchDomain ansibleSSHPrivateKeySecret := instance.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret @@ -240,7 +242,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req } // Generate NodeSet Inventory - roleSecret, err := deployment.GenerateNodeSetInventory(ctx, helper, instance, + _, err = deployment.GenerateNodeSetInventory(ctx, helper, instance, allIPSets, dnsAddresses) if err != nil { util.LogErrorForObject(helper, err, fmt.Sprintf("Unable to generate inventory for %s", instance.Name), instance) @@ -250,52 +252,8 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req // all setup tasks complete, mark SetupReadyCondition True instance.Status.Conditions.MarkTrue(dataplanev1.SetupReadyCondition, condition.ReadyMessage) - // Trigger executions based on the OpenStackDataPlane Controller state. - r.Log.Info("NodeSet", "DeployStrategy", instance.Spec.DeployStrategy.Deploy, - "NodeSet.Namespace", instance.Namespace, "NodeSet.Name", instance.Name) - if instance.Spec.DeployStrategy.Deploy { - logger.Info(fmt.Sprintf("Deploying NodeSet: %s", instance.Name)) - logger.Info("Set Status.Deployed to false", "instance", instance) - instance.Status.Deployed = false - logger.Info("Set DeploymentReadyCondition false", "instance", instance) - instance.Status.Conditions.Set(condition.FalseCondition( - condition.DeploymentReadyCondition, condition.RequestedReason, - condition.SeverityInfo, condition.DeploymentReadyRunningMessage)) - ansibleEESpec := instance.GetAnsibleEESpec() - if dnsClusterAddresses != nil && ctlplaneSearchDomain != "" { - ansibleEESpec.DNSConfig = &corev1.PodDNSConfig{ - Nameservers: dnsClusterAddresses, - Searches: []string{ctlplaneSearchDomain}, - } - } - deployResult, err := deployment.Deploy( - ctx, helper, instance, ansibleSSHPrivateKeySecret, - roleSecret, &instance.Status, ansibleEESpec, - instance.Spec.Services, instance) - if err != nil { - util.LogErrorForObject(helper, err, fmt.Sprintf("Unable to deploy %s", instance.Name), instance) - instance.Status.Conditions.Set(condition.FalseCondition( - condition.ReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - dataplanev1.DataPlaneNodeSetErrorMessage, - err.Error())) - return ctrl.Result{}, err - } - if deployResult != nil { - result = *deployResult - return result, nil - } - logger.Info("Set status deploy true", "instance", instance) - instance.Status.Deployed = true - logger.Info("Set DeploymentReadyCondition true", "instance", instance) - instance.Status.Conditions.Set(condition.TrueCondition(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage)) - - } - // Set DeploymentReadyCondition to False if it was unknown. - // Handles the case where the NodeSet is created with - // DeployStrategy.Deploy=false. + // Handles the case where the NodeSet is created, but not yet deployed. if instance.Status.Conditions.IsUnknown(condition.DeploymentReadyCondition) { logger.Info("Set DeploymentReadyCondition false") instance.Status.Conditions.Set(condition.FalseCondition(condition.DeploymentReadyCondition, condition.NotRequestedReason, condition.SeverityInfo, condition.DeploymentReadyInitMessage)) diff --git a/docs/openstack_dataplanedeloyment.md b/docs/openstack_dataplanedeloyment.md new file mode 100644 index 000000000..afa34706a --- /dev/null +++ b/docs/openstack_dataplanedeloyment.md @@ -0,0 +1,136 @@ + +### Custom Resources + +* [OpenStackDataPlaneDeployment](#openstackdataplanedeployment) + +### Sub Resources + +* [AnsibleEESpec](#ansibleeespec) +* [AnsibleOpts](#ansibleopts) +* [NetworkConfigSection](#networkconfigsection) +* [NodeSection](#nodesection) +* [NodeTemplate](#nodetemplate) +* [OpenStackDataPlaneDeploymentList](#openstackdataplanedeploymentlist) +* [OpenStackDataPlaneDeploymentSpec](#openstackdataplanedeploymentspec) +* [OpenStackDataPlaneDeploymentStatus](#openstackdataplanedeploymentstatus) + +#### AnsibleEESpec + +AnsibleEESpec is a specification of the ansible EE attributes + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| networkAttachments | NetworkAttachments is a list of NetworkAttachment resource names to pass to the ansibleee resource which allows to connect the ansibleee runner to the given network | []string | false | +| openStackAnsibleEERunnerImage | OpenStackAnsibleEERunnerImage image to use as the ansibleEE runner image | string | false | +| ansibleTags | AnsibleTags for ansible execution | string | false | +| ansibleLimit | AnsibleLimit for ansible execution | string | false | +| ansibleSkipTags | AnsibleSkipTags for ansible execution | string | false | +| extraMounts | ExtraMounts containing files which can be mounted into an Ansible Execution Pod | []storage.VolMounts | false | +| env | Env is a list containing the environment variables to pass to the pod | []corev1.EnvVar | false | +| dnsConfig | DNSConfig for setting dnsservers | *corev1.PodDNSConfig | false | + +[Back to Custom Resources](#custom-resources) + +#### AnsibleOpts + +AnsibleOpts defines a logical grouping of Ansible related configuration options. + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| ansibleUser | AnsibleUser SSH user for Ansible connection | string | false | +| ansibleHost | AnsibleHost SSH host for Ansible connection | string | false | +| ansiblePort | AnsiblePort SSH port for Ansible connection | int | false | +| ansibleVars | AnsibleVars for configuring ansible | map[string]json.RawMessage | false | + +[Back to Custom Resources](#custom-resources) + +#### NetworkConfigSection + +NetworkConfigSection is a specification of the Network configuration details + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| template | Template - Contains a Ansible j2 nic config template to use when applying node network configuration | string | false | + +[Back to Custom Resources](#custom-resources) + +#### NodeSection + +NodeSection defines the top level attributes inherited by nodes in the CR. + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| hostName | HostName - node name | string | false | +| networkConfig | NetworkConfig - Network configuration details. Contains os-net-config related properties. | [NetworkConfigSection](#networkconfigsection) | true | +| networks | Networks - Instance networks | []infranetworkv1.IPSetNetwork | false | +| managementNetwork | ManagementNetwork - Name of network to use for management (SSH/Ansible) | string | false | +| ansible | Ansible is the group of Ansible related configuration options. | [AnsibleOpts](#ansibleopts) | false | +| extraMounts | ExtraMounts containing files which can be mounted into an Ansible Execution Pod | []storage.VolMounts | false | +| userData | UserData node specific user-data | *corev1.SecretReference | false | +| networkData | NetworkData node specific network-data | *corev1.SecretReference | false | + +[Back to Custom Resources](#custom-resources) + +#### NodeTemplate + +NodeTemplate is a specification of the node attributes that override top level attributes. + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| ansibleSSHPrivateKeySecret | AnsibleSSHPrivateKeySecret Name of a private SSH key secret containing private SSH key for connecting to node. The named secret must be of the form: Secret.data.ssh-privatekey: | string | true | +| networkConfig | NetworkConfig - Network configuration details. Contains os-net-config related properties. | [NetworkConfigSection](#networkconfigsection) | false | +| networks | Networks - Instance networks | []infranetworkv1.IPSetNetwork | false | +| managementNetwork | ManagementNetwork - Name of network to use for management (SSH/Ansible) | string | false | +| ansible | Ansible is the group of Ansible related configuration options. | [AnsibleOpts](#ansibleopts) | false | +| extraMounts | ExtraMounts containing files which can be mounted into an Ansible Execution Pod | []storage.VolMounts | false | +| userData | UserData node specific user-data | *corev1.SecretReference | false | +| networkData | NetworkData node specific network-data | *corev1.SecretReference | false | + +[Back to Custom Resources](#custom-resources) + +#### OpenStackDataPlaneDeployment + +OpenStackDataPlaneDeployment is the Schema for the openstackdataplanedeployments API + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| metadata | | metav1.ObjectMeta | false | +| spec | | [OpenStackDataPlaneDeploymentSpec](#openstackdataplanedeploymentspec) | false | +| status | | [OpenStackDataPlaneDeploymentStatus](#openstackdataplanedeploymentstatus) | false | + +[Back to Custom Resources](#custom-resources) + +#### OpenStackDataPlaneDeploymentList + +OpenStackDataPlaneDeploymentList contains a list of OpenStackDataPlaneDeployment + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| metadata | | metav1.ListMeta | false | +| items | | [][OpenStackDataPlaneDeployment](#openstackdataplanedeployment) | true | + +[Back to Custom Resources](#custom-resources) + +#### OpenStackDataPlaneDeploymentSpec + +OpenStackDataPlaneDeploymentSpec defines the desired state of OpenStackDataPlaneDeployment + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| nodeSets | NodeSets is the list of NodeSets deployed | []string | true | +| ansibleTags | AnsibleTags for ansible execution | string | false | +| ansibleLimit | AnsibleLimit for ansible execution | string | false | +| ansibleSkipTags | AnsibleSkipTags for ansible execution | string | false | + +[Back to Custom Resources](#custom-resources) + +#### OpenStackDataPlaneDeploymentStatus + +OpenStackDataPlaneDeploymentStatus defines the observed state of OpenStackDataPlaneDeployment + +| Field | Description | Scheme | Required | +| ----- | ----------- | ------ | -------- | +| conditions | Conditions | condition.Conditions | false | +| deployed | Deployed | bool | false | + +[Back to Custom Resources](#custom-resources) diff --git a/docs/openstack_dataplanenodeset.md b/docs/openstack_dataplanenodeset.md index 218634e75..8c05e6946 100644 --- a/docs/openstack_dataplanenodeset.md +++ b/docs/openstack_dataplanenodeset.md @@ -7,7 +7,6 @@ * [AnsibleEESpec](#ansibleeespec) * [AnsibleOpts](#ansibleopts) -* [DeployStrategySection](#deploystrategysection) * [NetworkConfigSection](#networkconfigsection) * [NodeSection](#nodesection) * [NodeTemplate](#nodetemplate) @@ -45,19 +44,6 @@ AnsibleOpts defines a logical grouping of Ansible related configuration options. [Back to Custom Resources](#custom-resources) -#### DeployStrategySection - -DeployStrategySection for fields controlling the deployment - -| Field | Description | Scheme | Required | -| ----- | ----------- | ------ | -------- | -| deploy | Deploy boolean to trigger ansible execution | bool | true | -| ansibleTags | AnsibleTags for ansible execution | string | false | -| ansibleLimit | AnsibleLimit for ansible execution | string | false | -| ansibleSkipTags | AnsibleSkipTags for ansible execution | string | false | - -[Back to Custom Resources](#custom-resources) - #### NetworkConfigSection NetworkConfigSection is a specification of the Network configuration details @@ -136,7 +122,6 @@ OpenStackDataPlaneNodeSetSpec defines the desired state of OpenStackDataPlaneNod | nodes | Nodes - Map of Node Names and node specific data. Values here override defaults in the upper level section. | map[string][NodeSection](#nodesection) | true | | preProvisioned | \n\nPreProvisioned - Whether the nodes are actually pre-provisioned (True) or should be preprovisioned (False) | bool | false | | env | Env is a list containing the environment variables to pass to the pod | []corev1.EnvVar | false | -| deployStrategy | DeployStrategy section to control how the node is deployed | [DeployStrategySection](#deploystrategysection) | false | | networkAttachments | NetworkAttachments is a list of NetworkAttachment resource names to pass to the ansibleee resource which allows to connect the ansibleee runner to the given network | []string | false | | services | Services list | []string | true | @@ -150,5 +135,7 @@ OpenStackDataPlaneNodeSetStatus defines the observed state of OpenStackDataPlane | ----- | ----------- | ------ | -------- | | conditions | Conditions | condition.Conditions | false | | deployed | Deployed | bool | false | +| DNSClusterAddresses | DNSClusterAddresses | []string | false | +| CtlplaneSearchDomain | CtlplaneSearchDomain | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/openstack_dataplaneservice.md b/docs/openstack_dataplaneservice.md index a1d39c92c..92aa87241 100644 --- a/docs/openstack_dataplaneservice.md +++ b/docs/openstack_dataplaneservice.md @@ -7,7 +7,6 @@ * [AnsibleEESpec](#ansibleeespec) * [AnsibleOpts](#ansibleopts) -* [DeployStrategySection](#deploystrategysection) * [NetworkConfigSection](#networkconfigsection) * [NodeSection](#nodesection) * [NodeTemplate](#nodetemplate) @@ -46,19 +45,6 @@ AnsibleOpts defines a logical grouping of Ansible related configuration options. [Back to Custom Resources](#custom-resources) -#### DeployStrategySection - -DeployStrategySection for fields controlling the deployment - -| Field | Description | Scheme | Required | -| ----- | ----------- | ------ | -------- | -| deploy | Deploy boolean to trigger ansible execution | bool | true | -| ansibleTags | AnsibleTags for ansible execution | string | false | -| ansibleLimit | AnsibleLimit for ansible execution | string | false | -| ansibleSkipTags | AnsibleSkipTags for ansible execution | string | false | - -[Back to Custom Resources](#custom-resources) - #### NetworkConfigSection NetworkConfigSection is a specification of the Network configuration details diff --git a/main.go b/main.go index 543e7b522..82e904473 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1" + dataplanev1beta1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1" "github.com/openstack-k8s-operators/dataplane-operator/controllers" //+kubebuilder:scaffold:imports ) @@ -58,6 +59,7 @@ func init() { utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(baremetalv1.AddToScheme(scheme)) utilruntime.Must(infranetworkv1.AddToScheme(scheme)) + utilruntime.Must(dataplanev1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -135,6 +137,16 @@ func main() { } checker = mgr.GetWebhookServer().StartedChecker() } + + if err = (&controllers.OpenStackDataPlaneDeploymentReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + Log: ctrl.Log.WithName("controllers").WithName("OpenStackDataPlaneDeployment"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "OpenStackDataPlaneDeployment") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", checker); err != nil { diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 472ddf0f6..45c06be57 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -35,23 +35,18 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/storage" ansibleeev1alpha1 "github.com/openstack-k8s-operators/openstack-ansibleee-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) -// deployFuncDef so we can pass a function to ConditionalDeploy -type deployFuncDef func(context.Context, *helper.Helper, client.Object, string, string, dataplanev1.AnsibleEESpec, dataplanev1.OpenStackDataPlaneService) error - // Deploy function encapsulating primary deloyment handling func Deploy( ctx context.Context, helper *helper.Helper, - obj client.Object, - sshKeySecret string, + nodeSet *dataplanev1.OpenStackDataPlaneNodeSet, + deployment *dataplanev1.OpenStackDataPlaneDeployment, inventorySecret string, - status *dataplanev1.OpenStackDataPlaneNodeSetStatus, + status *dataplanev1.OpenStackDataPlaneDeploymentStatus, aeeSpec dataplanev1.AnsibleEESpec, services []string, - nodeSet *dataplanev1.OpenStackDataPlaneNodeSet, ) (*ctrl.Result, error) { log := helper.GetLogger() @@ -60,7 +55,6 @@ func Deploy( var readyMessage string var readyWaitingMessage string var readyErrorMessage string - var deployFunc deployFuncDef var deployName string var deployLabel string @@ -76,13 +70,12 @@ func Deploy( if err != nil { return &ctrl.Result{}, err } - deployFunc = DeployService deployName = foundService.Name deployLabel = foundService.Spec.Label - readyCondition = condition.Type(fmt.Sprintf(dataplanev1.ServiceReadyCondition, service)) - readyWaitingMessage = fmt.Sprintf(dataplanev1.ServiceReadyWaitingMessage, deployName) - readyMessage = fmt.Sprintf(dataplanev1.ServiceReadyMessage, deployName) - readyErrorMessage = dataplanev1.ServiceErrorMessage + readyCondition = condition.Type(fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyCondition, nodeSet.Name, service)) + readyWaitingMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyWaitingMessage, nodeSet.Name, deployName) + readyMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentReadyMessage, nodeSet.Name, deployName) + readyErrorMessage = fmt.Sprintf(dataplanev1.NodeSetServiceDeploymentErrorMessage, nodeSet.Name, deployName) aeeSpec.OpenStackAnsibleEERunnerImage = foundService.Spec.OpenStackAnsibleEERunnerImage // Reset ExtraMounts to its original value, and then add in service @@ -97,15 +90,14 @@ func Deploy( err = ConditionalDeploy( ctx, helper, - obj, - sshKeySecret, + nodeSet, + deployment, inventorySecret, status, readyCondition, readyMessage, readyWaitingMessage, readyErrorMessage, - deployFunc, deployName, deployLabel, aeeSpec, @@ -120,6 +112,7 @@ func Deploy( } if err != nil || !status.Conditions.IsTrue(readyCondition) { + log.Info(fmt.Sprintf("Condition %s not ready", readyCondition)) return &ctrl.Result{}, err } log.Info(fmt.Sprintf("Condition %s ready", readyCondition)) @@ -133,15 +126,14 @@ func Deploy( func ConditionalDeploy( ctx context.Context, helper *helper.Helper, - obj client.Object, - sshKeySecret string, + nodeSet *dataplanev1.OpenStackDataPlaneNodeSet, + deployment *dataplanev1.OpenStackDataPlaneDeployment, inventorySecret string, - status *dataplanev1.OpenStackDataPlaneNodeSetStatus, + status *dataplanev1.OpenStackDataPlaneDeploymentStatus, readyCondition condition.Type, readyMessage string, readyWaitingMessage string, readyErrorMessage string, - deployFunc deployFuncDef, deployName string, deployLabel string, aeeSpec dataplanev1.AnsibleEESpec, @@ -153,9 +145,16 @@ func ConditionalDeploy( if status.Conditions.IsUnknown(readyCondition) { log.Info(fmt.Sprintf("%s Unknown, starting %s", readyCondition, deployName)) - err = deployFunc(ctx, helper, obj, sshKeySecret, inventorySecret, aeeSpec, foundService) + err = DeployService( + ctx, + helper, + deployment, + nodeSet.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret, + inventorySecret, + aeeSpec, + foundService) if err != nil { - util.LogErrorForObject(helper, err, fmt.Sprintf("Unable to %s for %s", deployName, obj.GetName()), obj) + util.LogErrorForObject(helper, err, fmt.Sprintf("Unable to %s for %s", deployName, nodeSet.Name), nodeSet) return err } @@ -165,13 +164,12 @@ func ConditionalDeploy( condition.SeverityInfo, readyWaitingMessage)) - log.Info(fmt.Sprintf("Condition %s unknown", readyCondition)) return nil } if status.Conditions.IsFalse(readyCondition) { - ansibleEE, err := dataplaneutil.GetAnsibleExecution(ctx, helper, obj, deployLabel) + ansibleEE, err := dataplaneutil.GetAnsibleExecution(ctx, helper, deployment, deployLabel) if err != nil && k8s_errors.IsNotFound(err) { log.Info(fmt.Sprintf("%s OpenStackAnsibleEE not yet found", readyCondition)) return nil diff --git a/pkg/deployment/ipam.go b/pkg/deployment/ipam.go index 610a279f6..a104f19c3 100644 --- a/pkg/deployment/ipam.go +++ b/pkg/deployment/ipam.go @@ -127,7 +127,7 @@ func EnsureDNSData(ctx context.Context, helper *helper.Helper, allIPSets map[string]infranetworkv1.IPSet) ([]string, []string, string, bool, error) { // Verify dnsmasq CR exists - dnsAddresses, dnsClusterAddresses, isReady, err := checkDNSService( + dnsAddresses, dnsClusterAddresses, isReady, err := CheckDNSService( ctx, helper, instance) if err != nil || !isReady || dnsAddresses == nil { @@ -242,8 +242,8 @@ func reserveIPs(ctx context.Context, helper *helper.Helper, return allIPSets, nil } -// checkDNSService checks if DNS is configured and ready -func checkDNSService(ctx context.Context, helper *helper.Helper, +// CheckDNSService checks if DNS is configured and ready +func CheckDNSService(ctx context.Context, helper *helper.Helper, instance client.Object) ([]string, []string, bool, error) { dnsmasqList := &infranetworkv1.DNSMasqList{} listOpts := []client.ListOption{ diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index 2ecd53feb..89a129ffe 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -21,9 +21,6 @@ func CreateDataplaneNodeSet(name types.NamespacedName, spec dataplanev1.OpenStac func DefaultDataPlaneNodeSetSpec() dataplanev1.OpenStackDataPlaneNodeSetSpec { return dataplanev1.OpenStackDataPlaneNodeSetSpec{ - DeployStrategy: dataplanev1.DeployStrategySection{ - Deploy: false, - }, PreProvisioned: false, NodeTemplate: dataplanev1.NodeTemplate{ AnsibleSSHPrivateKeySecret: "dataplane-ansible-ssh-private-key-secret", @@ -38,9 +35,6 @@ func DefaultDataPlaneNodeSetSpec() dataplanev1.OpenStackDataPlaneNodeSetSpec { func DefaultDataPlaneNoNodeSetSpec() dataplanev1.OpenStackDataPlaneNodeSetSpec { return dataplanev1.OpenStackDataPlaneNodeSetSpec{ - DeployStrategy: dataplanev1.DeployStrategySection{ - Deploy: false, - }, PreProvisioned: true, NodeTemplate: dataplanev1.NodeTemplate{ AnsibleSSHPrivateKeySecret: "dataplane-ansible-ssh-private-key-secret", diff --git a/tests/functional/openstackdataplanenodeset_controller_test.go b/tests/functional/openstackdataplanenodeset_controller_test.go index 13249381e..055268740 100644 --- a/tests/functional/openstackdataplanenodeset_controller_test.go +++ b/tests/functional/openstackdataplanenodeset_controller_test.go @@ -53,10 +53,6 @@ var _ = Describe("Dataplane Role Test", func() { BeforeEach(func() { DeferCleanup(th.DeleteInstance, CreateDataplaneNodeSet(dataplaneNodeSetName, DefaultDataPlaneNoNodeSetSpec())) }) - It("should have the Spec fields initialized", func() { - dataplaneNodeSetInstance := GetDataplaneNodeSet(dataplaneNodeSetName) - Expect(dataplaneNodeSetInstance.Spec.DeployStrategy.Deploy).Should(BeFalse()) - }) It("should have the Status fields initialized", func() { dataplaneNodeSetInstance := GetDataplaneNodeSet(dataplaneNodeSetName) @@ -82,12 +78,6 @@ var _ = Describe("Dataplane Role Test", func() { dataplanev1.SetupReadyCondition, corev1.ConditionFalse, ) - th.ExpectCondition( - dataplaneNodeSetName, - ConditionGetterFunc(DataplaneConditionGetter), - condition.DeploymentReadyCondition, - corev1.ConditionUnknown, - ) }) It("Should not have created a Secret", func() { diff --git a/tests/kuttl/tests/dataplane-create-test/00-assert.yaml b/tests/kuttl/tests/dataplane-create-test/00-assert.yaml index b5a0a9a77..71348bc71 100644 --- a/tests/kuttl/tests/dataplane-create-test/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-create-test/00-assert.yaml @@ -3,8 +3,6 @@ kind: OpenStackDataPlaneNodeSet metadata: name: openstack-edpm spec: - deployStrategy: - deploy: false env: - name: ANSIBLE_FORCE_COLOR value: "True" @@ -115,12 +113,10 @@ status: conditions: - message: Deployment not started reason: NotRequested - severity: Info status: "False" type: Ready - message: Deployment not started reason: NotRequested - severity: Info status: "False" type: DeploymentReady - message: Input data complete diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-assert.yaml index f1e5f51af..bacddef84 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-assert.yaml @@ -4,6 +4,7 @@ metadata: name: edpm-compute-no-nodes namespace: openstack spec: + preProvisioned: true services: - download-cache - configure-network @@ -14,8 +15,6 @@ spec: - ovn - libvirt - nova - deployStrategy: - deploy: false env: - name: ANSIBLE_FORCE_COLOR value: "True" @@ -28,12 +27,10 @@ status: conditions: - message: Deployment not started reason: NotRequested - severity: Info status: "False" type: Ready - message: Deployment not started reason: NotRequested - severity: Info status: "False" type: DeploymentReady - message: Input data complete diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-dataplane-create.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-dataplane-create.yaml index ae258a610..ce92e32a5 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-dataplane-create.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/00-dataplane-create.yaml @@ -28,8 +28,6 @@ spec: - ovn - libvirt - nova - deployStrategy: - deploy: false env: - name: ANSIBLE_FORCE_COLOR value: "True" diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml index bc4071418..a9b0559bf 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-assert.yaml @@ -14,8 +14,6 @@ spec: - ovn - libvirt - nova - deployStrategy: - deploy: true env: - name: ANSIBLE_FORCE_COLOR value: "True" @@ -42,43 +40,6 @@ status: reason: Ready status: "True" type: SetupReady - - message: configure-network service ready - reason: Ready - status: "True" - type: configure-network service ready - - message: configure-os service ready - reason: Ready - status: "True" - type: configure-os service ready - - message: download-cache service ready - reason: Ready - status: "True" - type: download-cache service ready - - message: install-os service ready - reason: Ready - status: "True" - type: install-os service ready - - message: libvirt service ready - reason: Ready - status: "True" - type: libvirt service ready - - message: nova service ready - reason: Ready - status: "True" - type: nova service ready - - message: ovn service ready - reason: Ready - status: "True" - type: ovn service ready - - message: run-os service ready - reason: Ready - status: "True" - type: run-os service ready - - message: validate-network service ready - reason: Ready - status: "True" - type: validate-network service ready - deployed: true --- apiVersion: ansibleee.openstack.org/v1alpha1 kind: OpenStackAnsibleEE @@ -89,7 +50,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -145,7 +106,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -201,7 +162,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -257,7 +218,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -314,7 +275,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -370,7 +331,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -427,7 +388,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 @@ -494,7 +455,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: backoffLimit: 6 diff --git a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-dataplane-deploy.yaml b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-dataplane-deploy.yaml index d68d07b60..0669e5642 100644 --- a/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-dataplane-deploy.yaml +++ b/tests/kuttl/tests/dataplane-deploy-no-nodes-test/01-dataplane-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: dataplane.openstack.org/v1beta1 -kind: OpenStackDataPlaneNodeSet +kind: OpenStackDataPlaneDeployment metadata: name: edpm-compute-no-nodes spec: - deployStrategy: - deploy: true + nodeSets: + - edpm-compute-no-nodes diff --git a/tests/kuttl/tests/dataplane-extramounts/00-assert.yaml b/tests/kuttl/tests/dataplane-extramounts/00-assert.yaml index 980f96137..e2fdb5f63 100644 --- a/tests/kuttl/tests/dataplane-extramounts/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-extramounts/00-assert.yaml @@ -5,8 +5,6 @@ metadata: name: edpm-extramounts spec: preProvisioned: true - deployStrategy: - deploy: true services: - test-service nodes: {} @@ -31,7 +29,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-extramounts spec: extraMounts: diff --git a/tests/kuttl/tests/dataplane-extramounts/00-dataplane-create.yaml b/tests/kuttl/tests/dataplane-extramounts/00-dataplane-create.yaml index b35c6f4f5..9b0d25549 100644 --- a/tests/kuttl/tests/dataplane-extramounts/00-dataplane-create.yaml +++ b/tests/kuttl/tests/dataplane-extramounts/00-dataplane-create.yaml @@ -29,3 +29,11 @@ spec: persistentVolumeClaim: claimName: edpm-ansible readOnly: true +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-extramounts +spec: + nodeSets: + - edpm-extramounts diff --git a/tests/kuttl/tests/dataplane-service-config/00-assert.yaml b/tests/kuttl/tests/dataplane-service-config/00-assert.yaml index f9f5c6fbb..e376880fd 100644 --- a/tests/kuttl/tests/dataplane-service-config/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-service-config/00-assert.yaml @@ -8,7 +8,7 @@ metadata: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet + kind: OpenStackDataPlaneDeployment name: edpm-compute-no-nodes spec: debug: false diff --git a/tests/kuttl/tests/dataplane-service-config/00-create.yaml b/tests/kuttl/tests/dataplane-service-config/00-create.yaml index a26105d3b..b75dbf3d9 100644 --- a/tests/kuttl/tests/dataplane-service-config/00-create.yaml +++ b/tests/kuttl/tests/dataplane-service-config/00-create.yaml @@ -51,8 +51,6 @@ metadata: name: edpm-compute-no-nodes spec: preProvisioned: true - deployStrategy: - deploy: true env: - name: ANSIBLE_FORCE_COLOR value: "True" @@ -63,3 +61,11 @@ spec: ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret services: - kuttl-service +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-compute-no-nodes +spec: + nodeSets: + - edpm-compute-no-nodes diff --git a/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml b/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml index 3ea441d5a..7c6418de0 100644 --- a/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml +++ b/tests/kuttl/tests/dataplane-service-custom-image/00-assert.yaml @@ -5,8 +5,6 @@ metadata: name: edpm-no-nodes-custom-service spec: preProvisioned: true - deployStrategy: - deploy: true services: - custom-image-service nodes: {} @@ -17,14 +15,12 @@ spec: ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret status: conditions: - - message: custom-image-service service not yet ready - reason: Requested - severity: Info + - message: Deployment not started + reason: NotRequested status: "False" type: Ready - - message: Deployment in progress - reason: Requested - severity: Info + - message: Deployment not started + reason: NotRequested status: "False" type: DeploymentReady - message: Input data complete @@ -35,23 +31,18 @@ status: reason: Ready status: "True" type: SetupReady - - message: custom-image-service service not yet ready - reason: Requested - severity: Info - status: "False" - type: custom-image-service service ready --- apiVersion: ansibleee.openstack.org/v1alpha1 kind: OpenStackAnsibleEE metadata: - name: dp-custom-image-service-edpm-no-nodes-custom-service + name: dp-custom-image-service-edpm-compute-no-nodes namespace: openstack ownerReferences: - apiVersion: dataplane.openstack.org/v1beta1 blockOwnerDeletion: true controller: true - kind: OpenStackDataPlaneNodeSet - name: edpm-no-nodes-custom-service + kind: OpenStackDataPlaneDeployment + name: edpm-compute-no-nodes spec: backoffLimit: 6 extraMounts: diff --git a/tests/kuttl/tests/dataplane-service-custom-image/00-dataplane-create.yaml b/tests/kuttl/tests/dataplane-service-custom-image/00-dataplane-create.yaml index e08176179..5fe844e61 100644 --- a/tests/kuttl/tests/dataplane-service-custom-image/00-dataplane-create.yaml +++ b/tests/kuttl/tests/dataplane-service-custom-image/00-dataplane-create.yaml @@ -20,8 +20,6 @@ metadata: name: edpm-no-nodes-custom-service spec: preProvisioned: true - deployStrategy: - deploy: true services: - custom-image-service nodes: {} @@ -30,3 +28,11 @@ spec: ansibleUser: cloud-admin ansiblePort: 22 ansibleSSHPrivateKeySecret: dataplane-ansible-ssh-private-key-secret +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneDeployment +metadata: + name: edpm-compute-no-nodes +spec: + nodeSets: + - edpm-no-nodes-custom-service From 4547ac02d16cf1c989be6576306357ce665401ed Mon Sep 17 00:00:00 2001 From: James Slagle Date: Fri, 15 Sep 2023 12:42:03 -0400 Subject: [PATCH 2/2] Add Watch to NodeSet for Deployments Instead of the OpenStackDataPlaneDeployment controller updating the status of OpenStackDataPlaneNodeSet after a deployment, use a watch instead. The watch is added for the OpenStackDataPlaneNodeSet reconciler and it when a reconcile is triggerd, the deployment condition will be set to True if there any associated deployed OpenStackDataPlaneDeployments. Signed-off-by: James Slagle --- ...openstackdataplanedeployment_controller.go | 11 --- .../openstackdataplanenodeset_controller.go | 77 +++++++++++++++++++ 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go index 1f3e988cd..fd54f89f7 100644 --- a/controllers/openstackdataplanedeployment_controller.go +++ b/controllers/openstackdataplanedeployment_controller.go @@ -28,7 +28,6 @@ import ( "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "github.com/go-logr/logr" @@ -235,16 +234,6 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context, condition.TrueCondition( condition.Type(fmt.Sprintf(dataplanev1.NodeSetDeploymentReadyCondition, nodeSet.Name)), condition.DeploymentReadyMessage)) - - logger.Info("Set NodeSet DeploymentReadyCondition true", "nodeSet", nodeSet.Name) - _, err = controllerutil.CreateOrPatch(ctx, helper.GetClient(), &nodeSet, func() error { - nodeSet.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) - return nil - }) - if err != nil { - util.LogErrorForObject(helper, err, "Unable to set Status on NodeSet", &nodeSet) - return ctrl.Result{}, err - } } } diff --git a/controllers/openstackdataplanenodeset_controller.go b/controllers/openstackdataplanenodeset_controller.go index dabf81850..274470b6e 100644 --- a/controllers/openstackdataplanenodeset_controller.go +++ b/controllers/openstackdataplanenodeset_controller.go @@ -259,9 +259,65 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req instance.Status.Conditions.Set(condition.FalseCondition(condition.DeploymentReadyCondition, condition.NotRequestedReason, condition.SeverityInfo, condition.DeploymentReadyInitMessage)) } + deployedDeploymentsForNodeSet, err := r.GetDeployedDeploymentsForNodeSet(instance.Name) + if err != nil { + logger.Error(err, "Unable to get deployed OpenStackDataPlaneDeployments.") + return ctrl.Result{}, err + } + if len(deployedDeploymentsForNodeSet.Items) > 0 { + logger.Info("Set NodeSet DeploymentReadyCondition true") + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } + return ctrl.Result{}, nil } +// GetDeployedDeploymentsForNodeSet - Get the OpenStackDataPlaneDeployment +// resources that have been deployed and are for the given +// OpenStackDataPlaneNodeSet. +func (r *OpenStackDataPlaneNodeSetReconciler) GetDeployedDeploymentsForNodeSet(nodeSetName string) (*dataplanev1.OpenStackDataPlaneDeploymentList, error) { + // Get all deployments + deployments := &dataplanev1.OpenStackDataPlaneDeploymentList{} + err := r.Client.List(context.Background(), deployments) + if err != nil { + r.Log.Error(err, "Unable to retrieve OpenStackDataPlaneDeployment CRs %v") + return deployments, nil + } + deployedDeploymentsForNodeSet := &dataplanev1.OpenStackDataPlaneDeploymentList{} + for _, deployment := range deployments.Items { + for _, nodeSet := range deployment.Spec.NodeSets { + if nodeSet == nodeSetName { + if deployment.Status.Conditions.IsTrue(condition.Type(fmt.Sprintf(dataplanev1.NodeSetDeploymentReadyCondition, nodeSetName))) { + deployedDeploymentsForNodeSet.Items = append(deployedDeploymentsForNodeSet.Items, deployment) + } + } + } + } + + return deployedDeploymentsForNodeSet, err +} + +// GetNodeSetsForDeployment - Get the OpenStackDataPlaneNodeSet +// resources for the given OpenStackDataPlaneDeployment +func (r *OpenStackDataPlaneNodeSetReconciler) GetNodeSetsForDeployment(namespace string, deployment *dataplanev1.OpenStackDataPlaneDeployment) (dataplanev1.OpenStackDataPlaneNodeSetList, error) { + nodeSets := dataplanev1.OpenStackDataPlaneNodeSetList{} + var err error + for _, nodeSetName := range deployment.Spec.NodeSets { + nodeSet := &dataplanev1.OpenStackDataPlaneNodeSet{} + namespacedName := client.ObjectKey{ + Namespace: namespace, + Name: nodeSetName} + err = r.Client.Get(context.Background(), namespacedName, nodeSet) + if err != nil { + r.Log.Error(err, "Unable to retrieve OpenStackDataPlaneNodeSet CR %v") + return nodeSets, nil + } + nodeSets.Items = append(nodeSets.Items, *nodeSet) + } + + return nodeSets, err +} + // SetupWithManager sets up the controller with the Manager. func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(mgr ctrl.Manager) error { reconcileFunction := handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { @@ -296,6 +352,25 @@ func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(mgr ctrl.Manager) return nil }) + deploymentWatcher := handler.EnqueueRequestsFromMapFunc(func(obj client.Object) []reconcile.Request { + var namespace string = obj.GetNamespace() + result := []reconcile.Request{} + + deployment := obj.(*dataplanev1.OpenStackDataPlaneDeployment) + nodeSets, err := r.GetNodeSetsForDeployment(namespace, deployment) + if err != nil { + r.Log.Error(err, "Unable to retrieve OpenStackDataPlaneNodeSets %w") + return nil + } + for _, nodeSet := range nodeSets.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: nodeSet.Name} + result = append(result, reconcile.Request{NamespacedName: name}) + } + return result + }) + return ctrl.NewControllerManagedBy(mgr). For(&dataplanev1.OpenStackDataPlaneNodeSet{}). Owns(&v1alpha1.OpenStackAnsibleEE{}). @@ -305,5 +380,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(mgr ctrl.Manager) Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &infranetworkv1.DNSMasq{}}, reconcileFunction). + Watches(&source.Kind{Type: &dataplanev1.OpenStackDataPlaneDeployment{}}, + deploymentWatcher). Complete(r) }