From c17b0a5067f0fa83d900c799c2cb18c6bdecdf49 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Thu, 7 Sep 2023 10:49:51 +0100 Subject: [PATCH 1/2] Update docs Signed-off-by: Fabricio Aguiar --- PROJECT | 24 +-------- docs/deploying.md | 88 +++++++------------------------- docs/design.md | 5 +- docs/diagrams/out/edpm.svg | 2 +- docs/diagrams/src/edpm.puml | 10 +--- docs/inheritance.md | 82 ++--------------------------- docs/interacting_with_ansible.md | 2 +- mkdocs.yml | 6 +-- 8 files changed, 31 insertions(+), 188 deletions(-) diff --git a/PROJECT b/PROJECT index 543b4c22b..b06483daf 100644 --- a/PROJECT +++ b/PROJECT @@ -13,29 +13,7 @@ resources: controller: true domain: openstack.org group: dataplane - kind: OpenStackDataPlane - path: github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1 - version: v1beta1 - webhooks: - defaulting: true - validation: true - webhookVersion: v1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: openstack.org - group: dataplane - kind: OpenStackDataPlaneRole - path: github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1 - version: v1beta1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: openstack.org - group: dataplane - kind: OpenStackDataPlaneNode + kind: OpenStackDataPlaneNodeSet path: github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1 version: v1beta1 - api: diff --git a/docs/deploying.md b/docs/deploying.md index 19601b3ed..d4b400852 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -1,11 +1,10 @@ -# Deploying a DataPlane +# Deploying a DataPlaneNodeSet -Deploying a dataplane consists of creating the custom resources (whether -OpenStackDataPlane, OpenStackDataPlaneRole, or OpenStackDataPlaneNode) that +Deploying a dataplane consists of creating the OpenStackDataPlaneNodeSet custom resource that define the layout of the dataplane. This documentation will cover using each resource individually, as well as -using the OpenStackDataPlane resource to deploy everything in a single +using the OpenStackDataPlaneNodeSet resource to deploy everything in a single resource. ## Samples @@ -73,17 +72,17 @@ Verify the secret was created: oc describe secret dataplane-ansible-ssh-private-key-secret -### Create OpenStackDataPlane +### Create OpenStackDataPlaneNodeSet This document will cover writing the `YAML` document for an -`OpenStackDataPlane` resource. Once the document is ready, it will be created +`OpenStackDataPlaneNodeSet` resource. Once the document is ready, it will be created with `oc` as the last step. Start the `YAML` document in an `openstack-edpm.yaml` file and give the dataplane a name. apiVersion: dataplane.openstack.org/v1beta1 - kind: OpenStackDataPlane + kind: OpenStackDataPlaneNodeSet metadata: name: openstack-edpm @@ -92,7 +91,7 @@ added to the spec that contains `deploy: false`. This allows for creating the dataplane resources without triggering an Ansible execution immediately. apiVersion: dataplane.openstack.org/v1beta1 - kind: OpenStackDataPlane + kind: OpenStackDataPlaneNodeSet metadata: name: openstack-edpm spec: @@ -109,7 +108,7 @@ inheritance works. Within `nodeTemplate`, the fields shown are documented inline in the example. apiVersion: dataplane.openstack.org/v1beta1 - kind: OpenStackDataPlane + kind: OpenStackDataPlaneNodeSet metadata: name: openstack-edpm spec: @@ -135,7 +134,7 @@ inline in the example. # These vars are edpm_network_config role vars edpm_network_config_template: templates/single_nic_vlans/single_nic_vlans.j2 - # See config/samples/dataplane_v1beta1_openstackdataplane.yaml + # See config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml # for the other most common ansible varialbes that need to be set. The list of ansible variables that can be set under `ansibleVars` is extensive. @@ -192,7 +191,7 @@ With the nodes and the controlplane specific variables added, the full `openstack-datplane` `YAML` document looks like the following: apiVersion: dataplane.openstack.org/v1beta1 - kind: OpenStackDataPlane + kind: OpenStackDataPlaneNodeSet metadata: name: openstack-edpm spec: @@ -227,7 +226,7 @@ With the nodes and the controlplane specific variables added, the full edpm_ovn_dbs: - 192.168.24.1 - # See config/samples/dataplane_v1beta1_openstackdataplane.yaml + # See config/samples/dataplane_v1beta1_openstackdataplanenodeset.yaml # for the other most common ansible varialbes that need to be set. nodes: @@ -258,25 +257,16 @@ Create the dataplane using the `oc` command. oc create -f openstack-edpm.yaml -Verify that the dataplane, role, and nodes were created. +Verify that the dataplane nodeset were created. - oc get openstackdataplane - oc get openstackdataplanerole - oc get openstackdataplanenode + oc get openstackdataplanenodeset The output should be similar to: ```console -$ oc get openstackdataplane +$ oc get openstackdataplanenodeset NAME STATUS MESSAGE openstack-edpm False Deployment not started -$ oc get openstackdataplanerole -NAME STATUS MESSAGE -edpm-compute False Deployment not started -$ oc get openstackdataplanenode -NAME STATUS MESSAGE -edpm-compute-0 False Deployment not started -edpm-compute-1 False Deployment not started ``` ### Understanding OpenStackDataPlaneServices @@ -364,10 +354,10 @@ To deploy the `openstack-edpm` dataplane resource, the the deployment of all the configured services across the nodes. The field can be set with the following command to start the deployment: - oc patch openstackdataplane openstack-edpm -p='[{"op": "replace", "path": "/spec/deployStrategy/deploy", "value":true}]' --type json + oc patch openstackdataplanenodeset openstack-edpm -p='[{"op": "replace", "path": "/spec/deployStrategy/deploy", "value":true}]' --type json The `oc patch` command sets the `deploy` field to `True`, which starts the -deployment. `oc edit openstackdataplane openstack-edpm` could alternatively be +deployment. `oc edit OpenStackDataPlaneNodeSet openstack-edpm` could alternatively be used to edit the resource directly in an editor to set the field to `True`. With the deployment started, ansible will be executed to configure the nodes. @@ -375,16 +365,9 @@ When the deployment is complete, the status messages will change to indicate the deployment is ready. ```console -$ oc get openstackdataplane +$ oc get openstackdataplanenodeset NAME STATUS MESSAGE openstack-edpm True DataPlane Ready -$ oc get openstackdataplanerole -NAME STATUS MESSAGE -edpm-compute True DataPlaneRole Ready -$ oc get openstackdataplanenode -NAME STATUS MESSAGE -edpm-compute-0 True DataPlaneNode Ready -edpm-compute-1 True DataPlaneNode Ready ``` If the deployment involved adding new compute nodes then after the deployment @@ -400,46 +383,13 @@ Each dataplane resource has a series of conditions within their `status` subresource that indicate the overall state of the resource, including its deployment progress. -`OpenStackDataPlane` resource conditions: - -```console -$ oc get openstackdataplane openstack-edpm -o json | jq .status.conditions[].type -"Ready" -"DeploymentReady" -"SetupReady" -``` - -`OpenStackDataPlaneRole` resource conditions: - -```console -$ oc get openstackdataplanerole edpm-compute -o json | jq .status.conditions[].type -"Ready" -"DeploymentReady" -"RoleBaremetalProvisionReady" -"SetupReady" -"configure-network service ready" -"configure-os service ready" -"install-os service ready" -"libvirt service ready" -"nova service ready" -"run-os service ready" -"validate-network service ready" -``` - -`OpenStackDataPlaneNode` resource conditions: +`OpenStackDataPlaneNodeSet` resource conditions: ```console -$ oc get openstackdataplanenode edpm-compute-0 -o json | jq .status.conditions[].type +$ oc get openstackdataplanenodeset openstack-edpm -o json | jq .status.conditions[].type "Ready" "DeploymentReady" "SetupReady" -"configure-network service ready" -"configure-os service ready" -"install-os service ready" -"install-os service ready" -"libvirt service ready" -"run-os service ready" -"validate-network service ready" ``` Each resource has a `Ready`, `DeploymentReady`, and `SetupReady` conditions. diff --git a/docs/design.md b/docs/design.md index a55b9e785..269553531 100644 --- a/docs/design.md +++ b/docs/design.md @@ -10,17 +10,16 @@ software on the nodes. ## DataPlane Design and Resources -The dataplane-operator exposes the concepts of dataplanes, roles, nodes, and +The dataplane-operator exposes the concepts of nodesets and services as CRD's: -* [OpenStackDataPlane](https://github.com/openstack-k8s-operators/dataplane-operator/blob/main/config/crd/bases/dataplane.openstack.org_openstackdataplanes.yaml) * [OpenStackDataPlaneNodeSet](https://github.com/openstack-k8s-operators/dataplane-operator/blob/main/config/crd/bases/dataplane.openstack.org_openstackdataplanenodesets.yaml) * [OpenStackDataPlaneService](https://github.com/openstack-k8s-operators/dataplane-operator/blob/main/config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml) Each `NodeSet` in a dataplane is represented by a corresponding OpenStackDataPlaneNodeSet resource. The OpenStackDataPlaneNodeSet CRD provides for a logical grouping of nodes of a similar type. This is analogous to the concept of "roles" -in TripleO. Similarities within a `NodeSet` are defined by the user, and could be of a +in TripleO. Similarities within a `NodeSet` are defined by the user, and could be of a small scope (ansible port), or a large scope (same network config, nova config, provisioning config, etc). The properties that all nodes in a `NodeSet` share is set in the NodeTemplate field of the `NodeSet`'s Spec. Node specific parameters are then diff --git a/docs/diagrams/out/edpm.svg b/docs/diagrams/out/edpm.svg index 19f3c20d5..ac899db84 100644 --- a/docs/diagrams/out/edpm.svg +++ b/docs/diagrams/out/edpm.svg @@ -1 +1 @@ -Human OperatorOpenStack Control Plane CROpenStack Data Plane CROpenStack Data Plane Node CROpenStack Data Plane Role CROpenStack AnsibleEE CROpenStack OperatorDataplane OperatorAnsibleEE OperatorAnsible Job - InfraAnsible Job - Nova ConfigAEEconfigmapappliesappliesreconcilesreconcilescreatescreatesreconcilesreconcilescreatesreconcilesyieldsyieldsmounts \ No newline at end of file +Human OperatorOpenStack Control Plane CROpenStack Data Plane NodeSet CROpenStack AnsibleEE CROpenStack OperatorDataplane OperatorAnsibleEE OperatorAnsible Job - InfraAnsible Job - Nova ConfigAEEconfigmapappliesappliesreconcilesreconcilescreatesreconcilesyieldsyieldsmounts \ No newline at end of file diff --git a/docs/diagrams/src/edpm.puml b/docs/diagrams/src/edpm.puml index 4470bdd73..9de294381 100644 --- a/docs/diagrams/src/edpm.puml +++ b/docs/diagrams/src/edpm.puml @@ -2,9 +2,7 @@ actor "Human Operator" as user #blue;line:blue;line.bold;text:blue file "OpenStack Control Plane CR" as OCPCR #pink;line:red;line.bold;text:red -file "OpenStack Data Plane CR" as ODPCR #pink;line:red;line.bold;text:red -file "OpenStack Data Plane Node CR" as ODPNCR #line:red;line.bold;text:red -file "OpenStack Data Plane Role CR" as ODPRCR #line:red;line.bold;text:red +file "OpenStack Data Plane NodeSet CR" as ODPCR #pink;line:red;line.bold;text:red file "OpenStack AnsibleEE CR" as AEECR #line:red;line.bold;text:red rectangle "OpenStack Operator" as OSOp #line.bold @@ -19,11 +17,7 @@ folder AEEconfigmap #line.bold user --> OCPCR #line:red;line.bold;text:red : applies user --> ODPCR #line:red;line.bold;text:red : applies OCPCR <-- OSOp #line:brown;line.bold;text:brown : reconciles -ODPCR <-- OSOp #line:brown;line.bold;text:brown : reconciles -OSOp --> ODPNCR #line:purple;line.bold;text:purple : creates -OSOp --> ODPRCR #line:purple;line.bold;text:purple : creates -ODPNCR <-- DOp #line:brown;line.bold;text:brown : reconciles -ODPRCR <-- DOp #line:brown;line.bold;text:brown : reconciles +ODPCR <-- DOp #line:brown;line.bold;text:brown : reconciles DOp --> AEECR #line:purple;line.bold;text:purple : creates AEECR <-- AEEOp #line:brown;line.bold;text:brown : reconciles AEEOp --> AJI #line:blue;line.bold;text:blue : yields diff --git a/docs/inheritance.md b/docs/inheritance.md index f56087026..7a80acabb 100644 --- a/docs/inheritance.md +++ b/docs/inheritance.md @@ -1,7 +1,7 @@ # Inheritance -An `OpenStackDataPlaneNode` inherits any attribute of an -`OpenStackDataPlaneRole` but those attributes may also be overridden +An `node` inherits any attribute of an +`nodeTemplate` but those attributes may also be overridden on the node level. Suppose the following CR is created with `oc create -f @@ -38,46 +38,6 @@ spec: template: templates/net_config_bridge.j2 ``` -Then two CRs like the following, representing two nodes, will be -created automatically by the dataplane-operator and there is no -need to `oc create` separate files containing the following. - -Node 1 - -```yaml ---- -apiVersion: dataplane.openstack.org/v1beta1 -kind: OpenStackDataPlaneNode -metadata: - name: openstackdataplanenode-sample-1 -spec: - ansibleHost: 192.168.122.18 - hostName: openstackdataplanenode-sample-1.localdomain - node: - networks: - - fixedIP: 192.168.122.18 - network: ctlplane - role: openstackdataplanerole-sample -``` - -Node 2 - -```yaml ---- -apiVersion: dataplane.openstack.org/v1beta1 -kind: OpenStackDataPlaneNode -metadata: - name: openstackdataplanenode-sample-2 -spec: - ansibleHost: 192.168.122.19 - hostName: openstackdataplanenode-sample-2.localdomain - node: - networks: - - fixedIP: 192.168.122.19 - network: ctlplane - role: openstackdataplanerole-sample -``` - Because of inheritance, redundant information did not need to be provided to both nodes. Only the information which differed per node, e.g. `ansibleHost`, had to be specified. Furthermore, the redundant @@ -97,17 +57,8 @@ However, it's unambiguous that each node has `ansiblePort` 22 because they have `role: openstackdataplanerole-sample`. If the node is inspected however, port 22 will be set. -The node controller resolves this dynamically by looking at -`role.nodeSpec` and we can assume that the value was inherited. -If we need to override a setting for any single node so that it -doesn't use it's `nodeTemplate`, then we may do so by directly -updating only that node (e.g. with `oc edit`). In that case we'd see -an `ansiblePort` set directly in that node's CR. This allows the user -to change the `nodeTemplate` after creation and once reconciliation is -completed all existing nodes will inherit the new value. - Almost any top level property in a node overrides the whole property -in a role. E.g. if the role `nodeTemplate` had a list like the +in a nodeTemplate. E.g. if the nodeset `nodeTemplate` had a list like the following: ``` @@ -169,30 +120,3 @@ had a `edpm_chrony_ntp_servers` list with `clock3.redhat.com`, then the resultant inventory for the node would not have three NTP servers; it would only have `clock3.redhat.com`. I.e. there is no "deep merging". - -It's also possible to create a node directly outside of a role CR -and define its role. If the following CR is created: - -```yaml ---- -apiVersion: dataplane.openstack.org/v1beta1 -kind: OpenStackDataPlaneNode -metadata: - name: openstackdataplanenode-sample-3-from -spec: - role: openstackdataplanerole-sample - hostName: openstackdataplanenode-sample-3.localdomain - ansibleHost: 192.168.122.20 - node: - networks: - - network: ctlplane - fixedIP: 192.168.122.20 -``` - -After the above CR is created, the node -openstackdataplanenode-sample-3-from may then be inspected further -using a command like -`oc get OpenStackDataPlaneNode openstackdataplanenode-sample-3-from -o -yaml` which should show that it inherited values from the role -`nodeTemplate`. In cases like these, the `dataPlaneNodes` list will -not reflect all of the nodes within the role. diff --git a/docs/interacting_with_ansible.md b/docs/interacting_with_ansible.md index a083fdc7e..b1029ca58 100644 --- a/docs/interacting_with_ansible.md +++ b/docs/interacting_with_ansible.md @@ -89,7 +89,7 @@ fields. Example usage of these fields: apiVersion: dataplane.openstack.org/v1beta1 - kind: OpenStackDataPlane + kind: OpenStackDataPlaneNodeSet metadata: name: openstack-edpm spec: diff --git a/mkdocs.yml b/mkdocs.yml index 96582bba1..1ec55c319 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,14 +27,12 @@ theme: nav: - Home: index.md - Custom Resources: - - OpenStackDataPlane: openstack_dataplane.md - - OpenStackDataPlaneNode: openstack_dataplanenode.md - - OpenStackDataPlaneRole: openstack_dataplanerole.md + - OpenStackDataPlaneNodeSet: openstack_dataplanenodeset.md - OpenStackDataPlaneService: openstack_dataplaneservice.md - Architecture: architecture.md - Design: design.md - Inheritance: inheritance.md - - Deploying a DataPlane: + - Deploying a DataPlaneNodeSet: - deploying.md - composable_services.md - common_configurations.md From 115eeb804faaa2e1a093ee3576c14179e004f75a Mon Sep 17 00:00:00 2001 From: rabi Date: Thu, 7 Sep 2023 20:06:10 +0530 Subject: [PATCH 2/2] Fix Regression with IPAM We should use Spec.NodeTemplate.Networks if there are no node.Networks. Also, remove the block to remove ctlplane network. Fixes the sample used in CI. - ansible_user: cloud-admin - Remove all ansible vars for network ips - Remove BMH namespace from the sample as it is also used with CBO. We patch it in install_yamls for CI. --- ...kdataplanenodeset_baremetal_with_ipam.yaml | 58 ++++--------------- pkg/deployment/ipam.go | 14 ++--- 2 files changed, 17 insertions(+), 55 deletions(-) diff --git a/config/samples/dataplane_v1beta1_openstackdataplanenodeset_baremetal_with_ipam.yaml b/config/samples/dataplane_v1beta1_openstackdataplanenodeset_baremetal_with_ipam.yaml index 87eb576af..0c5de8e37 100644 --- a/config/samples/dataplane_v1beta1_openstackdataplanenodeset_baremetal_with_ipam.yaml +++ b/config/samples/dataplane_v1beta1_openstackdataplanenodeset_baremetal_with_ipam.yaml @@ -20,7 +20,6 @@ spec: deployStrategy: deploy: true baremetalSetTemplate: - bmhNamespace: openstack deploymentSSHSecret: dataplane-ansible-ssh-private-key-secret bmhLabelSelector: app: openstack @@ -33,29 +32,19 @@ spec: nodes: edpm-compute-0: hostName: edpm-compute-0 - ansible: - ansibleUser: root - ansibleHost: 192.168.122.100 - ansibleVars: - ctlplane_ip: 192.168.122.100 - internal_api_ip: 172.17.0.100 - storage_ip: 172.18.0.100 - tenant_ip: 172.19.0.100 - fqdn_internal_api: edpm-compute-0.example.com - networks: - - name: CtlPlane - subnetName: subnet1 - defaultRoute: true - fixedIP: 192.168.122.100 - - name: InternalApi - subnetName: subnet1 - - name: Storage - subnetName: subnet1 - - name: Tenant - subnetName: subnet1 + networks: + - name: CtlPlane + subnetName: subnet1 + defaultRoute: true + - name: InternalApi + subnetName: subnet1 + - name: Storage + subnetName: subnet1 + - name: Tenant + subnetName: subnet1 managementNetwork: ctlplane ansible: - ansibleUser: root + ansibleUser: cloud-admin ansiblePort: 22 ansibleVars: service_net_map: @@ -72,28 +61,6 @@ spec: # considered EDPM network defaults. neutron_physical_bridge_name: br-ex neutron_public_interface_name: eth0 - ctlplane_mtu: 1500 - ctlplane_subnet_cidr: 24 - ctlplane_gateway_ip: 192.168.122.1 - ctlplane_host_routes: - - ip_netmask: 0.0.0.0/0 - next_hop: 192.168.122.1 - external_mtu: 1500 - external_vlan_id: 44 - external_cidr: '24' - external_host_routes: [] - internal_api_mtu: 1500 - internal_api_vlan_id: 20 - internal_api_cidr: '24' - internal_api_host_routes: [] - storage_mtu: 1500 - storage_vlan_id: 21 - storage_cidr: '24' - storage_host_routes: [] - tenant_mtu: 1500 - tenant_vlan_id: 22 - tenant_cidr: '24' - tenant_host_routes: [] role_networks: - InternalApi - Storage @@ -111,9 +78,6 @@ spec: edpm_ovn_metadata_agent_metadata_agent_DEFAULT_nova_metadata_host: 127.0.0.1 edpm_ovn_metadata_agent_metadata_agent_DEFAULT_metadata_proxy_shared_secret: 12345678 edpm_ovn_metadata_agent_DEFAULT_bind_host: 127.0.0.1 - ctlplane_dns_nameservers: - - 192.168.122.1 - dns_search_domains: [] edpm_ovn_dbs: - 192.168.122.1 registry_url: quay.io/podified-antelope-centos9 diff --git a/pkg/deployment/ipam.go b/pkg/deployment/ipam.go index 602cec5fb..eb945cab5 100644 --- a/pkg/deployment/ipam.go +++ b/pkg/deployment/ipam.go @@ -73,7 +73,9 @@ func createOrPatchDNSData(ctx context.Context, helper *helper.Helper, // Build DNSData CR for nodeName, node := range instance.Spec.NodeTemplate.Nodes { nets := node.Networks - + if len(nets) == 0 { + nets = instance.Spec.NodeTemplate.Networks + } if len(nets) > 0 { // Get IPSet ipSet, ok := allIPSets[nodeName] @@ -211,14 +213,10 @@ func reserveIPs(ctx context.Context, helper *helper.Helper, for nodeName, node := range instance.Spec.NodeTemplate.Nodes { nets := node.Networks - if instance.Spec.PreProvisioned { - // Drop CtlPlaneNetwork - for i, v := range nets { - if v.Name == CtlPlaneNetwork { - nets = append(nets[:i], nets[i+1:]...) - } - } + if len(nets) == 0 { + nets = instance.Spec.NodeTemplate.Networks } + if len(nets) > 0 { util.LogForObject(helper, "Reconciling IPSet", instance) ipSet := &infranetworkv1.IPSet{