From b2ec362d338c0e5029d00950903768e9b5099d63 Mon Sep 17 00:00:00 2001 From: Miguel Angel Nieto Jimenez Date: Tue, 20 Aug 2024 12:52:43 +0200 Subject: [PATCH] Multiple nodesets for ovs-dpdk-sriov scenario Created a new dt based on ovs-dpdk-sriov va --- automation/vars/ovs-dpdk-sriov-2nodesets.yaml | 61 +++++ .../nfv-ovs-dpdk-sriov-2nodesets/.gitignore | 2 + .../nfv-ovs-dpdk-sriov-2nodesets/README.md | 26 ++ .../control-plane.md | 55 +++++ .../nfv-ovs-dpdk-sriov-2nodesets/dataplane.md | 54 +++++ .../edpm/.gitignore | 3 + .../edpm/deployment/kustomization.yaml | 21 ++ .../edpm/deployment/values.yaml | 10 + .../edpm/nodeset/kustomization.yaml | 12 + .../edpm/nodeset/values.yaml | 228 ++++++++++++++++++ .../edpm/nodeset2/kustomization.yaml | 51 ++++ .../edpm/nodeset2/values.yaml | 227 +++++++++++++++++ .../kustomization.yaml | 13 + .../nncp/kustomization.yaml | 24 ++ .../nncp/values.yaml | 199 +++++++++++++++ .../service-values.yaml | 80 ++++++ zuul.d/projects.yaml | 1 + zuul.d/validations.yaml | 12 + 18 files changed, 1079 insertions(+) create mode 100644 automation/vars/ovs-dpdk-sriov-2nodesets.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/.gitignore create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/README.md create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/control-plane.md create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/dataplane.md create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/.gitignore create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/kustomization.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/values.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/kustomization.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/values.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/kustomization.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/values.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/kustomization.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/kustomization.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/values.yaml create mode 100644 examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/service-values.yaml diff --git a/automation/vars/ovs-dpdk-sriov-2nodesets.yaml b/automation/vars/ovs-dpdk-sriov-2nodesets.yaml new file mode 100644 index 000000000..8f0971198 --- /dev/null +++ b/automation/vars/ovs-dpdk-sriov-2nodesets.yaml @@ -0,0 +1,61 @@ +--- +vas: + ovs-dpdk-sriov-2nodesets: + stages: + - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp + wait_conditions: + - >- + oc -n openstack wait nncp + -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=60s + values: + - name: network-values + src_file: values.yaml + build_output: nncp.yaml + + - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets + wait_conditions: + - >- + oc -n openstack wait osctlplane controlplane --for condition=Ready + --timeout=1200s + values: + - name: network-values + src_file: ./nncp/values.yaml + - name: service-values + src_file: service-values.yaml + build_output: control-plane.yaml + + - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset + wait_conditions: + - >- + oc -n openstack wait + osdpns openstack-edpm --for condition=SetupReady + --timeout=60m + values: + - name: edpm-nodeset-values + src_file: values.yaml + build_output: nodeset.yaml + + - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2 + wait_conditions: + - >- + oc -n openstack wait + osdpns openstack-edpm-2 --for condition=SetupReady + --timeout=60m + values: + - name: edpm-nodeset2-values + src_file: values.yaml + build_output: nodeset2.yaml + + - path: examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment + wait_conditions: + - >- + oc -n openstack + wait openstackdataplanedeployments.dataplane.openstack.org + edpm-deployment + --for condition=Ready --timeout=60m + values: + - name: edpm-deployment-values + src_file: values.yaml + build_output: deployment.yaml diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/.gitignore b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/.gitignore new file mode 100644 index 000000000..db8e5d02b --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/.gitignore @@ -0,0 +1,2 @@ +nncp.yaml +control-plane.yaml diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/README.md b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/README.md new file mode 100644 index 000000000..ceac82da5 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/README.md @@ -0,0 +1,26 @@ +# Deployed Topology - NFV/OVS-DPDK-SRIOV with 2 nodesets + +Deploys an OVS-DPDK-SRIOV environment 2 different nodesets. It is an extension +of the regular OVS-DPDK-SRIOV scenario, so all information in that scenario is +valid here too. Check [OpenStack OVS DPDK SRIOV](../../../va/nfv/ovs-dpdk-sriov/README.md) + +## Purpose + +This scenario is needed when compute nodes used to deploy Openstack dataplane +are different (different nics, cpu, memory, ...) so a different nodeset can +be created for each different compute node + +This DT is based on OVS-DPDK-SRIOV VA and it has been added just the code +to configure the second nodeset, reusing all of the other config + +## Stages + +All stages must be executed in the order listed below. Everything is required unless otherwise indicated. +Stages are the same than the OVS-DPDK-SRIOV scenario, the only difference is that it will +be created 2 nodesets instead of just 1 + +1. [Install the OpenStack K8S operators and their dependencies](../../../common/) +2. [Configuring networking and deploy the OpenStack control plane](control-plane.md) +3. [Configure and deploy the data plane](dataplane.md) + + diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/control-plane.md b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/control-plane.md new file mode 100644 index 000000000..34ab4df89 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/control-plane.md @@ -0,0 +1,55 @@ +# Configuring networking and deploy the OpenStack control plane + +## Assumptions + +- A storage class called `local-storage` should already exist. + +## Initialize + +Switch to the "openstack" namespace +``` +oc project openstack +``` +Change to the nfv/nfv-ovs-dpdk-sriov-2nodesets directory +``` +cd examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets +``` +Edit the [nncp/values.yaml](nncp/values.yaml) and +[service-values.yaml](service-values.yaml) files to suit +your environment. +``` +vi nncp/values.yaml +vi service-values.yaml +``` + +## Apply node network configuration + +Generate the node network configuration +``` +kustomize build nncp > nncp.yaml +``` + +Apply the NNCP CRs +``` +oc apply -f nncp.yaml +``` +Wait for NNCPs to be available +``` +oc wait nncp -l osp/nncm-config-type=standard --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured --timeout=300s +``` + +## Apply networking and control-plane configuration + +Generate the control-plane and networking CRs. +``` +kustomize build > control-plane.yaml +``` +Apply the CRs +``` +oc apply -f control-plane.yaml +``` + +Wait for control plane to be available +``` +oc wait osctlplane controlplane --for condition=Ready --timeout=600s +``` diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/dataplane.md b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/dataplane.md new file mode 100644 index 000000000..264f65c7e --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/dataplane.md @@ -0,0 +1,54 @@ +# Configuring and deploying the dataplane + +## Assumptions + +- The [control plane](control-plane.md) has been created and successfully deployed + +## Initialize + +Switch to the "openstack" namespace +``` +oc project openstack +``` +Change to the nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm directory +``` +cd examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm +``` +Edit the [nodeset/values.yaml](edpm/nodeset/values.yaml), [nodeset2/values.yaml](edpm/nodeset2/values.yaml) +and [deployment/values.yaml](deployment/values.yaml) files to suit your environment. +``` +vi nodeset/values.yaml +vi nodeset2/values.yaml +vi deployment/values.yaml +``` +Generate the dataplane nodesets CRs. +``` +kustomize build nodeset > dataplane-nodeset.yaml +kustomize build nodeset2 > dataplane-nodeset2.yaml +``` +Generate the dataplane deployment CR. +``` +kustomize build deployment > dataplane-deployment.yaml +``` + +## Create CRs +Create the nodesets CRs +``` +oc apply -f dataplane-nodeset.yaml +oc apply -f dataplane-nodeset2.yaml +``` +Wait for dataplane nodesets setup to finish +``` +oc wait osdpns openstack-edpm --for condition=SetupReady --timeout=600s +oc wait osdpns openstack-edpm-2 --for condition=SetupReady --timeout=600s +``` + +Start the deployment +``` +oc apply -f dataplane-deployment.yaml +``` + +Wait for dataplane deployment to finish +``` +oc wait osdpns openstack-edpm --for condition=Ready --timeout=40m +``` diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/.gitignore b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/.gitignore new file mode 100644 index 000000000..44b4a0dba --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/.gitignore @@ -0,0 +1,3 @@ +dataplane-nodeset.yaml +dataplane-nodeset-2.yaml +dataplane-deployment.yaml diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/kustomization.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/kustomization.yaml new file mode 100644 index 000000000..bb46b79d9 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/kustomization.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../va/nfv/ovs-dpdk-sriov/edpm/deployment + # - https://github.com/openstack-k8s-operators/architecture/va/nfv/ovs-dpdk-sriov/edpm/deployment?ref=main + ## It's possible to replace ../../../../../../va/nfv/ovs-dpdk-sriov/edpm/deployment/ with a git checkout URL + ## as per: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - values.yaml + +patches: + - target: + kind: OpenStackDataPlaneDeployment + name: edpm-deployment + patch: | + - op: add + path: /spec/nodeSets/- + value: openstack-edpm-2 diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/values.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/values.yaml new file mode 100644 index 000000000..7f371b2cc --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment/values.yaml @@ -0,0 +1,10 @@ +# yamllint disable rule:line-length +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-deployment-values + annotations: + config.kubernetes.io/local-config: "true" +data: {} diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/kustomization.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/kustomization.yaml new file mode 100644 index 000000000..ba395e0b2 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset + # - https://github.com/openstack-k8s-operators/architecture/va/nfv/ovs-dpdk-sriov/edpm/nodeset?ref=main + ## It's possible to replace ../../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset/ with a git checkout URL + ## as per: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - values.yaml diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/values.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/values.yaml new file mode 100644 index 000000000..a32226c03 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset/values.yaml @@ -0,0 +1,228 @@ +--- +# yamllint disable rule:line-length + +# local-config: referenced, but not emitted by kustomize +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values + annotations: + config.kubernetes.io/local-config: "true" +data: + root_password: cmVkaGF0Cg== + preProvisioned: false + baremetalSetTemplate: + ctlplaneInterface: eno2 # CHANGEME + cloudUserName: cloud-admin + provisioningInterface: enp1s0 # CHANGEME + bmhLabelSelector: + app: openstack # CHANGEME + passwordSecret: + name: baremetalset-password-secret + namespace: openstack + ssh_keys: + # Authorized keys that will have access to the dataplane computes via SSH + authorized: key # CHANGEME + # The private key that will have access to the dataplane computes via SSH + private: key2 # CHANGEME2 + # The public key that will have access to the dataplane computes via SSH + public: key3 # CHANGEME3 + nodeset: + ansible: + ansibleUser: cloud-admin + ansiblePort: 22 + ansibleVars: + # CHANGEME -- see https://access.redhat.com/solutions/253273 + # edpm_bootstrap_command: | + # subscription-manager register --username \ + # --password + # podman login -u -p registry.redhat.io + timesync_ntp_servers: + - hostname: pool.ntp.org + # CPU pinning settings + # edpm nfv ovs dpdk config + # CHANGEME + # yamllint disable-line rule:line-length + edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=64 iommu=pt intel_iommu=on tsx=off isolcpus=2-11,14-23" + edpm_tuned_profile: "cpu-partitioning-powersave" + edpm_tuned_isolated_cores: "2-11,14-23" + edpm_nova_libvirt_qemu_group: "hugetlbfs" + edpm_ovs_dpdk_pmd_core_list: "1,13,2,14,3,15" + edpm_ovs_dpdk_socket_memory: "4096" + edpm_ovs_dpdk_memory_channels: "4" + edpm_ovs_dpdk_vhost_postcopy_support: "true" + edpm_ovn_bridge_mappings: ['dpdk-mgmt:br-link1', 'dpdk2:br-link2'] + # edpm nfv sriov config + edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings: 'sriov1:eno5,sriov2:eno6' + # edpm_network_config + # These vars are edpm_network_config role vars + edpm_network_config_hide_sensitive_logs: false + edpm_network_config_os_net_config_mappings: + # Need to provide nic mapping based on system uuid or system product name + # here used systetm uuid + edpm-compute-0: # CHANGEME + nic1: 6c:fe:54:3f:8a:01 # CHANGEME + nic2: 6c:fe:54:3f:8a:02 # CHANGEME + nic3: 6c:fe:54:3f:8a:03 # CHANGEME + nic4: 6c:fe:54:3f:8a:04 # CHANGEME + nic5: 6c:fe:54:3f:8a:05 # CHANGEME + nic6: 6c:fe:54:3f:8a:06 # CHANGEME + nic7: 6c:fe:54:3f:8a:07 # CHANGEME + edpm_network_config_template: | + --- + {% set mtu_list = [ctlplane_mtu] %} + {% for network in nodeset_networks %} + {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }} + {%- endfor %} + {% set min_viable_mtu = mtu_list | max %} + network_config: + - type: ovs_bridge + name: {{ neutron_physical_bridge_name }} + mtu: {{ min_viable_mtu }} + use_dhcp: false + dns_servers: {{ ctlplane_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: {{ ctlplane_host_routes }} + members: + - type: interface + name: nic2 + mtu: {{ min_viable_mtu }} + # force the MAC address of the bridge to this interface + primary: true + {% for network in nodeset_networks if network not in ['external', 'tenant'] %} + - type: vlan + mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }} + vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }} + addresses: + - ip_netmask: {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }} + routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }} + {% endfor %} + - type: ovs_user_bridge + name: br-link1 + use_dhcp: false + ovs_extra: "set port br-link1 tag={{ lookup('vars', networks_lower['tenant'] ~ '_vlan_id') }}" + addresses: + - ip_netmask: {{ lookup('vars', networks_lower['tenant'] ~ '_ip') }}/{{ lookup('vars', networks_lower['tenant'] ~ '_cidr') }} + mtu: {{ lookup('vars', networks_lower['tenant'] ~ '_mtu') }} + members: + - type: ovs_dpdk_bond + name: dpdkbond0 + mtu: 9000 + rx_queue: 1 + ovs_options: "bond_mode=balance-tcp lacp=active other_config:lacp-time=fast other-config:lacp-fallback-ab=true other_config:lb-output-action=true" + members: + - type: ovs_dpdk_port + name: dpdk0 + members: + - type: interface + name: nic3 + - type: ovs_dpdk_port + name: dpdk1 + members: + - type: interface + name: nic4 + + - type: ovs_user_bridge + name: br-link2 + mtu: 9000 + use_dhcp: false + members: + - type: ovs_dpdk_port + name: dpdk2 + mtu: 9000 + rx_queue: 2 + members: + - type: interface + name: nic5 + - type: sriov_pf + name: nic6 + numvfs: 10 + mtu: 9000 + use_dhcp: false + promisc: true + - type: sriov_pf + name: nic7 + numvfs: 10 + mtu: 9000 + use_dhcp: false + promisc: true + # These vars are for the network config templates themselves and are + # considered EDPM network defaults. + neutron_physical_bridge_name: br-ex + neutron_public_interface_name: nic1 + # edpm_nodes_validation + edpm_nodes_validation_validate_controllers_icmp: false + edpm_nodes_validation_validate_gateway_icmp: false + dns_search_domains: [] + gather_facts: false + # edpm firewall, change the allowed CIDR if needed + edpm_sshd_configure_firewall: true + edpm_sshd_allowed_ranges: + - 192.168.122.0/24 + networks: + - defaultRoute: true + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + nodes: + edpm-compute-0: + hostName: edpm-compute-0 + services: + - bootstrap + - download-cache + - reboot-os + - configure-ovs-dpdk + - configure-network + - validate-network + - install-os + - configure-os + - ssh-known-hosts + - run-os + - install-certs + - ovn + - neutron-ovn-igmp + - neutron-metadata + - neutron-sriov + - libvirt + - nova-custom-ovsdpdksriov + - telemetry + nova: + compute: + conf: | + # CHANGEME + [DEFAULT] + reserved_host_memory_mb = 4096 + [compute] + cpu_shared_set = 0-3,24-27 + cpu_dedicated_set = 8-23,32-47 + [neutron] + physnets = dpdk1, dpdk2 + [neutron_physnet_dpdk1] + numa_nodes = 0 + [neutron_physnet_dpdk2] + numa_nodes = 0 + [neutron_tunnel] + numa_nodes = 0 + migration: + ssh_keys: + private: key4 # CHANGEME4 + public: key5 # CHANGEME5 + pci: + # yamllint disable-line rule:line-length + conf: | + # CHANGEME + [pci] + device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:19:00.3", "physical_network":"sriov1", "trusted":"true"} + device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:20:00.3", "physical_network":"sriov2", "trusted":"true"} + neutron: + igmp: + conf: | + [ovs] + igmp_snooping_enable = False diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/kustomization.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/kustomization.yaml new file mode 100644 index 000000000..6ba60388c --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/kustomization.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset + # - https://github.com/openstack-k8s-operators/architecture/va/nfv/ovs-dpdk-sriov/edpm/nodeset?ref=main + ## It's possible to replace ../../../../../../va/nfv/ovs-dpdk-sriov/edpm/nodeset/ with a git checkout URL + ## as per: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - values.yaml + +patches: + - target: + kind: ConfigMap + name: sriov-nova + patch: |- + - op: replace + path: /metadata/name + value: sriov-nova-2 + - target: + kind: ConfigMap + name: ovs-dpdk-sriov-cpu-pinning-nova + patch: |- + - op: replace + path: /metadata/name + value: ovs-dpdk-sriov-cpu-pinning-nova-2 + - target: + kind: OpenStackDataPlaneService + name: nova-custom-ovsdpdksriov + patch: |- + - op: replace + path: /metadata/name + value: nova-custom-ovsdpdksriov-2 + - op: replace + path: /spec/label + value: nova-custom-ovsdpdksriov-2 + - op: replace + path: /spec/dataSources/0/configMapRef/name + value: ovs-dpdk-sriov-cpu-pinning-nova-2 + - op: replace + path: /spec/dataSources/1/configMapRef/name + value: sriov-nova-2 + - target: + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm + patch: |- + - op: replace + path: /metadata/name + value: openstack-edpm-2 diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/values.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/values.yaml new file mode 100644 index 000000000..1982d4855 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2/values.yaml @@ -0,0 +1,227 @@ +--- +# yamllint disable rule:line-length + +# local-config: referenced, but not emitted by kustomize +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values + annotations: + config.kubernetes.io/local-config: "true" +data: + root_password: cmVkaGF0Cg== + preProvisioned: false + baremetalSetTemplate: + ctlplaneInterface: eno2 # CHANGEME + cloudUserName: cloud-admin + provisioningInterface: enp1s0 # CHANGEME + bmhLabelSelector: + app: openstack-2 # CHANGEME + passwordSecret: + name: baremetalset-password-secret + namespace: openstack + ssh_keys: + # Authorized keys that will have access to the dataplane computes via SSH + authorized: key # CHANGEME + # The private key that will have access to the dataplane computes via SSH + private: key2 # CHANGEME2 + # The public key that will have access to the dataplane computes via SSH + public: key3 # CHANGEME3 + nodeset: + ansible: + ansibleUser: cloud-admin + ansiblePort: 22 + ansibleVars: + # CHANGEME -- see https://access.redhat.com/solutions/253273 + # edpm_bootstrap_command: | + # subscription-manager register --username \ + # --password + # podman login -u -p registry.redhat.io + timesync_ntp_servers: + - hostname: pool.ntp.org + # CPU pinning settings + # edpm nfv ovs dpdk config + # CHANGEME + # yamllint disable-line rule:line-length + edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=64 iommu=pt intel_iommu=on tsx=off isolcpus=2-11,14-23" + edpm_tuned_profile: "cpu-partitioning-powersave" + edpm_tuned_isolated_cores: "2-11,14-23" + edpm_nova_libvirt_qemu_group: "hugetlbfs" + edpm_ovs_dpdk_pmd_core_list: "1,13,2,14,3,15" + edpm_ovs_dpdk_socket_memory: "4096" + edpm_ovs_dpdk_memory_channels: "4" + edpm_ovs_dpdk_vhost_postcopy_support: "true" + edpm_ovn_bridge_mappings: ['dpdk-mgmt:br-link1', 'dpdk2:br-link2'] + # edpm nfv sriov config + edpm_neutron_sriov_agent_SRIOV_NIC_physical_device_mappings: 'sriov1:eno5,sriov2:eno6' + # edpm_network_config + # These vars are edpm_network_config role vars + edpm_network_config_hide_sensitive_logs: false + edpm_network_config_os_net_config_mappings: + # Need to provide nic mapping based on system uuid or system product name + # here used systetm uuid + edpm-compute-1: # CHANGEME + nic1: 6c:fe:54:3f:8a:01 # CHANGEME + nic2: 6c:fe:54:3f:8a:02 # CHANGEME + nic3: 6c:fe:54:3f:8a:03 # CHANGEME + nic4: 6c:fe:54:3f:8a:04 # CHANGEME + nic5: 6c:fe:54:3f:8a:05 # CHANGEME + nic6: 6c:fe:54:3f:8a:06 # CHANGEME + nic7: 6c:fe:54:3f:8a:07 # CHANGEME + edpm_network_config_template: | + --- + {% set mtu_list = [ctlplane_mtu] %} + {% for network in nodeset_networks %} + {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }} + {%- endfor %} + {% set min_viable_mtu = mtu_list | max %} + network_config: + - type: ovs_bridge + name: {{ neutron_physical_bridge_name }} + mtu: {{ min_viable_mtu }} + use_dhcp: false + dns_servers: {{ ctlplane_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: {{ ctlplane_host_routes }} + members: + - type: interface + name: nic2 + mtu: {{ min_viable_mtu }} + # force the MAC address of the bridge to this interface + primary: true + {% for network in nodeset_networks if network not in ['external', 'tenant'] %} + - type: vlan + mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }} + vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }} + addresses: + - ip_netmask: {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }} + routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }} + {% endfor %} + - type: ovs_user_bridge + name: br-link1 + use_dhcp: false + ovs_extra: "set port br-link1 tag={{ lookup('vars', networks_lower['tenant'] ~ '_vlan_id') }}" + addresses: + - ip_netmask: {{ lookup('vars', networks_lower['tenant'] ~ '_ip') }}/{{ lookup('vars', networks_lower['tenant'] ~ '_cidr') }} + mtu: {{ lookup('vars', networks_lower['tenant'] ~ '_mtu') }} + members: + - type: ovs_dpdk_bond + name: dpdkbond0 + mtu: 9000 + rx_queue: 1 + ovs_options: "bond_mode=balance-tcp lacp=active other_config:lacp-time=fast other-config:lacp-fallback-ab=true other_config:lb-output-action=true" + members: + - type: ovs_dpdk_port + name: dpdk0 + members: + - type: interface + name: nic3 + - type: ovs_dpdk_port + name: dpdk1 + members: + - type: interface + name: nic4 + + - type: ovs_user_bridge + name: br-link2 + mtu: 9000 + use_dhcp: false + members: + - type: ovs_dpdk_port + name: dpdk2 + mtu: 9000 + rx_queue: 2 + members: + - type: interface + name: nic5 + - type: sriov_pf + name: nic6 + numvfs: 10 + mtu: 9000 + use_dhcp: false + promisc: true + - type: sriov_pf + name: nic7 + numvfs: 10 + mtu: 9000 + use_dhcp: false + promisc: true + # These vars are for the network config templates themselves and are + # considered EDPM network defaults. + neutron_physical_bridge_name: br-ex + neutron_public_interface_name: nic1 + # edpm_nodes_validation + edpm_nodes_validation_validate_controllers_icmp: false + edpm_nodes_validation_validate_gateway_icmp: false + dns_search_domains: [] + gather_facts: false + # edpm firewall, change the allowed CIDR if needed + edpm_sshd_configure_firewall: true + edpm_sshd_allowed_ranges: + - 192.168.122.0/24 + networks: + - defaultRoute: true + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + nodes: + edpm-compute-1: + hostName: edpm-compute-1 + services: + - bootstrap + - download-cache + - reboot-os + - configure-ovs-dpdk + - configure-network + - validate-network + - install-os + - configure-os + - run-os + - install-certs + - ovn + - neutron-ovn-igmp + - neutron-metadata + - neutron-sriov + - libvirt + - nova-custom-ovsdpdksriov-2 + - telemetry + nova: + compute: + conf: | + # CHANGEME + [DEFAULT] + reserved_host_memory_mb = 4096 + [compute] + cpu_shared_set = 0-3,24-27 + cpu_dedicated_set = 8-23,32-47 + [neutron] + physnets = dpdk1, dpdk2 + [neutron_physnet_dpdk1] + numa_nodes = 0 + [neutron_physnet_dpdk2] + numa_nodes = 0 + [neutron_tunnel] + numa_nodes = 0 + migration: + ssh_keys: + private: key4 # CHANGEME4 + public: key5 # CHANGEME5 + pci: + # yamllint disable-line rule:line-length + conf: | + # CHANGEME + [pci] + device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:19:00.3", "physical_network":"sriov1", "trusted":"true"} + device_spec = {"vendor_id":"8086", "product_id":"1572", "address": "0000:20:00.3", "physical_network":"sriov2", "trusted":"true"} + neutron: + igmp: + conf: | + [ovs] + igmp_snooping_enable = False diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/kustomization.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/kustomization.yaml new file mode 100644 index 000000000..497ada39d --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/kustomization.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../va/nfv/ovs-dpdk-sriov/ + # - https://github.com/openstack-k8s-operators/architecture/va/nfv/ovs-dpdk-sriov?ref=main + ## It's possible to replace ../../../va/nfv/ovs-dpdk-sriov/ with a git checkout URL as per: + ## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - nncp/values.yaml + - service-values.yaml diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/kustomization.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/kustomization.yaml new file mode 100644 index 000000000..c92bc55fa --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/kustomization.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../../lib/nncp + +resources: + - values.yaml diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/values.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/values.yaml new file mode 100644 index 000000000..35e74d861 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp/values.yaml @@ -0,0 +1,199 @@ +--- +# local-config: referenced, but not emitted by kustomize +apiVersion: v1 +kind: ConfigMap +metadata: + name: network-values + annotations: + config.kubernetes.io/local-config: "true" +data: + # nodes + node_0: + name: ostest-master-0 + internalapi_ip: 172.17.0.5 + tenant_ip: 172.19.0.5 + ctlplane_ip: 192.168.122.10 + storage_ip: 172.18.0.5 + node_1: + name: ostest-master-1 + internalapi_ip: 172.17.0.6 + tenant_ip: 172.19.0.6 + ctlplane_ip: 192.168.122.11 + storage_ip: 172.18.0.6 + node_2: + name: ostest-master-2 + internalapi_ip: 172.17.0.7 + tenant_ip: 172.19.0.7 + ctlplane_ip: 192.168.122.12 + storage_ip: 172.18.0.7 + + # networks + ctlplane: + dnsDomain: ctlplane.example.com + subnets: + - allocationRanges: + - end: 192.168.122.120 + start: 192.168.122.100 + - end: 192.168.122.200 + start: 192.168.122.150 + cidr: 192.168.122.0/24 + gateway: 192.168.122.1 + name: subnet1 + prefix-length: 24 + iface: enp7s0 + mtu: 1500 + lb_addresses: + - 192.168.122.80-192.168.122.90 + endpoint_annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "ctlplane", + "type": "macvlan", + "master": "ospbr", + "ipam": { + "type": "whereabouts", + "range": "192.168.122.0/24", + "range_start": "192.168.122.30", + "range_end": "192.168.122.70" + } + } + internalapi: + dnsDomain: internalapi.example.com + subnets: + - allocationRanges: + - end: 172.17.0.250 + start: 172.17.0.100 + cidr: 172.17.0.0/24 + name: subnet1 + vlan: 20 + mtu: 1500 + prefix-length: 24 + iface: enp7s0.20 + vlan: 20 + base_iface: enp7s0 + lb_addresses: + - 172.17.0.80-172.17.0.90 + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "internalapi", + "type": "macvlan", + "master": "internalapi", + "ipam": { + "type": "whereabouts", + "range": "172.17.0.0/24", + "range_start": "172.17.0.30", + "range_end": "172.17.0.70" + } + } + storage: + dnsDomain: storage.example.com + subnets: + - allocationRanges: + - end: 172.18.0.250 + start: 172.18.0.100 + cidr: 172.18.0.0/24 + name: subnet1 + vlan: 21 + mtu: 1500 + prefix-length: 24 + iface: enp7s0.21 + vlan: 21 + base_iface: enp7s0 + lb_addresses: + - 172.18.0.80-172.18.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "storage", + "type": "macvlan", + "master": "storage", + "ipam": { + "type": "whereabouts", + "range": "172.18.0.0/24", + "range_start": "172.18.0.30", + "range_end": "172.18.0.70" + } + } + tenant: + dnsDomain: tenant.example.com + subnets: + - allocationRanges: + - end: 172.19.0.250 + start: 172.19.0.100 + cidr: 172.19.0.0/24 + name: subnet1 + vlan: 22 + mtu: 1500 + prefix-length: 24 + iface: enp7s0.22 + vlan: 22 + base_iface: enp7s0 + lb_addresses: + - 172.19.0.80-172.19.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "tenant", + "type": "macvlan", + "master": "tenant", + "ipam": { + "type": "whereabouts", + "range": "172.19.0.0/24", + "range_start": "172.19.0.30", + "range_end": "172.19.0.70" + } + } + external: + dnsDomain: external.example.com + subnets: + - allocationRanges: + - end: 10.0.0.250 + start: 10.0.0.100 + cidr: 10.0.0.0/24 + gateway: 10.0.0.1 + name: subnet1 + mtu: 1500 + datacentre: + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "datacentre", + "type": "bridge", + "bridge": "ospbr", + "ipam": {} + } + + dns-resolver: + config: + server: + - 192.168.122.1 + search: [] + options: + - key: server + values: + - 192.168.122.1 + + routes: + config: [] + + rabbitmq: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.85 + rabbitmq-cell1: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.86 + + lbServiceType: LoadBalancer + storageClass: local-storage + bridgeName: ospbr diff --git a/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/service-values.yaml b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/service-values.yaml new file mode 100644 index 000000000..fa27780e4 --- /dev/null +++ b/examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/service-values.yaml @@ -0,0 +1,80 @@ +# local-config: referenced, but not emitted by kustomize +# +# CHANGEME: +# +# - Change network_vlan_ranges based on EDPM OvS DPDK SRIOV environment. +# - change numa_nodes in the neutron and neutron_tunnel section +# based on DPDK NIC's and NUMA alignment in the OvS DPDK & SRIOV compute node. +# +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" +data: + preserveJobs: false + neutron: + customServiceConfig: | + [DEFAULT] + global_physnet_mtu = 9000 + [ml2] + mechanism_drivers = ovn,sriovnicswitch + [ovn] + vhost_sock_dir = /var/lib/vhost_sockets + [ml2_type_vlan] + network_vlan_ranges = dpdk1:206:209,dpdk2:206:209,sriov1,sriov2 # CHANGEME + [ovs] + igmp_snooping_enable = True + [oslo_policy] + policy_file=/etc/neutron/policy.d/policy.yaml + ovn: + ovnController: + nicMappings: + datacentre: ospbr + + glance: + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:swift + [glance_store] + default_backend = default_backend + [default_backend] + swift_store_create_container_on_put = True + swift_store_auth_version = 3 + swift_store_auth_address = {{ .KeystoneInternalURL }} + swift_store_endpoint_type = internalURL + swift_store_user = service:glance + swift_store_key = {{ .ServicePassword }} + default: + replicas: 1 + swift: + enabled: true + telemetry: + enabled: true + template: + ceilometer: + enabled: true + metricStorage: + enabled: true + extraMounts: + - name: v1 + region: r1 + extraVol: + - propagation: + - NeutronAPI + extraVolType: Policy + volumes: + - name: neutron-policy + configMap: + name: neutron-policy + mounts: + - name: neutron-policy + mountPath: /etc/neutron/policy.d + readOnly: true + nova: + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + enabled_filtes = AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,PciPassthroughFilter,NUMATopologyFilter,AggregateInstanceExtraSpecsFilter diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index c4834104f..7c295b8ed 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -15,6 +15,7 @@ - rhoso-architecture-validate-osasinfra-ipv6 - rhoso-architecture-validate-ovs-dpdk - rhoso-architecture-validate-ovs-dpdk-sriov + - rhoso-architecture-validate-ovs-dpdk-sriov-2nodesets - rhoso-architecture-validate-pidone - rhoso-architecture-validate-sriov - rhoso-architecture-validate-uni01alpha diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index cba46f305..aa188e48a 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -163,6 +163,18 @@ parent: rhoso-architecture-base-job vars: cifmw_architecture_scenario: ovs-dpdk-sriov +- job: + files: + - examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets + - examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/deployment + - examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset + - examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/edpm/nodeset2 + - examples/dt/nfv/nfv-ovs-dpdk-sriov-2nodesets/nncp + - lib + name: rhoso-architecture-validate-ovs-dpdk-sriov-2nodesets + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: ovs-dpdk-sriov-2nodesets - job: files: - automation/net-env/pidone.yaml