From d591e29deb817dfe7c321e88735c35acff626772 Mon Sep 17 00:00:00 2001
From: Bohdan Dobrelia <bdobreli@redhat.com>
Date: Mon, 6 Nov 2023 16:14:44 +0100
Subject: [PATCH] Nova services FFU during adoption (no extra cell)

Update EDPM adoption docs and tests to execute Nova compute post-FFU.

Verify no dataplane disruptions during the FFU/adoption process.

Verify Nova services still control pre-created VM workload after
FFU/adotpion is done.

Signed-off-by: Bohdan Dobrelia <bdobreli@redhat.com>
---
 docs/openstack/edpm_adoption.md               | 203 ++++++++++++++++-
 docs/openstack/nova_adoption.md               |   2 +-
 .../roles/dataplane_adoption/tasks/main.yaml  |   4 +
 .../dataplane_adoption/tasks/nova_ffu.yaml    | 208 ++++++++++++++++++
 4 files changed, 415 insertions(+), 2 deletions(-)
 create mode 100644 tests/roles/dataplane_adoption/tasks/nova_ffu.yaml

diff --git a/docs/openstack/edpm_adoption.md b/docs/openstack/edpm_adoption.md
index e020852d0..1ae507efb 100644
--- a/docs/openstack/edpm_adoption.md
+++ b/docs/openstack/edpm_adoption.md
@@ -6,7 +6,15 @@
 
 ## Variables
 
-(There are no shell variables necessary currently.)
+Define the shell variables used in the Fast-forward upgrade steps below.
+The values are just illustrative, use values that are correct for your environment:
+
+```bash
+PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d)
+CONTROLLER_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100"
+
+alias openstack="oc exec -t openstackclient -- openstack"
+```
 
 ## Pre-checks
 
@@ -263,3 +271,196 @@ EOF
     ```
     oc wait --for condition=Ready osdpns/openstack --timeout=30m
     ```
+
+## Nova compute services fast-forward upgrade from Wallaby to Antelope
+
+Nova services rolling upgrade cannot be done during adoption,
+there is in a lock-step with Nova control plane services, because those
+are managed independently by EDPM ansible, and Kubernetes operators.
+Nova service operator and OpenStack Dataplane operator ensure upgrading
+is done independently of each other, by configuring
+`[upgrade_levels]compute=auto` for Nova services. Nova control plane
+services apply the change right after CR is patched. Nova compute EDPM
+services will catch up the same config change with ansible deployment
+later on.
+
+> **NOTE**: Additional orchestration happening around the FFU workarounds
+> configuration for Nova compute EDPM service is a subject of future changes.
+
+* Configure pre-FFU workarounds for Nova compute EDPM services to update its version records:
+
+    ```yaml
+    oc apply -f - <<EOF
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: nova-compute-workarounds
+      namespace: openstack
+    data:
+      19-nova-compute-cell1-workarounds.conf: |
+        [workarounds]
+        disable_compute_service_check_for_ffu=true
+    EOF
+
+    oc apply -f - <<EOF
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneService
+    metadata:
+      name: nova-compute-workarounds
+      namespace: openstack
+    spec:
+      label: nova.compute.workarounds
+      configMaps:
+        - nova-compute-workarounds
+      playbook: osp.edpm.nova
+    ---
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneDeployment
+    metadata:
+      name: openstack-nova-compute-workarounds
+      namespace: openstack
+    spec:
+      nodeSets:
+        - openstack
+      servicesOverride:
+        - nova-compute-workarounds
+    EOF
+    ```
+
+* Wait for cell1 Nova compute EDPM services version updated (it may take some time):
+
+    ```bash
+    oc exec -it mariadb-openstack-cell1 -- mysql --user=root --password=${PODIFIED_DB_ROOT_PASSWORD} \
+      -e "select a.version from nova_cell1.services a join nova_cell1.services b where a.version!=b.version and a.binary='nova-compute';"
+    ```
+  The above query should return an empty result as a completion criterion.
+
+* Remove pre-FFU workarounds for Nova control plane services:
+
+    ```yaml
+    oc patch openstackcontrolplane openstack -n openstack --type=merge --patch '
+    spec:
+      nova:
+        template:
+          cellTemplates:
+            cell0:
+              conductorServiceTemplate:
+                customServiceConfig: |
+                  [workarounds]
+                  disable_compute_service_check_for_ffu=false
+            cell1:
+              metadataServiceTemplate:
+                customServiceConfig: |
+                  [workarounds]
+                  disable_compute_service_check_for_ffu=false
+              conductorServiceTemplate:
+                customServiceConfig: |
+                  [workarounds]
+                  disable_compute_service_check_for_ffu=false
+          apiServiceTemplate:
+            customServiceConfig: |
+              [workarounds]
+              disable_compute_service_check_for_ffu=false
+          metadataServiceTemplate:
+            customServiceConfig: |
+              [workarounds]
+              disable_compute_service_check_for_ffu=false
+          schedulerServiceTemplate:
+            customServiceConfig: |
+              [workarounds]
+              disable_compute_service_check_for_ffu=false
+    '
+    ```
+
+* Wait for Nova control plane services' CRs to become ready:
+
+    ```bash
+    oc get novaapis --field-selector metadata.name=nova-api -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novacells --field-selector metadata.name=nova-cell0 -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novacells --field-selector metadata.name=nova-cell1 -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novaconductors --field-selector metadata.name=nova-cell0-conductor -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novaconductors --field-selector metadata.name=nova-cell1-conductor -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novametadata --field-selector metadata.name=nova-metadata -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novanovncproxies --field-selector metadata.name=nova-cell1-novncproxy -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    oc get novaschedulers --field-selector metadata.name=nova-scheduler -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+    ```
+
+* Remove pre-FFU workarounds for Nova compute EDPM services:
+
+    ```yaml
+    oc apply -f - <<EOF
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: nova-compute-ffu
+      namespace: openstack
+    data:
+      20-nova-compute-cell1-ffu-cleanup.conf: |
+        [workarounds]
+        disable_compute_service_check_for_ffu=false
+    EOF
+
+    oc apply -f - <<EOF
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneService
+    metadata:
+      name: nova-compute-ffu
+      namespace: openstack
+    spec:
+      label: nova.compute.ffu
+      configMaps:
+        - nova-compute-ffu
+      playbook: osp.edpm.nova
+    ---
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneDeployment
+    metadata:
+      name: openstack-nova-compute-ffu
+      namespace: openstack
+    spec:
+      nodeSets:
+        - openstack
+      servicesOverride:
+        - nova-compute-ffu
+    EOF
+    ```
+
+* Wait for Nova compute EDPM service to become ready:
+
+    ```bash
+    oc wait --for condition=Ready osdpd/openstack-nova-compute-ffu --timeout=5m
+    ```
+
+* Run Nova DB online migrations to complete FFU:
+
+    ```bash
+    oc exec -it nova-cell0-conductor-0 -- nova-manage db online_data_migrations
+    oc exec -it nova-cell1-conductor-0 -- nova-manage db online_data_migrations
+    ```
+
+* Verify no Nova compute dataplane disruptions during the adoption/upgrade process:
+
+  ```bash
+  $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001   running'
+  ```
+
+* Verify if Nova services control the existing VM instance:
+
+  ```bash
+  openstack server list | grep -qF '| test | ACTIVE |' && openstack server stop test
+  openstack server list | grep -qF '| test | SHUTOFF |'
+  $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001   shut off'
+
+  openstack server list | grep -qF '| test | SHUTOFF |' && openstack server start test
+  openstack server list | grep -F '| test | ACTIVE |'
+  $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001   running'
+  ```
+  Note that in this guide, the same host acts as a controller, and also as a compute.
diff --git a/docs/openstack/nova_adoption.md b/docs/openstack/nova_adoption.md
index 9f8234849..69460473a 100644
--- a/docs/openstack/nova_adoption.md
+++ b/docs/openstack/nova_adoption.md
@@ -133,7 +133,7 @@ Compare the following outputs with the topology specific configuration
   * Default cell is renamed to `cell1` (in a multi-cell setup, it should become indexed as the last cell instead).
   * RabbitMQ transport URL no longer uses `guest`.
 
-* Verify no Nova compute dataplane disruptions during the adoption process:
+* Verify no Nova compute dataplane disruptions during the adoption/upgrade process:
 
   ```bash
   $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001   running'
diff --git a/tests/roles/dataplane_adoption/tasks/main.yaml b/tests/roles/dataplane_adoption/tasks/main.yaml
index 6404afc25..75277710b 100644
--- a/tests/roles/dataplane_adoption/tasks/main.yaml
+++ b/tests/roles/dataplane_adoption/tasks/main.yaml
@@ -265,3 +265,7 @@
     oc wait --for condition=Ready osdpns/openstack --timeout=40m
   # TODO: work on network configuration for making possible to run this task on other IP ranges
   when: "edpm_node_ip.startswith('192.168.122')"
+
+- name: Complete Nova services Wallaby->Antelope FFU
+  ansible.builtin.include_tasks:
+    file: nova_ffu.yaml
diff --git a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml
new file mode 100644
index 000000000..6083c4b76
--- /dev/null
+++ b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml
@@ -0,0 +1,208 @@
+- name: get podified MariaDB service cluster IP
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header    oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}'
+  register: podified_mariadb_ip_result
+
+- name: get podified cell1 MariaDB IP
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}'
+  register: podified_cell1_mariadb_ip_result
+
+- name: set podified MariaDB copy shell vars
+  no_log: "{{ use_no_log }}"
+  ansible.builtin.set_fact:
+    mariadb_copy_shell_vars: |
+      PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }}
+      PODIFIED_CELL1_MARIADB_IP={{podified_cell1_mariadb_ip_result.stdout}}
+      PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}"
+
+- name: configure pre-FFU workarounds for Nova compute EDPM services to update its version records
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc apply -f - <<EOF
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: nova-compute-workarounds
+      namespace: openstack
+    data:
+      19-nova-compute-cell1-workarounds.conf: |
+        [workarounds]
+        disable_compute_service_check_for_ffu=true
+    EOF
+
+    oc apply -f - <<EOF
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneService
+    metadata:
+      name: nova-compute-workarounds
+      namespace: openstack
+    spec:
+      label: nova.compute.workarounds
+      configMaps:
+        - nova-compute-workarounds
+      playbook: osp.edpm.nova
+    ---
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneDeployment
+    metadata:
+      name: openstack-nova-compute-workarounds
+      namespace: openstack
+    spec:
+      nodeSets:
+        - openstack
+      servicesOverride:
+        - nova-compute-workarounds
+    EOF
+
+- name: wait for cell1 Nova compute EDPM services version updated
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    {{ mariadb_copy_shell_vars }}
+    oc exec -it mariadb-openstack-cell1 -- mysql --user=root --password=${PODIFIED_DB_ROOT_PASSWORD} \
+      -e "select a.version from nova_cell1.services a join nova_cell1.services b where a.version!=b.version and a.binary='nova-compute';"
+  register: records_check_results
+  until: records_check_results.rc == 0 and records_check_results.stdout_lines | length == 0
+  retries: 20
+  delay: 6
+
+- name: remove pre-FFU workarounds for Nova control plane services
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc patch openstackcontrolplane openstack -n openstack --type=merge --patch '
+    spec:
+      nova:
+        template:
+          cellTemplates:
+            cell0:
+              conductorServiceTemplate:
+                customServiceConfig: |
+                  [workarounds]
+                  disable_compute_service_check_for_ffu=false
+            cell1:
+              metadataServiceTemplate:
+                customServiceConfig: |
+                  [workarounds]
+                  disable_compute_service_check_for_ffu=false
+              conductorServiceTemplate:
+                customServiceConfig: |
+                  [workarounds]
+                  disable_compute_service_check_for_ffu=false
+          apiServiceTemplate:
+            customServiceConfig: |
+              [workarounds]
+              disable_compute_service_check_for_ffu=false
+          metadataServiceTemplate:
+            customServiceConfig: |
+              [workarounds]
+              disable_compute_service_check_for_ffu=false
+          schedulerServiceTemplate:
+            customServiceConfig: |
+              [workarounds]
+              disable_compute_service_check_for_ffu=false
+    '
+# NOTE(bogdando): Status phase 'Running' doesn't necessarily mean it IS running in fact.
+# Instead, check for CRs status, then attempt exec'ing on the conductors pods to run live migrations,
+# with retries as guardrails for real running statuses of pods
+- name: wait for Nova control plane services' CRs to become ready
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc get {{ service.cr }} --field-selector metadata.name={{ service.name }} -o jsonpath='{.items[0].status.conditions}' \
+    | jq -e '.[]|select(.type=="Ready" and .status=="True")'
+  register: nova_crs_ready_result
+  until: nova_crs_ready_result is success
+  retries: 30
+  delay: 5
+  loop_control:
+    loop_var: service
+  loop:
+    - cr: novaapis
+      name: nova-api
+    - cr: novacells
+      name: nova-cell0
+    - cr: novacells
+      name: nova-cell1
+    - cr: novaconductors
+      name: nova-cell0-conductor
+      cond: DeploymentReady
+    - cr: novaconductors
+      name: nova-cell1-conductor
+    - cr: novametadata
+      name: nova-metadata
+    - cr: novanovncproxies
+      name: nova-cell1-novncproxy
+    - cr: novaschedulers
+      name: nova-scheduler
+
+- name: remove pre-FFU workarounds for Nova compute EDPM services
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc apply -f - <<EOF
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: nova-compute-ffu
+      namespace: openstack
+    data:
+      20-nova-compute-cell1-ffu-cleanup.conf: |
+        [workarounds]
+        disable_compute_service_check_for_ffu=false
+    EOF
+
+    oc apply -f - <<EOF
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneService
+    metadata:
+      name: nova-compute-ffu
+      namespace: openstack
+    spec:
+      label: nova.compute.ffu
+      configMaps:
+        - nova-compute-ffu
+      playbook: osp.edpm.nova
+    ---
+    apiVersion: dataplane.openstack.org/v1beta1
+    kind: OpenStackDataPlaneDeployment
+    metadata:
+      name: openstack-nova-compute-ffu
+      namespace: openstack
+    spec:
+      nodeSets:
+        - openstack
+      servicesOverride:
+        - nova-compute-ffu
+    EOF
+
+- name: wait for Nova compute EDPM services to become ready
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc wait --for condition=Ready osdpd/openstack-nova-compute-ffu --timeout=5m
+  register: nova_ffu_edpm_result
+  until: nova_ffu_edpm_result is success
+  retries: 10
+  delay: 6
+
+- name: run Nova DB migrations to complete Wallaby->antelope FFU
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc exec -it nova-cell0-conductor-0 -- nova-manage db online_data_migrations
+    oc exec -it nova-cell1-conductor-0 -- nova-manage db online_data_migrations
+  register: nova_exec_result
+  until: nova_exec_result is success
+  retries: 10
+  delay: 6
+
+- name: Adopted Nova FFU post-checks
+  ansible.builtin.include_role:
+    name: nova_adoption
+    tasks_from: verify.yaml