From 2cc55f38dfa8d52ae22c12550cd3a1f1f0081869 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Mon, 6 Nov 2023 17:40:59 +0100 Subject: [PATCH] Nova: Refactor mariadb pre-checks, add post-checks Get services topology specific configuration in pull_openstack_configuration. Add missing role for that as well. Split maridb source vs podified env vars shell headers to use it in corresponding places. Update and fix the composition of the services pre-check list to execute it before stopping services. Update and fix the composition of the list of the services to be stopped (cannot pull data from stopped services). Verify no dataplane disruptions during the FFU/adoption process. Verify Nova services still control pre-created VM workload after FFU/adotpion is done. Signed-off-by: Bohdan Dobrelia --- docs/openstack/edpm_adoption.md | 21 +++++ docs/openstack/mariadb_copy.md | 62 +++++++++---- docs/openstack/nova_adoption.md | 33 +++++++ .../openstack/pull_openstack_configuration.md | 60 ++++++++++++- docs/openstack/stop_openstack_services.md | 3 + mkdocs.yml | 2 +- .../dataplane_adoption/tasks/nova_ffu.yaml | 15 ++-- .../mariadb_copy/tasks/env_vars_dst.yaml | 27 ++++++ .../mariadb_copy/tasks/env_vars_src.yaml | 8 ++ tests/roles/mariadb_copy/tasks/main.yaml | 48 ++++------ .../mariadb_copy/templates/dump_dbs.bash | 2 +- .../mariadb_copy/templates/post_checks.bash | 38 +++++++- .../mariadb_copy/templates/pre_checks.bash | 8 +- .../mariadb_copy/templates/restore_dbs.bash | 3 +- tests/roles/nova_adoption/tasks/main.yaml | 4 + tests/roles/nova_adoption/tasks/verify.yaml | 37 ++++++++ .../tasks/main.yaml | 88 +++++++++++++++++++ 17 files changed, 388 insertions(+), 71 deletions(-) create mode 100644 tests/roles/mariadb_copy/tasks/env_vars_dst.yaml create mode 100644 tests/roles/mariadb_copy/tasks/env_vars_src.yaml create mode 100644 tests/roles/nova_adoption/tasks/verify.yaml create mode 100644 tests/roles/pull_openstack_configuration/tasks/main.yaml diff --git a/docs/openstack/edpm_adoption.md b/docs/openstack/edpm_adoption.md index 6e5958c19..1ae507efb 100644 --- a/docs/openstack/edpm_adoption.md +++ b/docs/openstack/edpm_adoption.md @@ -11,6 +11,9 @@ The values are just illustrative, use values that are correct for your environme ```bash PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) +CONTROLLER_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100" + +alias openstack="oc exec -t openstackclient -- openstack" ``` ## Pre-checks @@ -443,3 +446,21 @@ later on. oc exec -it nova-cell1-conductor-0 -- nova-manage db online_data_migrations ``` +* Verify no Nova compute dataplane disruptions during the adoption/upgrade process: + + ```bash + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + +* Verify if Nova services control the existing VM instance: + + ```bash + openstack server list | grep -qF '| test | ACTIVE |' && openstack server stop test + openstack server list | grep -qF '| test | SHUTOFF |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 shut off' + + openstack server list | grep -qF '| test | SHUTOFF |' && openstack server start test + openstack server list | grep -F '| test | ACTIVE |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + Note that in this guide, the same host acts as a controller, and also as a compute. diff --git a/docs/openstack/mariadb_copy.md b/docs/openstack/mariadb_copy.md index 0bd0627f7..e89af417c 100644 --- a/docs/openstack/mariadb_copy.md +++ b/docs/openstack/mariadb_copy.md @@ -18,6 +18,8 @@ cluster. * Podified MariaDB and RabbitMQ are running. No other podified control plane services are running. + * Required services specific topology [configuration collected](pull_openstack_configuration.md#get-services-topology-specific-configuration) + * OpenStack services have been [stopped](stop_openstack_services.md) * There must be network routability between: @@ -40,38 +42,28 @@ Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment: ```bash -MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified - PODIFIED_MARIADB_IP=$(oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}') PODIFIED_CELL1_MARIADB_IP=$(oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}') PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) -# Replace with your environment's MariaDB IP: -SOURCE_MARIADB_IP=192.168.122.100 -SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') - # The CHARACTER_SET and collation should match the source DB # if the do not then it will break foreign key relationships # for any tables that are created in the future as part of db sync CHARACTER_SET=utf8 COLLATION=utf8_general_ci +MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified +# Replace with your environment's MariaDB IP: +SOURCE_MARIADB_IP=192.168.122.100 +SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') ``` ## Pre-checks -* Test connection to the original DB (show databases): +* Get the count of not-OK source databases: ```bash - podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ - mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;' - ``` - -* Run mysqlcheck on the original DB to look for things that are not OK: - - ```bash - podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK + test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" ``` * Test connection to podified DBs (show databases): @@ -169,20 +161,52 @@ COLLATION=utf8_general_ci ## Post-checks +Compare the following outputs with the topology specific configuration +[collected earlier](pull_openstack_configuration.md#get-services-topology-specific-configuration): + * Check that the databases were imported correctly: ```bash oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep keystone + | grep -q keystone + # ensure neutron db is renamed from ovs_neutron oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep neutron + | grep -q neutron + echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep ' ovs_neutron ' + # ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep nova_cell1 + | grep -q nova_cell1 + + # ensure default cell renamed to cell1, and the cell UUIDs retained intact + NOVADB_MAPPED_CELLS=$(oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ + nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') + uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' + left_behind=$(comm -23 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) + changed=$(comm -13 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) + test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 + default=$(grep -E ' default$' <<<$left_behind) + test $(grep -Ec ' \S+$' <<<$changed) -eq 1 + grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed + + # ensure the registered Nova compute service name has not changed + oc rsh mariadb-openstack-cell1 mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_cell1 \ + -e "select host from services where services.binary='nova-compute';" \ + | grep -qF '| standalone.localdomain |' + echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES | grep -qF ' standalone.localdomain' + + # ensure the VM instance cell ID has not changed + oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from instance_mappings;" | grep -qF -m1 ' 2 |' + echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS | grep -q -m1 ' 2$' ``` * During the pre/post checks the pod `mariadb-client` might have returned a pod security warning diff --git a/docs/openstack/nova_adoption.md b/docs/openstack/nova_adoption.md index 31697ef5d..69460473a 100644 --- a/docs/openstack/nova_adoption.md +++ b/docs/openstack/nova_adoption.md @@ -118,3 +118,36 @@ alias openstack="oc exec -t openstackclient -- openstack" openstack endpoint list | grep nova openstack server list ``` + +Compare the following outputs with the topology specific configuration +[collected earlier](pull_openstack_configuration.md#get-services-topology-specific-configuration): + +* Query the superconductor for cell1 existance: + + ```bash + oc rsh nova-cell0-conductor-0 nova-manage cell_v2 list_cells | grep -F '| cell1 |' + ``` + + The expected changes to happen: + * cell1's `nova` DB and user name become `nova_cell1`. + * Default cell is renamed to `cell1` (in a multi-cell setup, it should become indexed as the last cell instead). + * RabbitMQ transport URL no longer uses `guest`. + +* Verify no Nova compute dataplane disruptions during the adoption/upgrade process: + + ```bash + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + + * Verify if Nova services control the existing VM instance: + + ```bash + openstack server list | grep -qF '| test | ACTIVE |' && openstack server stop test + openstack server list | grep -qF '| test | SHUTOFF |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 shut off' + + openstack server list | grep -qF '| test | SHUTOFF |' && openstack server start test + openstack server list | grep -F '| test | ACTIVE |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + Note that in this guide, the same host acts as a controller, and also as a compute. diff --git a/docs/openstack/pull_openstack_configuration.md b/docs/openstack/pull_openstack_configuration.md index 7169ddc67..808a17f44 100644 --- a/docs/openstack/pull_openstack_configuration.md +++ b/docs/openstack/pull_openstack_configuration.md @@ -8,7 +8,7 @@ missed or misconfigured. Make sure you have pull the os-diff repository and configure according to your environment: -[Configure os-diff](planning.md#Configuration tooling) +[Configure os-diff](planning.md#configuration-tooling) ## Pull configuration from a TripleO deployment @@ -45,3 +45,61 @@ Once the ansible playbook has been run, you should have into your local director ▾ collect_tripleo_configs/ ▾ glance/ ``` + +## Get services topology specific configuration + +Define the shell variables used in the steps below. The values are +just illustrative, use values that are correct for your environment: + +```bash +MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified +SOURCE_MARIADB_IP=192.168.122.100 +SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') +``` + +Note the following outputs to compare it with post-adoption values later on: + +* Test connection to the original DB: + + ```bash + PULL_OPENSTACK_CONFIGURATION_DATABASES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;') + ``` + Note the `nova`, `nova_api`, `nova_cell0` databases residing in the same DB host. + +* Run mysqlcheck on the original DB to look for things that are not OK: + + ```bash + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK) + ``` + +* Get Nova cells mappings from database: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') + ``` + +* Get Nova instances cell_ids from database: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from nova_api.instance_mappings;") + ``` + +* Get the host names of the registered Nova compute services: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select host from nova.services where services.binary='nova-compute';") + ``` + +* Get the list of mapped Nova cells: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells) + ``` diff --git a/docs/openstack/stop_openstack_services.md b/docs/openstack/stop_openstack_services.md index 456bf303e..9fedc93fb 100644 --- a/docs/openstack/stop_openstack_services.md +++ b/docs/openstack/stop_openstack_services.md @@ -58,6 +58,9 @@ openstack volume backup list --all-projects -c ID -c Status |grep -E '\| .+ing \ openstack image list -c ID -c Status |grep -E '\| .+ing \|' ``` +Also collect the [topology configuration](pull_openstack_configuration.md#get-services-topology-specific-configuration), +before stopping services required to gather it live. You will need it to compare it +with the post-adoption values later on. ## Stopping control plane services diff --git a/mkdocs.yml b/mkdocs.yml index c959a208f..45cbf0a18 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -29,7 +29,7 @@ nav: - OpenStack: - openstack/planning.md - openstack/backend_services_deployment.md - - oepsntack/pull_openstack_configuration.md + - openstack/pull_openstack_configuration.md - openstack/stop_openstack_services.md - openstack/mariadb_copy.md - openstack/ovn_adoption.md diff --git a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml index b2326d041..e4f80cde7 100644 --- a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml +++ b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml @@ -1,8 +1,7 @@ -- name: set podified MariaDB copy shell vars - no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - mariadb_copy_shell_vars: | - PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" +- name: get dst database service environment variables + ansible.builtin.include_role: + name: mariadb_copy + tasks_from: env_vars_dst.yaml - name: configure pre-FFU workarounds for Nova compute EDPM services to update its version records ansible.builtin.shell: | @@ -48,7 +47,7 @@ ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} - {{ mariadb_copy_shell_vars }} + {{ mariadb_copy_shell_vars_dst }} oc exec -it mariadb-openstack-cell1 -- mysql --user=root --password=${PODIFIED_DB_ROOT_PASSWORD} \ -e "select a.version from nova_cell1.services a join nova_cell1.services b where a.version!=b.version and a.binary='nova-compute';" register: records_check_results @@ -187,3 +186,7 @@ retries: 10 delay: 6 +- name: Adopted Nova FFU post-checks + ansible.builtin.include_role: + name: nova_adoption + tasks_from: verify.yaml diff --git a/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml b/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml new file mode 100644 index 000000000..fa7d423df --- /dev/null +++ b/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml @@ -0,0 +1,27 @@ +- name: get podified MariaDB service cluster IP + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}' + register: podified_mariadb_ip_result + +- name: get podified cell1 MariaDB IP + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}' + register: podified_cell1_mariadb_ip_result + +- name: set MariaDB copy shell vars + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + mariadb_copy_shell_vars_dst: | + PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} + PODIFIED_CELL1_MARIADB_IP={{podified_cell1_mariadb_ip_result.stdout}} + PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" + + # The CHARACTER_SET and collation should match the source DB + # if the do not then it will break foreign key relationships + # for any tables that are created in the future as part of db sync + CHARACTER_SET=utf8 + COLLATION=utf8_general_ci diff --git a/tests/roles/mariadb_copy/tasks/env_vars_src.yaml b/tests/roles/mariadb_copy/tasks/env_vars_src.yaml new file mode 100644 index 000000000..66bbd5312 --- /dev/null +++ b/tests/roles/mariadb_copy/tasks/env_vars_src.yaml @@ -0,0 +1,8 @@ +- name: set src MariaDB copy shell vars + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + mariadb_copy_shell_vars_src: | + MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified + # TODO: remove the default(external_...) when CI is transitioned to use 'source_...' + SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }} + SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}" diff --git a/tests/roles/mariadb_copy/tasks/main.yaml b/tests/roles/mariadb_copy/tasks/main.yaml index f284c3fd6..ad9155aed 100644 --- a/tests/roles/mariadb_copy/tasks/main.yaml +++ b/tests/roles/mariadb_copy/tasks/main.yaml @@ -1,38 +1,19 @@ -- name: get podified MariaDB service cluster IP - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}' - register: podified_mariadb_ip_result +- name: get the source database service environment variables + ansible.builtin.include_tasks: + file: env_vars_src.yaml -- name: get podified cell1 MariaDB IP - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}' - register: podified_cell1_mariadb_ip_result +- name: get the destination database service environment variables + ansible.builtin.include_tasks: + file: env_vars_dst.yaml -- name: set MariaDB copy shell vars +- name: Get the count of not-OK source databases no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - mariadb_copy_shell_vars: | - MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified - - PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} - PODIFIED_CELL1_MARIADB_IP={{podified_cell1_mariadb_ip_result.stdout}} - PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" - - # TODO: remove the default(external_...) when CI is transitioned to use 'source_...' - SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }} - SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}" - - # The CHARACTER_SET and collation should match the source DB - # if the do not then it will break foreign key relationships - # for any tables that are created in the future as part of db sync - CHARACTER_SET=utf8 - COLLATION=utf8_general_ci + environment: + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK: "{{ pull_openstack_configuration_mysqlcheck_nok }}" + ansible.builtin.shell: + cmd: test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" -- name: MariaDB copy pre checks +- name: test connection to podified DBs (show databases) no_log: "{{ use_no_log }}" ansible.builtin.shell: cmd: "{{ lookup('ansible.builtin.template', 'pre_checks.bash') }}" @@ -49,5 +30,10 @@ - name: MariaDB post-checks no_log: "{{ use_no_log }}" + environment: + PULL_OPENSTACK_CONFIGURATION_DATABASES: "{{ pull_openstack_configuration_databases }}" + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS: "{{ pull_openstack_configuration_novadb_mapped_cells }}" + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES: "{{ pull_openstack_configuration_nova_compute_hostnames }}" + PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS: "{{ pull_openstack_configuration_novadb_instances_cell_ids }}" ansible.builtin.shell: cmd: "{{ lookup('ansible.builtin.template', 'post_checks.bash') }}" diff --git a/tests/roles/mariadb_copy/templates/dump_dbs.bash b/tests/roles/mariadb_copy/templates/dump_dbs.bash index 3c73ed20d..45266d03f 100644 --- a/tests/roles/mariadb_copy/templates/dump_dbs.bash +++ b/tests/roles/mariadb_copy/templates/dump_dbs.bash @@ -1,7 +1,7 @@ #!/bin/bash {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_src }} mkdir -p {{ mariadb_copy_tmp_dir }} cd {{ mariadb_copy_tmp_dir }} diff --git a/tests/roles/mariadb_copy/templates/post_checks.bash b/tests/roles/mariadb_copy/templates/post_checks.bash index dbe92c043..e71d2e696 100644 --- a/tests/roles/mariadb_copy/templates/post_checks.bash +++ b/tests/roles/mariadb_copy/templates/post_checks.bash @@ -1,14 +1,44 @@ {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_src }} +{{ mariadb_copy_shell_vars_dst }} oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep keystone + | grep -q keystone + # ensure neutron db is renamed from ovs_neutron oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep neutron + | grep -q neutron +echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep ' ovs_neutron ' + # ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep nova_cell1 \ No newline at end of file + | grep -q nova_cell1 + +# ensure default cell renamed to cell1, and the cell UUIDs retained intact +NOVADB_MAPPED_CELLS=$(oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ + nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') +uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' +left_behind=$(comm -23 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) +changed=$(comm -13 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) +test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 +default=$(grep -E ' default$' <<<$left_behind) +test $(grep -Ec ' \S+$' <<<$changed) -eq 1 +grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed + +# ensure the registered Nova compute service name has not changed +oc rsh mariadb-openstack-cell1 mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_cell1 \ + -e "select host from services where services.binary='nova-compute';" \ + | grep -qF '| standalone.localdomain |' +echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES | grep -qF ' standalone.localdomain' + +# ensure the VM instance cell ID has not changed +oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from instance_mappings;" | grep -qF -m1 ' 2 |' +echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS | grep -q -m1 ' 2$' diff --git a/tests/roles/mariadb_copy/templates/pre_checks.bash b/tests/roles/mariadb_copy/templates/pre_checks.bash index 9f61b135c..3c4d62afc 100644 --- a/tests/roles/mariadb_copy/templates/pre_checks.bash +++ b/tests/roles/mariadb_copy/templates/pre_checks.bash @@ -1,14 +1,8 @@ #!/bin/bash {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_dst }} -# Test connection to the original DB (show databases) -podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ - mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;' -# Run mysqlcheck on the original DB to look for things that are not OK -podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" # Test connection to podified DBs (show databases) oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' diff --git a/tests/roles/mariadb_copy/templates/restore_dbs.bash b/tests/roles/mariadb_copy/templates/restore_dbs.bash index bba3dba2c..062dd548f 100755 --- a/tests/roles/mariadb_copy/templates/restore_dbs.bash +++ b/tests/roles/mariadb_copy/templates/restore_dbs.bash @@ -1,7 +1,8 @@ #!/bin/bash {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_src }} +{{ mariadb_copy_shell_vars_dst }} cd {{ mariadb_copy_tmp_dir }} # db schemas to rename on import diff --git a/tests/roles/nova_adoption/tasks/main.yaml b/tests/roles/nova_adoption/tasks/main.yaml index 6a7ae7b6d..a371690c9 100644 --- a/tests/roles/nova_adoption/tasks/main.yaml +++ b/tests/roles/nova_adoption/tasks/main.yaml @@ -97,3 +97,7 @@ until: nova_responding_result is success retries: 60 delay: 2 + +- name: Nova adoption post-checks + ansible.builtin.include_tasks: + file: verify.yaml diff --git a/tests/roles/nova_adoption/tasks/verify.yaml b/tests/roles/nova_adoption/tasks/verify.yaml new file mode 100644 index 000000000..58f212ce8 --- /dev/null +++ b/tests/roles/nova_adoption/tasks/verify.yaml @@ -0,0 +1,37 @@ +- name: set Nova services shell vars + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + nova_header: | + alias openstack="oc exec -t openstackclient -- openstack" + CONTROLLER_SSH="{{ controller1_ssh }}" + +# TODO(bogdando): provide automated checks for 'The expected changes to happen' +- name: query the superconductor for cell1 existance + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc rsh nova-cell0-conductor-0 nova-manage cell_v2 list_cells | grep -F '| cell1 |' + +- name: verify no Nova compute dataplane disruptions during the adoption/upgrade process + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ nova_header }} + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + +- name: verify if Nova services control the existing VM instance + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ nova_header }} + ${BASH_ALIASES[openstack]} server list | grep -qF '| test | ACTIVE |' && ${BASH_ALIASES[openstack]} server stop test + ${BASH_ALIASES[openstack]} server list | grep -qF '| test | SHUTOFF |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 shut off' + + ${BASH_ALIASES[openstack]} server list | grep -qF '| test | SHUTOFF |' && ${BASH_ALIASES[openstack]} server start test + ${BASH_ALIASES[openstack]} server list | grep -F '| test | ACTIVE |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + register: nova_verify_result + until: nova_verify_result is success + retries: 10 + delay: 6 diff --git a/tests/roles/pull_openstack_configuration/tasks/main.yaml b/tests/roles/pull_openstack_configuration/tasks/main.yaml new file mode 100644 index 000000000..8af393f85 --- /dev/null +++ b/tests/roles/pull_openstack_configuration/tasks/main.yaml @@ -0,0 +1,88 @@ +- name: set shell vars pull openstack configuration ssh commands + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + pull_openstack_configuration_ssh_shell_vars: | + CONTROLLER_SSH="{{ controller1_ssh }}" + +- name: get src database service environment variables + ansible.builtin.include_role: + name: mariadb_copy + tasks_from: env_vars_src.yaml + +# NOTE(bogdando): env variables must be used to keep this consistent with documentation, +# where the stored values need to be compared with post-adoption ones w/o using ansible specifics +- name: test connection to the original DB + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_DATABASES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;') + echo $PULL_OPENSTACK_CONFIGURATION_DATABASES + register: _databases_check + +- name: run mysqlcheck on the original DB to look for things that are not OK + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK) + echo $PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK + register: _mysqlnok_check + +- name: get source Nova services topology specific configuration + no_log: "{{ use_no_log }}" + block: + - name: get Nova cells mappings from database + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') + echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS + register: _novadb_mapped_cells_check + + - name: get Nova instances cell_ids from database + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from nova_api.instance_mappings;") + echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS + register: _novadb_instances_cell_ids_check + + - name: get the host names of the registered Nova compute services + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select host from nova.services where services.binary='nova-compute';") + echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES + register: _nova_compute_hostnames_check + + - name: get the list of mapped Nova cells + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ pull_openstack_configuration_ssh_shell_vars }} + PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells) + echo $PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS + register: _nova_cell_mappings_check + +- name: set facts for pulled openstack services configuration + ansible.builtin.set_fact: + pull_openstack_configuration_databases: "{{ _databases_check.stdout }}" + pull_openstack_configuration_mysqlcheck_nok: " {{ _mysqlnok_check.stdout }}" + pull_openstack_configuration_novadb_mapped_cells: "{{ _novadb_mapped_cells_check.stdout }}" + pull_openstack_configuration_novadb_instances_cell_ids: "{{ _novadb_instances_cell_ids_check.stdout }}" + pull_openstack_configuration_nova_compute_hostnames: "{{ _nova_compute_hostnames_check.stdout }}" + pull_openstack_configuration_novamanage_cell_mappings: "{{ _nova_cell_mappings_check.stdout }}"