diff --git a/docs/openstack/edpm_adoption.md b/docs/openstack/edpm_adoption.md index 6e5958c19..1ae507efb 100644 --- a/docs/openstack/edpm_adoption.md +++ b/docs/openstack/edpm_adoption.md @@ -11,6 +11,9 @@ The values are just illustrative, use values that are correct for your environme ```bash PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) +CONTROLLER_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100" + +alias openstack="oc exec -t openstackclient -- openstack" ``` ## Pre-checks @@ -443,3 +446,21 @@ later on. oc exec -it nova-cell1-conductor-0 -- nova-manage db online_data_migrations ``` +* Verify no Nova compute dataplane disruptions during the adoption/upgrade process: + + ```bash + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + +* Verify if Nova services control the existing VM instance: + + ```bash + openstack server list | grep -qF '| test | ACTIVE |' && openstack server stop test + openstack server list | grep -qF '| test | SHUTOFF |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 shut off' + + openstack server list | grep -qF '| test | SHUTOFF |' && openstack server start test + openstack server list | grep -F '| test | ACTIVE |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + Note that in this guide, the same host acts as a controller, and also as a compute. diff --git a/docs/openstack/mariadb_copy.md b/docs/openstack/mariadb_copy.md index 0bd0627f7..e89af417c 100644 --- a/docs/openstack/mariadb_copy.md +++ b/docs/openstack/mariadb_copy.md @@ -18,6 +18,8 @@ cluster. * Podified MariaDB and RabbitMQ are running. No other podified control plane services are running. + * Required services specific topology [configuration collected](pull_openstack_configuration.md#get-services-topology-specific-configuration) + * OpenStack services have been [stopped](stop_openstack_services.md) * There must be network routability between: @@ -40,38 +42,28 @@ Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment: ```bash -MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified - PODIFIED_MARIADB_IP=$(oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}') PODIFIED_CELL1_MARIADB_IP=$(oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}') PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) -# Replace with your environment's MariaDB IP: -SOURCE_MARIADB_IP=192.168.122.100 -SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') - # The CHARACTER_SET and collation should match the source DB # if the do not then it will break foreign key relationships # for any tables that are created in the future as part of db sync CHARACTER_SET=utf8 COLLATION=utf8_general_ci +MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified +# Replace with your environment's MariaDB IP: +SOURCE_MARIADB_IP=192.168.122.100 +SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') ``` ## Pre-checks -* Test connection to the original DB (show databases): +* Get the count of not-OK source databases: ```bash - podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ - mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;' - ``` - -* Run mysqlcheck on the original DB to look for things that are not OK: - - ```bash - podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK + test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" ``` * Test connection to podified DBs (show databases): @@ -169,20 +161,52 @@ COLLATION=utf8_general_ci ## Post-checks +Compare the following outputs with the topology specific configuration +[collected earlier](pull_openstack_configuration.md#get-services-topology-specific-configuration): + * Check that the databases were imported correctly: ```bash oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep keystone + | grep -q keystone + # ensure neutron db is renamed from ovs_neutron oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep neutron + | grep -q neutron + echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep ' ovs_neutron ' + # ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep nova_cell1 + | grep -q nova_cell1 + + # ensure default cell renamed to cell1, and the cell UUIDs retained intact + NOVADB_MAPPED_CELLS=$(oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ + nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') + uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' + left_behind=$(comm -23 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) + changed=$(comm -13 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) + test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 + default=$(grep -E ' default$' <<<$left_behind) + test $(grep -Ec ' \S+$' <<<$changed) -eq 1 + grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed + + # ensure the registered Nova compute service name has not changed + oc rsh mariadb-openstack-cell1 mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_cell1 \ + -e "select host from services where services.binary='nova-compute';" \ + | grep -qF '| standalone.localdomain |' + echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES | grep -qF ' standalone.localdomain' + + # ensure the VM instance cell ID has not changed + oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from instance_mappings;" | grep -qF -m1 ' 2 |' + echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS | grep -q -m1 ' 2$' ``` * During the pre/post checks the pod `mariadb-client` might have returned a pod security warning diff --git a/docs/openstack/nova_adoption.md b/docs/openstack/nova_adoption.md index 31697ef5d..69460473a 100644 --- a/docs/openstack/nova_adoption.md +++ b/docs/openstack/nova_adoption.md @@ -118,3 +118,36 @@ alias openstack="oc exec -t openstackclient -- openstack" openstack endpoint list | grep nova openstack server list ``` + +Compare the following outputs with the topology specific configuration +[collected earlier](pull_openstack_configuration.md#get-services-topology-specific-configuration): + +* Query the superconductor for cell1 existance: + + ```bash + oc rsh nova-cell0-conductor-0 nova-manage cell_v2 list_cells | grep -F '| cell1 |' + ``` + + The expected changes to happen: + * cell1's `nova` DB and user name become `nova_cell1`. + * Default cell is renamed to `cell1` (in a multi-cell setup, it should become indexed as the last cell instead). + * RabbitMQ transport URL no longer uses `guest`. + +* Verify no Nova compute dataplane disruptions during the adoption/upgrade process: + + ```bash + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + + * Verify if Nova services control the existing VM instance: + + ```bash + openstack server list | grep -qF '| test | ACTIVE |' && openstack server stop test + openstack server list | grep -qF '| test | SHUTOFF |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 shut off' + + openstack server list | grep -qF '| test | SHUTOFF |' && openstack server start test + openstack server list | grep -F '| test | ACTIVE |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + ``` + Note that in this guide, the same host acts as a controller, and also as a compute. diff --git a/docs/openstack/pull_openstack_configuration.md b/docs/openstack/pull_openstack_configuration.md index 7169ddc67..808a17f44 100644 --- a/docs/openstack/pull_openstack_configuration.md +++ b/docs/openstack/pull_openstack_configuration.md @@ -8,7 +8,7 @@ missed or misconfigured. Make sure you have pull the os-diff repository and configure according to your environment: -[Configure os-diff](planning.md#Configuration tooling) +[Configure os-diff](planning.md#configuration-tooling) ## Pull configuration from a TripleO deployment @@ -45,3 +45,61 @@ Once the ansible playbook has been run, you should have into your local director ▾ collect_tripleo_configs/ ▾ glance/ ``` + +## Get services topology specific configuration + +Define the shell variables used in the steps below. The values are +just illustrative, use values that are correct for your environment: + +```bash +MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified +SOURCE_MARIADB_IP=192.168.122.100 +SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') +``` + +Note the following outputs to compare it with post-adoption values later on: + +* Test connection to the original DB: + + ```bash + PULL_OPENSTACK_CONFIGURATION_DATABASES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;') + ``` + Note the `nova`, `nova_api`, `nova_cell0` databases residing in the same DB host. + +* Run mysqlcheck on the original DB to look for things that are not OK: + + ```bash + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK) + ``` + +* Get Nova cells mappings from database: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') + ``` + +* Get Nova instances cell_ids from database: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from nova_api.instance_mappings;") + ``` + +* Get the host names of the registered Nova compute services: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select host from nova.services where services.binary='nova-compute';") + ``` + +* Get the list of mapped Nova cells: + + ```bash + PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells) + ``` diff --git a/docs/openstack/stop_openstack_services.md b/docs/openstack/stop_openstack_services.md index 456bf303e..9fedc93fb 100644 --- a/docs/openstack/stop_openstack_services.md +++ b/docs/openstack/stop_openstack_services.md @@ -58,6 +58,9 @@ openstack volume backup list --all-projects -c ID -c Status |grep -E '\| .+ing \ openstack image list -c ID -c Status |grep -E '\| .+ing \|' ``` +Also collect the [topology configuration](pull_openstack_configuration.md#get-services-topology-specific-configuration), +before stopping services required to gather it live. You will need it to compare it +with the post-adoption values later on. ## Stopping control plane services diff --git a/mkdocs.yml b/mkdocs.yml index c959a208f..45cbf0a18 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -29,7 +29,7 @@ nav: - OpenStack: - openstack/planning.md - openstack/backend_services_deployment.md - - oepsntack/pull_openstack_configuration.md + - openstack/pull_openstack_configuration.md - openstack/stop_openstack_services.md - openstack/mariadb_copy.md - openstack/ovn_adoption.md diff --git a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml index b2326d041..e4f80cde7 100644 --- a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml +++ b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml @@ -1,8 +1,7 @@ -- name: set podified MariaDB copy shell vars - no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - mariadb_copy_shell_vars: | - PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" +- name: get dst database service environment variables + ansible.builtin.include_role: + name: mariadb_copy + tasks_from: env_vars_dst.yaml - name: configure pre-FFU workarounds for Nova compute EDPM services to update its version records ansible.builtin.shell: | @@ -48,7 +47,7 @@ ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} - {{ mariadb_copy_shell_vars }} + {{ mariadb_copy_shell_vars_dst }} oc exec -it mariadb-openstack-cell1 -- mysql --user=root --password=${PODIFIED_DB_ROOT_PASSWORD} \ -e "select a.version from nova_cell1.services a join nova_cell1.services b where a.version!=b.version and a.binary='nova-compute';" register: records_check_results @@ -187,3 +186,7 @@ retries: 10 delay: 6 +- name: Adopted Nova FFU post-checks + ansible.builtin.include_role: + name: nova_adoption + tasks_from: verify.yaml diff --git a/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml b/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml new file mode 100644 index 000000000..5e49114bf --- /dev/null +++ b/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml @@ -0,0 +1,27 @@ +- name: get podified MariaDB service cluster IP + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}' + register: podified_mariadb_ip_result + +- name: get podified cell1 MariaDB IP + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}' + register: podified_cell1_mariadb_ip_result + +- name: set MariaDB copy shell vars + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + mariadb_copy_shell_vars_dst: | + PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} + PODIFIED_CELL1_MARIADB_IP={{ podified_cell1_mariadb_ip_result.stdout }} + PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" + + # The CHARACTER_SET and collation should match the source DB + # if the do not then it will break foreign key relationships + # for any tables that are created in the future as part of db sync + CHARACTER_SET=utf8 + COLLATION=utf8_general_ci diff --git a/tests/roles/mariadb_copy/tasks/env_vars_src.yaml b/tests/roles/mariadb_copy/tasks/env_vars_src.yaml new file mode 100644 index 000000000..66bbd5312 --- /dev/null +++ b/tests/roles/mariadb_copy/tasks/env_vars_src.yaml @@ -0,0 +1,8 @@ +- name: set src MariaDB copy shell vars + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + mariadb_copy_shell_vars_src: | + MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified + # TODO: remove the default(external_...) when CI is transitioned to use 'source_...' + SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }} + SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}" diff --git a/tests/roles/mariadb_copy/tasks/main.yaml b/tests/roles/mariadb_copy/tasks/main.yaml index f284c3fd6..ad9155aed 100644 --- a/tests/roles/mariadb_copy/tasks/main.yaml +++ b/tests/roles/mariadb_copy/tasks/main.yaml @@ -1,38 +1,19 @@ -- name: get podified MariaDB service cluster IP - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - oc get svc --selector "cr=mariadb-openstack" -ojsonpath='{.items[0].spec.clusterIP}' - register: podified_mariadb_ip_result +- name: get the source database service environment variables + ansible.builtin.include_tasks: + file: env_vars_src.yaml -- name: get podified cell1 MariaDB IP - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - oc get svc --selector "cr=mariadb-openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}' - register: podified_cell1_mariadb_ip_result +- name: get the destination database service environment variables + ansible.builtin.include_tasks: + file: env_vars_dst.yaml -- name: set MariaDB copy shell vars +- name: Get the count of not-OK source databases no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - mariadb_copy_shell_vars: | - MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified - - PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} - PODIFIED_CELL1_MARIADB_IP={{podified_cell1_mariadb_ip_result.stdout}} - PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" - - # TODO: remove the default(external_...) when CI is transitioned to use 'source_...' - SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }} - SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}" - - # The CHARACTER_SET and collation should match the source DB - # if the do not then it will break foreign key relationships - # for any tables that are created in the future as part of db sync - CHARACTER_SET=utf8 - COLLATION=utf8_general_ci + environment: + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK: "{{ pull_openstack_configuration_mysqlcheck_nok }}" + ansible.builtin.shell: + cmd: test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" -- name: MariaDB copy pre checks +- name: test connection to podified DBs (show databases) no_log: "{{ use_no_log }}" ansible.builtin.shell: cmd: "{{ lookup('ansible.builtin.template', 'pre_checks.bash') }}" @@ -49,5 +30,10 @@ - name: MariaDB post-checks no_log: "{{ use_no_log }}" + environment: + PULL_OPENSTACK_CONFIGURATION_DATABASES: "{{ pull_openstack_configuration_databases }}" + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS: "{{ pull_openstack_configuration_novadb_mapped_cells }}" + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES: "{{ pull_openstack_configuration_nova_compute_hostnames }}" + PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS: "{{ pull_openstack_configuration_novadb_instances_cell_ids }}" ansible.builtin.shell: cmd: "{{ lookup('ansible.builtin.template', 'post_checks.bash') }}" diff --git a/tests/roles/mariadb_copy/templates/dump_dbs.bash b/tests/roles/mariadb_copy/templates/dump_dbs.bash index 3c73ed20d..45266d03f 100644 --- a/tests/roles/mariadb_copy/templates/dump_dbs.bash +++ b/tests/roles/mariadb_copy/templates/dump_dbs.bash @@ -1,7 +1,7 @@ #!/bin/bash {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_src }} mkdir -p {{ mariadb_copy_tmp_dir }} cd {{ mariadb_copy_tmp_dir }} diff --git a/tests/roles/mariadb_copy/templates/post_checks.bash b/tests/roles/mariadb_copy/templates/post_checks.bash index dbe92c043..e71d2e696 100644 --- a/tests/roles/mariadb_copy/templates/post_checks.bash +++ b/tests/roles/mariadb_copy/templates/post_checks.bash @@ -1,14 +1,44 @@ {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_src }} +{{ mariadb_copy_shell_vars_dst }} oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep keystone + | grep -q keystone + # ensure neutron db is renamed from ovs_neutron oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep neutron + | grep -q neutron +echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep ' ovs_neutron ' + # ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ - | grep nova_cell1 \ No newline at end of file + | grep -q nova_cell1 + +# ensure default cell renamed to cell1, and the cell UUIDs retained intact +NOVADB_MAPPED_CELLS=$(oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ + nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') +uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' +left_behind=$(comm -23 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) +changed=$(comm -13 \ + <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ + <(echo $NOVADB_MAPPED_CELLS | tr -s "| " " " | grep -oE " $uuidf \S+")) +test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 +default=$(grep -E ' default$' <<<$left_behind) +test $(grep -Ec ' \S+$' <<<$changed) -eq 1 +grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed + +# ensure the registered Nova compute service name has not changed +oc rsh mariadb-openstack-cell1 mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_cell1 \ + -e "select host from services where services.binary='nova-compute';" \ + | grep -qF '| standalone.localdomain |' +echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES | grep -qF ' standalone.localdomain' + +# ensure the VM instance cell ID has not changed +oc rsh mariadb-openstack mysql -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from instance_mappings;" | grep -qF -m1 ' 2 |' +echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS | grep -q -m1 ' 2$' diff --git a/tests/roles/mariadb_copy/templates/pre_checks.bash b/tests/roles/mariadb_copy/templates/pre_checks.bash index 9f61b135c..3c4d62afc 100644 --- a/tests/roles/mariadb_copy/templates/pre_checks.bash +++ b/tests/roles/mariadb_copy/templates/pre_checks.bash @@ -1,14 +1,8 @@ #!/bin/bash {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_dst }} -# Test connection to the original DB (show databases) -podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ - mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;' -# Run mysqlcheck on the original DB to look for things that are not OK -podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" # Test connection to podified DBs (show databases) oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' diff --git a/tests/roles/mariadb_copy/templates/restore_dbs.bash b/tests/roles/mariadb_copy/templates/restore_dbs.bash index bba3dba2c..062dd548f 100755 --- a/tests/roles/mariadb_copy/templates/restore_dbs.bash +++ b/tests/roles/mariadb_copy/templates/restore_dbs.bash @@ -1,7 +1,8 @@ #!/bin/bash {{ shell_header }} {{ oc_header }} -{{ mariadb_copy_shell_vars }} +{{ mariadb_copy_shell_vars_src }} +{{ mariadb_copy_shell_vars_dst }} cd {{ mariadb_copy_tmp_dir }} # db schemas to rename on import diff --git a/tests/roles/nova_adoption/tasks/main.yaml b/tests/roles/nova_adoption/tasks/main.yaml index 6a7ae7b6d..a371690c9 100644 --- a/tests/roles/nova_adoption/tasks/main.yaml +++ b/tests/roles/nova_adoption/tasks/main.yaml @@ -97,3 +97,7 @@ until: nova_responding_result is success retries: 60 delay: 2 + +- name: Nova adoption post-checks + ansible.builtin.include_tasks: + file: verify.yaml diff --git a/tests/roles/nova_adoption/tasks/verify.yaml b/tests/roles/nova_adoption/tasks/verify.yaml new file mode 100644 index 000000000..58f212ce8 --- /dev/null +++ b/tests/roles/nova_adoption/tasks/verify.yaml @@ -0,0 +1,37 @@ +- name: set Nova services shell vars + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + nova_header: | + alias openstack="oc exec -t openstackclient -- openstack" + CONTROLLER_SSH="{{ controller1_ssh }}" + +# TODO(bogdando): provide automated checks for 'The expected changes to happen' +- name: query the superconductor for cell1 existance + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc rsh nova-cell0-conductor-0 nova-manage cell_v2 list_cells | grep -F '| cell1 |' + +- name: verify no Nova compute dataplane disruptions during the adoption/upgrade process + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ nova_header }} + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + +- name: verify if Nova services control the existing VM instance + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ nova_header }} + ${BASH_ALIASES[openstack]} server list | grep -qF '| test | ACTIVE |' && ${BASH_ALIASES[openstack]} server stop test + ${BASH_ALIASES[openstack]} server list | grep -qF '| test | SHUTOFF |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 shut off' + + ${BASH_ALIASES[openstack]} server list | grep -qF '| test | SHUTOFF |' && ${BASH_ALIASES[openstack]} server start test + ${BASH_ALIASES[openstack]} server list | grep -F '| test | ACTIVE |' + $CONTROLLER_SSH sudo podman exec -it libvirt_virtqemud virsh list --all | grep 'instance-00000001 running' + register: nova_verify_result + until: nova_verify_result is success + retries: 10 + delay: 6 diff --git a/tests/roles/pull_openstack_configuration/tasks/main.yaml b/tests/roles/pull_openstack_configuration/tasks/main.yaml new file mode 100644 index 000000000..8af393f85 --- /dev/null +++ b/tests/roles/pull_openstack_configuration/tasks/main.yaml @@ -0,0 +1,88 @@ +- name: set shell vars pull openstack configuration ssh commands + no_log: "{{ use_no_log }}" + ansible.builtin.set_fact: + pull_openstack_configuration_ssh_shell_vars: | + CONTROLLER_SSH="{{ controller1_ssh }}" + +- name: get src database service environment variables + ansible.builtin.include_role: + name: mariadb_copy + tasks_from: env_vars_src.yaml + +# NOTE(bogdando): env variables must be used to keep this consistent with documentation, +# where the stored values need to be compared with post-adoption ones w/o using ansible specifics +- name: test connection to the original DB + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_DATABASES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;') + echo $PULL_OPENSTACK_CONFIGURATION_DATABASES + register: _databases_check + +- name: run mysqlcheck on the original DB to look for things that are not OK + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE \ + mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK) + echo $PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK + register: _mysqlnok_check + +- name: get source Nova services topology specific configuration + no_log: "{{ use_no_log }}" + block: + - name: get Nova cells mappings from database + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') + echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS + register: _novadb_mapped_cells_check + + - name: get Nova instances cell_ids from database + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select cell_id from nova_api.instance_mappings;") + echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_INSTANCES_CELL_IDS + register: _novadb_instances_cell_ids_check + + - name: get the host names of the registered Nova compute services + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_copy_shell_vars_src }} + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(podman run -i --rm --userns=keep-id -u $UID $MARIADB_IMAGE mysql \ + -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ + "select host from nova.services where services.binary='nova-compute';") + echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES + register: _nova_compute_hostnames_check + + - name: get the list of mapped Nova cells + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ pull_openstack_configuration_ssh_shell_vars }} + PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells) + echo $PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS + register: _nova_cell_mappings_check + +- name: set facts for pulled openstack services configuration + ansible.builtin.set_fact: + pull_openstack_configuration_databases: "{{ _databases_check.stdout }}" + pull_openstack_configuration_mysqlcheck_nok: " {{ _mysqlnok_check.stdout }}" + pull_openstack_configuration_novadb_mapped_cells: "{{ _novadb_mapped_cells_check.stdout }}" + pull_openstack_configuration_novadb_instances_cell_ids: "{{ _novadb_instances_cell_ids_check.stdout }}" + pull_openstack_configuration_nova_compute_hostnames: "{{ _nova_compute_hostnames_check.stdout }}" + pull_openstack_configuration_novamanage_cell_mappings: "{{ _nova_cell_mappings_check.stdout }}"