From 227f73e2c7bc8ebda641151fef95e9509cfa8b99 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Fri, 10 May 2024 13:21:00 +0200 Subject: [PATCH] Multi-cell database/mq adoption Provide a static multi-cell config for databases and messaging for adoption guide and tests, which comprises a 3 cells. Keep renaming 'default' cell consistent for single and multi cells: Default becomes cellX (or it can be imported as is, for a multi-cell case only) cell1 becomes mapped to openstack-cell1 osdp node set cell2 becomes mapped to openstack-cell2 osdp node set, etc. cellX (X=3 here) becomes mapped to openstack-cell3. Alternatively, default cell retains its name for the openstack-default osdpns mapping Evaluate podified MariaDB passwords for cells from osp-secret to align the tests with documented commands. Remove no longer needed podified DB password variable. Make ansible and shell variables compute cells aware. Rework vars and secrets YAML values for the source and edpm nodes to not confuse its different naming schemes for cells in OSP/TripleO and RHOSO. Remove cached fact for pulled OSP configuration as it can no longer be generated in a multi-cell setup, where related shell variables become bash arrays. Simplify ENV headers management by collecting in a single place. Adjust storage/storageRequests values to make it better fitting a multi-cell test scenarios. Also provide values in docs and add a comment to adjust them as needed. Remove source_db_root_password as it is directly evaluated from tripleo passwords into an env var. Run mysql commands in individual pods. Finished pods take time to terminate, avoid errors where consequent mysql commands failing because the old and new pod use the same name. Signed-off-by: Bohdan Dobrelia --- docs_dev/assemblies/tests.adoc | 6 +- .../proc_deploying-backend-services.adoc | 116 ++++--- ...rating-databases-to-mariadb-instances.adoc | 326 +++++++++++------- ...pology-specific-service-configuration.adoc | 153 +++++--- ...cture-management-and-compute-services.adoc | 20 +- .../proc_stopping-openstack-services.adoc | 24 +- tests/roles/backend_services/tasks/main.yaml | 23 +- .../tasks/ospdo_backend_services.yaml | 1 + .../templates/openstack_control_plane.j2 | 39 ++- .../roles/common_defaults/defaults/main.yaml | 112 +++++- .../dataplane_adoption/tasks/nova_ffu.yaml | 5 - .../tasks/main.yaml | 114 +++--- tests/roles/mariadb_copy/defaults/main.yaml | 2 - .../mariadb_copy/tasks/env_vars_dst.yaml | 27 -- .../mariadb_copy/tasks/env_vars_src.yaml | 15 - .../tasks/env_vars_src_ospdo.yaml | 12 +- tests/roles/mariadb_copy/tasks/main.yaml | 41 +-- .../mariadb_copy/tasks/mariadb_verify.yaml | 8 - .../mariadb_copy/templates/dump_dbs.bash | 22 +- .../mariadb_copy/templates/post_checks.bash | 46 +-- .../mariadb_copy/templates/pre_checks.bash | 13 +- .../mariadb_copy/templates/restore_dbs.bash | 111 +++--- tests/roles/pcp_cleanup/tasks/main.yaml | 5 + tests/secrets.sample.yaml | 44 +-- tests/vars.sample.yaml | 26 +- 25 files changed, 818 insertions(+), 493 deletions(-) delete mode 100644 tests/roles/mariadb_copy/tasks/env_vars_dst.yaml delete mode 100644 tests/roles/mariadb_copy/tasks/env_vars_src.yaml diff --git a/docs_dev/assemblies/tests.adoc b/docs_dev/assemblies/tests.adoc index 6300d8609..4203a436d 100644 --- a/docs_dev/assemblies/tests.adoc +++ b/docs_dev/assemblies/tests.adoc @@ -25,8 +25,10 @@ work out of the box. The comments in the YAML files will guide you regarding the expected values. You may want to double check that these variables suit your environment: ** `install_yamls_path` - ** `tripleo_passwords` - ** `controller*_ssh` + ** `controller*_ssh` (for each {OpenStackPreviousInstaller} controller in each Heat stack on the source cloud) + ** `tripleo_passwords` (for each {OpenStackPreviousInstaller} Heat stack on the source cloud) + ** `source_galera_members` (for each cell controller on the source cloud) + ** `source_mariadb_ip` (for each cell controller on the source cloud) ** `edpm_privatekey_path` ** `timesync_ntp_servers` diff --git a/docs_user/modules/proc_deploying-backend-services.adoc b/docs_user/modules/proc_deploying-backend-services.adoc index e1e43f8bf..631a6871a 100644 --- a/docs_user/modules/proc_deploying-backend-services.adoc +++ b/docs_user/modules/proc_deploying-backend-services.adoc @@ -42,12 +42,9 @@ ADMIN_PASSWORD=SomePassword To use the existing {OpenStackShort} deployment password: + ---- -ifeval::["{build}" != "downstream"] -ADMIN_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' AdminPassword:' | awk -F ': ' '{ print $2; }') -endif::[] -ifeval::["{build}" == "downstream"] -ADMIN_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' AdminPassword:' | awk -F ': ' '{ print $2; }') -endif::[] +declare -A TRIPLEO_PASSWORDS +TRIPLEO_PASSWORDS[default]="$HOME/overcloud-passwords.yaml" +ADMIN_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' AdminPassword:' | awk -F ': ' '{ print $2; }') ---- * Set the service password variables to match the original deployment. Database passwords can differ in the control plane environment, but @@ -56,40 +53,22 @@ you must synchronize the service account passwords. For example, in developer environments with {OpenStackPreviousInstaller} Standalone, the passwords can be extracted: + ---- -ifeval::["{build}" != "downstream"] -AODH_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' AodhPassword:' | awk -F ': ' '{ print $2; }') -BARBICAN_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' BarbicanPassword:' | awk -F ': ' '{ print $2; }') -CEILOMETER_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' CeilometerPassword:' | awk -F ': ' '{ print $2; }') -CINDER_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' CinderPassword:' | awk -F ': ' '{ print $2; }') -GLANCE_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' GlancePassword:' | awk -F ': ' '{ print $2; }') -HEAT_AUTH_ENCRYPTION_KEY=$(cat ~/tripleo-standalone-passwords.yaml | grep ' HeatAuthEncryptionKey:' | awk -F ': ' '{ print $2; }') -HEAT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' HeatPassword:' | awk -F ': ' '{ print $2; }') -HEAT_STACK_DOMAIN_ADMIN_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' HeatStackDomainAdminPassword:' | awk -F ': ' '{ print $2; }') -IRONIC_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' IronicPassword:' | awk -F ': ' '{ print $2; }') -MANILA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' ManilaPassword:' | awk -F ': ' '{ print $2; }') -NEUTRON_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' NeutronPassword:' | awk -F ': ' '{ print $2; }') -NOVA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' NovaPassword:' | awk -F ': ' '{ print $2; }') -OCTAVIA_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' OctaviaPassword:' | awk -F ': ' '{ print $2; }') -PLACEMENT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' PlacementPassword:' | awk -F ': ' '{ print $2; }') -SWIFT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' SwiftPassword:' | awk -F ': ' '{ print $2; }') -endif::[] -ifeval::["{build}" == "downstream"] -AODH_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' AodhPassword:' | awk -F ': ' '{ print $2; }') -BARBICAN_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' BarbicanPassword:' | awk -F ': ' '{ print $2; }') -CEILOMETER_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' CeilometerPassword:' | awk -F ': ' '{ print $2; }') -CINDER_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' CinderPassword:' | awk -F ': ' '{ print $2; }') -GLANCE_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' GlancePassword:' | awk -F ': ' '{ print $2; }') -HEAT_AUTH_ENCRYPTION_KEY=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' HeatAuthEncryptionKey:' | awk -F ': ' '{ print $2; }') -HEAT_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' HeatPassword:' | awk -F ': ' '{ print $2; }') -HEAT_STACK_DOMAIN_ADMIN_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' HeatStackDomainAdminPassword:' | awk -F ': ' '{ print $2; }') -IRONIC_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' IronicPassword:' | awk -F ': ' '{ print $2; }') -MANILA_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' ManilaPassword:' | awk -F ': ' '{ print $2; }') -NEUTRON_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' NeutronPassword:' | awk -F ': ' '{ print $2; }') -NOVA_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' NovaPassword:' | awk -F ': ' '{ print $2; }') -OCTAVIA_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' OctaviaPassword:' | awk -F ': ' '{ print $2; }') -PLACEMENT_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' PlacementPassword:' | awk -F ': ' '{ print $2; }') -SWIFT_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' SwiftPassword:' | awk -F ': ' '{ print $2; }') -endif::[] +AODH_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' AodhPassword:' | awk -F ': ' '{ print $2; }') +BARBICAN_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' BarbicanPassword:' | awk -F ': ' '{ print $2; }') +CEILOMETER_METERING_SECRET=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' CeilometerMeteringSecret:' | awk -F ': ' '{ print $2; }') +CEILOMETER_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' CeilometerPassword:' | awk -F ': ' '{ print $2; }') +CINDER_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' CinderPassword:' | awk -F ': ' '{ print $2; }') +GLANCE_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' GlancePassword:' | awk -F ': ' '{ print $2; }') +HEAT_AUTH_ENCRYPTION_KEY=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' HeatAuthEncryptionKey:' | awk -F ': ' '{ print $2; }') +HEAT_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' HeatPassword:' | awk -F ': ' '{ print $2; }') +HEAT_STACK_DOMAIN_ADMIN_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' HeatStackDomainAdminPassword:' | awk -F ': ' '{ print $2; }') +IRONIC_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' IronicPassword:' | awk -F ': ' '{ print $2; }') +MANILA_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' ManilaPassword:' | awk -F ': ' '{ print $2; }') +NEUTRON_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' NeutronPassword:' | awk -F ': ' '{ print $2; }') +NOVA_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' NovaPassword:' | awk -F ': ' '{ print $2; }') +OCTAVIA_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' OctaviaPassword:' | awk -F ': ' '{ print $2; }') +PLACEMENT_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' PlacementPassword:' | awk -F ': ' '{ print $2; }') +SWIFT_PASSWORD=$(cat ${TRIPLEO_PASSWORDS[default]} | grep ' SwiftPassword:' | awk -F ': ' '{ print $2; }') ---- .Procedure @@ -289,12 +268,19 @@ endif::[] openstack: secret: osp-secret replicas: 3 - storageRequest: 500M + storageRequest: 5G openstack-cell1: secret: osp-secret replicas: 3 - storageRequest: 500M - + storageRequest: 5G + openstack-cell2: + secret: osp-secret + replicas: 1 + storageRequest: 5G + openstack-cell3: + secret: osp-secret + replicas: 1 + storageRequest: 5G memcached: enabled: true templates: @@ -358,6 +344,8 @@ endif::[] spec: type: LoadBalancer rabbitmq-cell1: + persistence: + storage: 1G override: service: metadata: @@ -372,7 +360,28 @@ ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] endif::[] spec: type: LoadBalancer - + rabbitmq-cell2: + persistence: + storage: 1G + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.87 + spec: + type: LoadBalancer + rabbitmq-cell3: + persistence: + storage: 1G + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.88 + spec: + type: LoadBalancer telemetry: enabled: false @@ -396,11 +405,24 @@ endif::[] + <1> Select an existing storage class in your {OpenShiftShort} cluster. +This example provides required infrastructure database and messaging services for a 3 compute cells +named `cell1`, `cell2`, and `cell3`. Adjust the names, counts, IP addresses, and numbers, +like `replicas`, `storage`, or `storageRequest`, as needed. + .Verification -* Verify that MariaDB is running: +* Verify that MariaDB and RabbitMQ are running, for all defined cells: + ---- -$ oc get pod openstack-galera-0 -o jsonpath='{.status.phase}{"\n"}' -$ oc get pod openstack-cell1-galera-0 -o jsonpath='{.status.phase}{"\n"}' +$ RENAMED_CELLS="cell1 cell2 cell3" +$ oc get pod openstack-galera-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running +$ oc get pod rabbitmq-server-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running +$ for CELL in $(echo $RENAMED_CELLS); do + oc get pod openstack-$CELL-galera-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running + oc get pod rabbitmq-$CELL-server-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running +done ---- +Futher on, we will be referring to the given cells names via an environment +variable `RENAMED_CELLS`. + +* Verify that you can access the `OpenStackClient` pod. For more information, see link:{defaultURL}/maintaining_the_red_hat_openstack_services_on_openshift_deployment/assembly_accessing-the-rhoso-cloud#proc_accessing-the-OpenStackClient-pod_cloud-access-admin[Accessing the OpenStackClient pod] in _Maintaining the {rhos_long_noacro} deployment_. diff --git a/docs_user/modules/proc_migrating-databases-to-mariadb-instances.adoc b/docs_user/modules/proc_migrating-databases-to-mariadb-instances.adoc index aaf6891a2..5073afb30 100644 --- a/docs_user/modules/proc_migrating-databases-to-mariadb-instances.adoc +++ b/docs_user/modules/proc_migrating-databases-to-mariadb-instances.adoc @@ -19,16 +19,6 @@ Migrate your databases from the original {rhos_prev_long} ({OpenStackShort}) dep * Define the following shell variables. Replace the following example values with values that are correct for your environment: + ---- -PODIFIED_MARIADB_IP=$(oc get svc --selector "mariadb/name=openstack" -ojsonpath='{.items[0].spec.clusterIP}') -PODIFIED_CELL1_MARIADB_IP=$(oc get svc --selector "mariadb/name=openstack-cell1" -ojsonpath='{.items[0].spec.clusterIP}') -PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) - -# The CHARACTER_SET and collation should match the source DB -# if the do not then it will break foreign key relationships -# for any tables that are created in the future as part of db sync -CHARACTER_SET=utf8 -COLLATION=utf8_general_ci - ifeval::["{build}" != "downstream"] STORAGE_CLASS=crc-csi-hostpath-provisioner MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified @@ -37,29 +27,106 @@ ifeval::["{build}" == "downstream"] STORAGE_CLASS=local-storage MARIADB_IMAGE=registry.redhat.io/rhosp-dev-preview/openstack-mariadb-rhel9:18.0 endif::[] -# Replace with your environment's MariaDB Galera cluster VIP and backend IPs: -SOURCE_MARIADB_IP=172.17.0.2 -declare -A SOURCE_GALERA_MEMBERS -SOURCE_GALERA_MEMBERS=( + +CELLS="default cell1 cell2" +DEFAULT_CELL_NAME="cell3" +RENAMED_CELLS="cell1 cell2 $DEFAULT_CELL_NAME" + +# The CHARACTER_SET and collation should match the source DB +# if the do not then it will break foreign key relationships +# for any tables that are created in the future as part of db sync +CHARACTER_SET=utf8 +COLLATION=utf8_general_ci + +declare -A PODIFIED_DB_ROOT_PASSWORD +for CELL in $(echo "super $RENAMED_CELLS"); do + PODIFIED_DB_ROOT_PASSWORD[$CELL]=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) +done + +declare -A PODIFIED_MARIADB_IP +for CELL in $(echo "super $RENAMED_CELLS"); do + if [ "$CELL" = "super" ]; then + PODIFIED_MARIADB_IP[$CELL]=$(oc get svc --selector "mariadb/name=openstack" -ojsonpath='{.items[0].spec.clusterIP}') + else + PODIFIED_MARIADB_IP[$CELL]=$(oc get svc --selector "mariadb/name=openstack-$CELL" -ojsonpath='{.items[0].spec.clusterIP}') + fi +done + +declare -A TRIPLEO_PASSWORDS +for CELL in $(echo $CELLS); do + if [ "$CELL" = "default" ]; then + TRIPLEO_PASSWORDS[default]="$HOME/overcloud-passwords.yaml" + else + # in a split-stack source cloud, it should take a stack-specific passwords file instead + TRIPLEO_PASSWORDS[$CELL]="$HOME/overcloud-passwords.yaml" + fi +done + +declare -A SOURCE_DB_ROOT_PASSWORD +for CELL in $(echo $CELLS); do + SOURCE_DB_ROOT_PASSWORD[$CELL]=$(cat ${TRIPLEO_PASSWORDS[$CELL]} | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') +done + +declare -A SOURCE_MARIADB_IP +SOURCE_MARIADB_IP[default]=** +SOURCE_MARIADB_IP[cell1]=** +SOURCE_MARIADB_IP[cell2]=** +# ... + +declare -A SOURCE_GALERA_MEMBERS_DEFAULT +SOURCE_GALERA_MEMBERS_DEFAULT=( ["standalone.localdomain"]=172.17.0.100 + # [...]=... +) +declare -A SOURCE_GALERA_MEMBERS_CELL1 +SOURCE_GALERA_MEMBERS_CELL1=( # ... ) -ifeval::["{build}" != "downstream"] -SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') -endif::[] -ifeval::["{build}" == "downstream"] -SOURCE_DB_ROOT_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') -endif::[] +declare -A SOURCE_GALERA_MEMBERS_CELL2 +SOURCE_GALERA_MEMBERS_CELL2=( + # ... +) +# ... ---- + -To get the value to set `SOURCE_MARIADB_IP`, query the puppet-generated configurations in a Controller node: +Here, `CELLS` and `RENAMED_CELLS` represent changes that are going to be made +after importing databases: the `default` cell takes a new name from `DEFAULT_CELL_NAME`. +In a multi-cell adoption scenario, it may retain its original 'default' name as well. + +Note that the `super` is not a cell, but the top-scope Nova +API "upcall" database instance. A super conductor connects to that database. +In this guide, the upcall and cells' databases are going to use the same password +defined in `osp-secret`. Old passwords are only needed to prepare the data exports. + +To get the values for `SOURCE_MARIADB_IP`, query the puppet-generated configurations in a Controller and CellController nodes: + ---- $ sudo grep -rI 'listen mysql' -A10 /var/lib/config-data/puppet-generated/ | grep bind ---- +To get the values for `SOURCE_GALERA_MEMBERS_*`, query the puppet-generated configurations in a Controller and CellController nodes: ++ +---- +$ sudo grep -rI 'listen mysql' -A10 /var/lib/config-data/puppet-generated/ | grep server +---- + +The source cloud always uses the same password for cells' databases by design. +For that reason, we chose to use the same passwords file for all cells' stacks. +There is a split-stack topology, however, that allows using different passwords +files for each stack. + * Prepare the MariaDB adoption helper pod: + +. Create a temporary folder to store the adoption helper pod. Choose storage requests that fit the MySQL database size: +A standalone TripleO only creates a 'default' cell, which should be the only `CELLS` value for such a case +(and `DEFAULT_CELL_NAME` should be `cell1`). + +** For each cell defined in `CELLS` +*** Replace `["standalone.localdomain"]="172.17.0.100"`, and complete `SOURCE_GALERA_MEMBERS_*` with the names of MariaDB Galera cluster members and its IP address. +*** Replace `SOURCE_MARIADB_IP[*]= ...`, and complete the records lists for the cell names and VIP addresses of MariaDB Galera clusters. +*** Replace `SOURCE_GALERA_MEMBERS_*[*]= ...`, and complete the records lists for the cell names and IP addresses of MariaDB Galera cluster members. + . Create a temporary volume claim and a pod for the database data copy. Edit the volume claim storage request if necessary, to give it enough space for the overcloud databases: + [source,yaml] @@ -117,111 +184,133 @@ $ oc wait --for condition=Ready pod/mariadb-copy-data --timeout=30s .Procedure -. Check that the source Galera database cluster members are online and synced: +. Check that the source Galera database cluster(s) members are online and synced: + ---- -for i in "${!SOURCE_GALERA_MEMBERS[@]}"; do - echo "Checking for the database node $i WSREP status Synced" - oc rsh mariadb-copy-data mysql \ - -h "${SOURCE_GALERA_MEMBERS[$i]}" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" \ - -e "show global status like 'wsrep_local_state_comment'" | \ - grep -qE "\bSynced\b" +for CELL in $(echo $CELLS); do + MEMBERS=SOURCE_GALERA_MEMBERS_$(echo ${CELL}|tr '[:lower:]' '[:upper:]')[@] + for i in "${!MEMBERS}"; do + echo "Checking for the database node $i WSREP status Synced" + oc rsh mariadb-copy-data mysql \ + -h "$i" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" \ + -e "show global status like 'wsrep_local_state_comment'" | \ + grep -qE "\bSynced\b" + done done ---- ++ +Each additional Nova v2 cell runs a dedicated Galera database cluster, so the checking is done for all of it. . Get the count of source databases with the `NOK` (not-OK) status: + ---- -$ oc rsh mariadb-copy-data mysql -h "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" -e "SHOW databases;" +for CELL in $(echo $CELLS); do + oc rsh mariadb-copy-data mysql -h "${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" -e "SHOW databases;" +end ---- . Check that `mysqlcheck` had no errors: + ---- -. ~/.source_cloud_exported_variables -test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" || [ "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" = " " ] && echo "OK" || echo "CHECK FAILED" +$ for CELL in $(echo $CELLS); do + set +u + . ~/.source_cloud_exported_variables_$CELL + set -u +done +$ test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" || [ "x$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" = "x " ] && echo "OK" || echo "CHECK FAILED" ---- -. Test the connection to the control plane databases: +. Test the connection to the control plane "upcall" and cells' databases: + ---- -$ oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -rsh "$PODIFIED_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' -$ oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -rsh "$PODIFIED_CELL1_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' +for CELL in $(echo "super $RENAMED_CELLS"); do + oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -rsh "${PODIFIED_MARIADB_IP[$CELL]}" -uroot -p"${PODIFIED_DB_ROOT_PASSWORD[$CELL]}" -e 'SHOW databases;' +done ---- + [NOTE] -You must transition {compute_service_first_ref} services that are imported later into a superconductor architecture by deleting the old service records in the cell databases, starting with `cell1`. New records are registered with different hostnames provided by the {compute_service} operator. All Compute services, except the Compute agent, have no internal state, and their service records can be safely deleted. You also need to rename the former `default` cell to `cell1`. +You must transition {compute_service_first_ref} services that are imported later into a superconductor architecture by deleting the old service records in the cell databases, starting with `cell1`. New records are registered with different hostnames provided by the {compute_service} operator. All Compute services, except the Compute agent, have no internal state, and their service records can be safely deleted. You also need to rename the former `default` cell to `DEFAULT_CELL_NAME`. . Create a dump of the original databases: + ---- -$ oc rsh mariadb-copy-data << EOF - mysql -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \ - -N -e "show databases" | grep -E -v "schema|mysql|gnocchi|aodh" | \ - while read dbname; do - echo "Dumping \${dbname}"; - mysqldump -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \ - --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \ - "\${dbname}" > /backup/"\${dbname}".sql; - done +$ for CELL in $(echo $CELLS); do + oc rsh mariadb-copy-data << EOF + mysql -h"${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" \ + -N -e "show databases" | grep -E -v "schema|mysql|gnocchi|aodh" | \ + while read dbname; do + echo "Dumping $CELL cell \${dbname}"; + mysqldump -h"${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" \ + --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \ + "\${dbname}" > /backup/"${CELL}.\${dbname}".sql; + done EOF +done ---- ++ +Note filtering the information and performance schema tables. +Gnocchi is no longer used as a metric store as well . Restore the databases from `.sql` files into the control plane MariaDB: + ---- +$ for CELL in $(echo $CELLS); do + RCELL=$CELL + [ "$CELL" = "default" ] && RCELL=$DEFAULT_CELL_NAME ifeval::["{OpenStackPreviousInstaller}" != "director_operator"] -$ oc rsh mariadb-copy-data << EOF + oc rsh mariadb-copy-data << EOF endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] -$ oc rsh -n $OSPDO_NAMESPACE mariadb-copy-data << EOF + oc rsh -n $OSPDO_NAMESPACE mariadb-copy-data << EOF endif::[] - # db schemas to rename on import - declare -A db_name_map - db_name_map['nova']='nova_cell1' - db_name_map['ovs_neutron']='neutron' - db_name_map['ironic-inspector']='ironic_inspector' - - # db servers to import into - declare -A db_server_map - db_server_map['default']=${PODIFIED_MARIADB_IP} - db_server_map['nova_cell1']=${PODIFIED_CELL1_MARIADB_IP} - - # db server root password map - declare -A db_server_password_map - db_server_password_map['default']=${PODIFIED_DB_ROOT_PASSWORD} - db_server_password_map['nova_cell1']=${PODIFIED_DB_ROOT_PASSWORD} - - cd /backup - for db_file in \$(ls *.sql); do - db_name=\$(echo \${db_file} | awk -F'.' '{ print \$1; }') - if [[ -v "db_name_map[\${db_name}]" ]]; then - echo "renaming \${db_name} to \${db_name_map[\${db_name}]}" - db_name=\${db_name_map[\${db_name}]} - fi - db_server=\${db_server_map["default"]} - if [[ -v "db_server_map[\${db_name}]" ]]; then - db_server=\${db_server_map[\${db_name}]} - fi - db_password=\${db_server_password_map['default']} - if [[ -v "db_server_password_map[\${db_name}]" ]]; then - db_password=\${db_server_password_map[\${db_name}]} - fi - echo "creating \${db_name} in \${db_server}" - mysql -h"\${db_server}" -uroot "-p\${db_password}" -e \ - "CREATE DATABASE IF NOT EXISTS \${db_name} DEFAULT \ - CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};" - echo "importing \${db_name} into \${db_server}" - mysql -h "\${db_server}" -uroot "-p\${db_password}" "\${db_name}" < "\${db_file}" - done - mysql -h "\${db_server_map['default']}" -uroot -p"\${db_server_password_map['default']}" -e \ - "update nova_api.cell_mappings set name='cell1' where name='default';" - mysql -h "\${db_server_map['nova_cell1']}" -uroot -p"\${db_server_password_map['nova_cell1']}" -e \ - "delete from nova_cell1.services where host not like '%nova-cell1-%' and services.binary != 'nova-compute';" + declare -A db_name_map + db_name_map['nova']="nova_$RCELL" + db_name_map['ovs_neutron']='neutron' + db_name_map['ironic-inspector']='ironic_inspector' + + declare -A db_server_map + db_server_map['default']=${PODIFIED_MARIADB_IP['super']} + db_server_map["nova_$RCELL"]=${PODIFIED_MARIADB_IP[$RCELL]} + + declare -A db_server_password_map + db_server_password_map['default']=${PODIFIED_DB_ROOT_PASSWORD['super']} + db_server_password_map["nova_$RCELL"]=${PODIFIED_DB_ROOT_PASSWORD[$RCELL]} + + cd /backup + for db_file in \$(ls ${CELL}.*.sql); do + db_name=\$(echo \${db_file} | awk -F'.' '{ print \$2; }') + renamed_db_file="${RCELL}_new.\${db_name}.sql" + mv -f \${db_file} \${renamed_db_file} + if [[ -v "db_name_map[\${db_name}]" ]]; then + echo "renaming $CELL cell \${db_name} to $RCELL \${db_name_map[\${db_name}]}" + db_name=\${db_name_map[\${db_name}]} + fi + db_server=\${db_server_map["default"]} + if [[ -v "db_server_map[\${db_name}]" ]]; then + db_server=\${db_server_map[\${db_name}]} + fi + db_password=\${db_server_password_map['default']} + if [[ -v "db_server_password_map[\${db_name}]" ]]; then + db_password=\${db_server_password_map[\${db_name}]} + fi + echo "creating $RCELL cell \${db_name} in \${db_server}" + mysql -h"\${db_server}" -uroot "-p\${db_password}" -e \ + "CREATE DATABASE IF NOT EXISTS \${db_name} DEFAULT \ + CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};" + echo "importing $RCELL cell \${db_name} into \${db_server} from \${renamed_db_file}" + mysql -h "\${db_server}" -uroot "-p\${db_password}" "\${db_name}" < "\${renamed_db_file}" + done + + if [ "$CELL" = "default" ] ; then + mysql -h "\${db_server_map['default']}" -uroot -p"\${db_server_password_map['default']}" -e \ + "update nova_api.cell_mappings set name='$DEFAULT_CELL_NAME' where name='default';" + fi + mysql -h "\${db_server_map["nova_$RCELL"]}" -uroot -p"\${db_server_password_map["nova_$RCELL"]}" -e \ + "delete from nova_${RCELL}.services where host not like '%nova_${RCELL}-%' and services.binary != 'nova-compute';" EOF +done ---- .Verification @@ -232,39 +321,40 @@ For more information, see xref:proc_retrieving-topology-specific-service-configu . Check that the databases are imported correctly: + ---- -. ~/.source_cloud_exported_variables - -# use 'oc exec' and 'mysql -rs' to maintain formatting -dbs=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;') -echo $dbs | grep -Eq '\bkeystone\b' && echo "OK" || echo "CHECK FAILED" - -# ensure neutron db is renamed from ovs_neutron -echo $dbs | grep -Eq '\bneutron\b' -echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep -Eq '\bovs_neutron\b' && echo "OK" || echo "CHECK FAILED" - -# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 -c1dbs=$(oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;') -echo $c1dbs | grep -Eq '\bnova_cell1\b' && echo "OK" || echo "CHECK FAILED" - -# ensure default cell renamed to cell1, and the cell UUIDs retained intact -novadb_mapped_cells=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ +$ set +u +$ . ~/.source_cloud_exported_variables_default +$ set -u +$ dbs=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot -p"${PODIFIED_DB_ROOT_PASSWORD['super']}" -e 'SHOW databases;') +$ echo $dbs | grep -Eq '\bkeystone\b' && echo "OK" || echo "CHECK FAILED" +$ echo $dbs | grep -Eq '\bneutron\b' +$ echo "${PULL_OPENSTACK_CONFIGURATION_DATABASES[@]}" | grep -Eq '\bovs_neutron\b' && echo "OK" || echo "CHECK FAILED" +$ novadb_mapped_cells=$(oc exec openstack-galera-0 -c galera -- mysql -rs -uroot -p"${PODIFIED_DB_ROOT_PASSWORD['super']}" \ nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') -uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' -left_behind=$(comm -23 \ +$ uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' +$ left_behind=$(comm -23 \ <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ <(echo $novadb_mapped_cells | tr -s "| " " " | grep -oE " $uuidf \S+")) -changed=$(comm -13 \ +$ changed=$(comm -13 \ <(echo $PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS | grep -oE " $uuidf \S+") \ <(echo $novadb_mapped_cells | tr -s "| " " " | grep -oE " $uuidf \S+")) -test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 && echo "OK" || echo "CHECK FAILED" -default=$(grep -E ' default$' <<<$left_behind) -test $(grep -Ec ' \S+$' <<<$changed) -eq 1 && echo "OK" || echo "CHECK FAILED" -grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed && echo "OK" || echo "CHECK FAILED" - -# ensure the registered Compute service name has not changed -novadb_svc_records=$(oc exec openstack-cell1-galera-0 -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ - nova_cell1 -e "select host from services where services.binary='nova-compute' order by host asc;") -diff -Z <(echo $novadb_svc_records) <(echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES) && echo "OK" || echo "CHECK FAILED" +$ test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 && echo "OK" || echo "CHECK FAILED" +$ default=$(grep -E ' default$' <<<$left_behind) +$ test $(grep -Ec ' \S+$' <<<$changed) -eq 1 && echo "OK" || echo "CHECK FAILED" +$ grep -qE " $(awk '{print $1}' <<<$default) ${DEFAULT_CELL_NAME}$" <<<$changed && echo "OK" || echo "CHECK FAILED" + +$ for CELL in $(echo $CELLS); do + set +u + . ~/.source_cloud_exported_variables_$CELL + set -u + RCELL=$CELL + [ "$CELL" = "default" ] && RCELL=$DEFAULT_CELL_NAME + c1dbs=$(oc exec openstack-$RCELL-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot -p${PODIFIED_DB_ROOT_PASSWORD[$RCELL]} -e 'SHOW databases;') + echo $c1dbs | grep -Eq "\bnova_${RCELL}\b" && echo "OK" || echo "CHECK FAILED" + + novadb_svc_records=$(oc exec openstack-$RCELL-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot -p${PODIFIED_DB_ROOT_PASSWORD[$RCELL]} \ + nova_$RCELL -e "select host from services where services.binary='nova-compute' order by host asc;") + diff -Z <(echo $novadb_svc_records) <(echo ${PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$RCELL]}) && echo "OK" || echo "CHECK FAILED" +done ---- . Delete the `mariadb-data` pod and the `mariadb-copy-data` persistent volume claim that contains the database backup: diff --git a/docs_user/modules/proc_retrieving-topology-specific-service-configuration.adoc b/docs_user/modules/proc_retrieving-topology-specific-service-configuration.adoc index 74383b786..94ade1c55 100644 --- a/docs_user/modules/proc_retrieving-topology-specific-service-configuration.adoc +++ b/docs_user/modules/proc_retrieving-topology-specific-service-configuration.adoc @@ -16,46 +16,74 @@ Before you migrate your databases to the {rhos_long} control plane, retrieve the + ---- ifeval::["{OpenStackPreviousInstaller}" != "director_operator"] +CELLS="default cell1 cell2" <1> ifeval::["{build}" != "downstream"] CONTROLLER1_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100" MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified endif::[] ifeval::["{build}" == "downstream"] -CONTROLLER1_SSH="ssh -i ** root@**" +CONTROLLER1_SSH="ssh -i ** root@**" <1> MARIADB_IMAGE=registry.redhat.io/rhosp-dev-preview/openstack-mariadb-rhel9:18.0 endif::[] -SOURCE_MARIADB_IP=172.17.0.2 -ifeval::["{build}" != "downstream"] -SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') -endif::[] -ifeval::["{build}" == "downstream"] -SOURCE_DB_ROOT_PASSWORD=$(cat ~/overcloud-deploy/overcloud/overcloud-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') -endif::[] endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] MARIADB_CLIENT_ANNOTATIONS='--annotations=k8s.v1.cni.cncf.io/networks=internalapi' endif::[] +declare -A TRIPLEO_PASSWORDS +for CELL in $(echo $CELLS); do + if [ "$CELL" = "default" ]; then + TRIPLEO_PASSWORDS[default]="$HOME/overcloud-passwords.yaml" + else + # in a split-stack source cloud, it should take a stack-specific passwords file instead + TRIPLEO_PASSWORDS[$CELL]="$HOME/overcloud-passwords.yaml" +fi +done + +declare -A SOURCE_DB_ROOT_PASSWORD +for CELL in $(echo $CELLS); do + SOURCE_DB_ROOT_PASSWORD[$CELL]=$(cat ${TRIPLEO_PASSWORDS[$CELL]} | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') +done + +declare -A SOURCE_MARIADB_IP <2> +SOURCE_MARIADB_IP[default]=** +SOURCE_MARIADB_IP[cell1]=** +SOURCE_MARIADB_IP[cell2]=** +# ... ---- + -To get the value to set `SOURCE_MARIADB_IP`, query the puppet-generated configurations in a Controller node: +<1> Complete `CONTROLLER1_SSH` settings with SSH connection details for any non-cell controller of the source {OpenStackPreviousInstaller} cloud. +<2> For each cell defined in `CELLS`, replace `SOURCE_MARIADB_IP[*]= ...`, and complete the records lists for the cell names and VIP addresses of MariaDB Galera clusters, include cells, of the source {OpenStackPreviousInstaller} cloud. + +To get the values for `SOURCE_MARIADB_IP`, query the puppet-generated configurations in a Controller and CellController node: + ---- $ sudo grep -rI 'listen mysql' -A10 /var/lib/config-data/puppet-generated/ | grep bind ---- +[NOTE] +The source cloud always uses the same password for cells' databases by design. +For that reason, we chose to use the same passwords file for all cells' stacks. +There is a split-stack topology, however, that allows using different passwords +files for each stack. + .Procedure . Export the shell variables for the following outputs and test the connection to the {OpenStackShort} database: + ---- +$ unset PULL_OPENSTACK_CONFIGURATION_DATABASES +$ declare -xA PULL_OPENSTACK_CONFIGURATION_DATABASES +$ for CELL in $(echo $CELLS); do ifeval::["{OpenStackPreviousInstaller}" != "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_DATABASES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + PULL_OPENSTACK_CONFIGURATION_DATABASES[$CELL]=$(oc run mariadb-client-1-$CELL ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_DATABASES=$(oc run mariadb-client --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + PULL_OPENSTACK_CONFIGURATION_DATABASES[$CELL]=$(oc run mariadb-client-1-$CELL --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] - mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;') -echo "$PULL_OPENSTACK_CONFIGURATION_DATABASES" + mysql -rsh "${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" -e 'SHOW databases;') +done + +<-- FIXME: this is broken now --> ---- endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] @@ -127,76 +155,111 @@ $ export PULL_OPENSTACK_CONFIGURATION_DATABASES="$(oc run mariadb-client -q --im endif::[] + [NOTE] -The `nova`, `nova_api`, and `nova_cell0` databases are included in the same database host. +<-- /end FIXME: this is broken now --> +The `nova`, `nova_api`, and `nova_cell0` databases are inlcuded in the same database host for the main overcloud Heat stack. +Additional cells use the `nova` database of their local Galera clusters. . Run `mysqlcheck` on the {OpenStackShort} database to check for inaccuracies: + ---- +$ unset PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK +$ declare -xA PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK +$ run_mysqlcheck() { ifeval::["{OpenStackPreviousInstaller}" != "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client-2-$1 ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client-2-$1 --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root -p"$SOURCE_DB_ROOT_PASSWORD" | grep -v OK) -echo "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" + mysqlcheck --all-databases -h ${SOURCE_MARIADB_IP[$CELL]} -u root -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" | grep -v OK) +} +$ for CELL in $(echo $CELLS); do + run_mysqlcheck $CELL +done +$ if [ "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" != "" ]; then + # Try mysql_upgrade to fix mysqlcheck failure + for CELL in $(echo $CELLS); do + MYSQL_UPGRADE=$(oc run mariadb-client-3 ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql_upgrade -v -h ${SOURCE_MARIADB_IP[$CELL]} -u root -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}") + # rerun mysqlcheck to check if problem is resolved + run_mysqlcheck + done +fi ---- ++ . Get the {compute_service_first_ref} cell mappings: + ---- ifeval::["{OpenStackPreviousInstaller}" != "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ +export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client-1 ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ +export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client-1 --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] - mysql -rsh "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" nova_api -e \ + mysql -rsh "${SOURCE_MARIADB_IP['default]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD['default']}" nova_api -e \ 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') -echo "$PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS" ---- ++ . Get the hostnames of the registered Compute services: + ---- +$ unset PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES +$ declare -xA PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES +$ for CELL in $(echo $CELLS); do ifeval::["{OpenStackPreviousInstaller}" != "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$CELL]=$(oc run mariadb-client-4-$CELL ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ endif::[] ifeval::["{OpenStackPreviousInstaller}" == "director_operator"] -export PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(oc run mariadb-client --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ -endif::[] - mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ - "select host from nova.services where services.binary='nova-compute';") -echo "$PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES" + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$CELL]=$(oc run mariadb-client-4-$CELL --overrides="$RUN_OVERRIDES" -n $OSPDO_NAMESPACE -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh "${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" -e \ + "select host from nova.services where services.binary='nova-compute';") +done ---- . Get the list of the mapped {compute_service} cells: + ---- -export PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER1_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells) -echo "$PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS" +export PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER1_SSH sudo podman exec -it nova_conductor nova-manage cell_v2 list_cells) ---- + [IMPORTANT] After the {OpenStackShort} control plane services are shut down, if any of the exported values are lost, re-running the command fails because the control plane services are no longer running on the source cloud, and the data cannot be retrieved. To avoid data loss, preserve the exported values in an environment file before shutting down the control plane services. -. If `neutron-sriov-nic-agent` agents are running in your {OpenStackShort} deployment, get the configuration to use for the data plane adoption: -+ ----- -SRIOV_AGENTS=$ oc run mariadb-client mysql -rsh "$SOURCE_MARIADB_IP" \ --uroot -p"$SOURCE_DB_ROOT_PASSWORD" ovs_neutron -e \ -"select host, configurations from agents where agents.binary='neutron-sriov-nic-agent';" ----- - . Store the exported variables for future use: + ---- -$ cat >~/.source_cloud_exported_variables < +$ declare -xA PULL_OPENSTACK_CONFIGURATION_DATABASES +$ declare -xA PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK +$ declare -xA PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES +$ for CELL in $(echo $CELLS); do + cat > ~/.source_cloud_exported_variables_$CELL << EOF +PULL_OPENSTACK_CONFIGURATION_DATABASES[$CELL]="$(oc run mariadb-client-5-$CELL ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh ${SOURCE_MARIADB_IP[$CELL]} -uroot -p${SOURCE_DB_ROOT_PASSWORD[$CELL]} -e 'SHOW databases;')" +PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK[$CELL]="$(oc run mariadb-client-6-$CELL ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysqlcheck --all-databases -h ${SOURCE_MARIADB_IP[$CELL]} -u root -p${SOURCE_DB_ROOT_PASSWORD[$CELL]} | grep -v OK)" +PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$CELL]="$(oc run mariadb-client-7-$CELL ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh ${SOURCE_MARIADB_IP[$CELL]} -uroot -p${SOURCE_DB_ROOT_PASSWORD[$CELL]} -e \ + "select host from nova.services where services.binary='nova-compute';")" +EOF + done +$ cat >> ~/.source_cloud_exported_variables_default << EOF +PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS="$(oc run mariadb-client-2 ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh ${SOURCE_MARIADB_IP['default']} -uroot -p${SOURCE_DB_ROOT_PASSWORD['default']} -e \ + 'select uuid,name,transport_url,database_connection,disabled from nova_api.cell_mappings;' || echo None)" +PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS="$($CONTROLLER1_SSH sudo podman exec -it nova_conductor nova-manage cell_v2 list_cells)" EOF +$ chmod 0600 ~/.source_cloud_exported_variables* ---- +<1> If `neutron-sriov-nic-agent` agents are running in your {OpenStackShort} deployment, get the configuration to use for the data plane adoption + +[NOTE] +==== +This configuration will be required later, during the data plane adoption post-checks. +==== diff --git a/docs_user/modules/proc_stopping-infrastructure-management-and-compute-services.adoc b/docs_user/modules/proc_stopping-infrastructure-management-and-compute-services.adoc index 88c07fa5d..e13eb8b9c 100644 --- a/docs_user/modules/proc_stopping-infrastructure-management-and-compute-services.adoc +++ b/docs_user/modules/proc_stopping-infrastructure-management-and-compute-services.adoc @@ -13,21 +13,25 @@ The following procedure applies to a single node standalone {OpenStackPreviousIn [subs=+quotes] ---- ifeval::["{build}" != "downstream"] +CONTROLLER1_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@" +# ... +endif::[] +ifeval::["{build}" == "downstream"] +CONTROLLER1_SSH="ssh -i root@" +# ... +endif::[] +# ... <1> +ifeval::["{build}" != "downstream"] EDPM_PRIVATEKEY_PATH="~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa" endif::[] ifeval::["{build}" == "downstream"] -EDPM_PRIVATEKEY_PATH="" +EDPM_PRIVATEKEY_PATH="" <2> endif::[] -declare -A computes -computes=( - ["standalone.localdomain"]="192.168.122.100" - # ... -) ---- + -** Replace `["standalone.localdomain"]="192.168.122.100"` with the name and IP address of the Compute node. +<1> Complete `CONTROLLER_SSH` settings with SSH connection details for all controllers, including cell controllers, of the source {OpenStackPreviousInstaller} cloud. ifeval::["{build}" == "downstream"] -** Replace `` with the path to your SSH key. +<2> Replace `` with the path to your SSH key. endif::[] .Procedure diff --git a/docs_user/modules/proc_stopping-openstack-services.adoc b/docs_user/modules/proc_stopping-openstack-services.adoc index 5ad49efc4..465c1b292 100644 --- a/docs_user/modules/proc_stopping-openstack-services.adoc +++ b/docs_user/modules/proc_stopping-openstack-services.adoc @@ -34,11 +34,31 @@ ifeval::["{build}" != "downstream"] CONTROLLER1_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100" endif::[] ifeval::["{build}" == "downstream"] -CONTROLLER1_SSH="ssh -i ** root@**" -endif::[] +CONTROLLER1_SSH="ssh -i ** root@**" <1> CONTROLLER2_SSH="ssh -i ** root@**" CONTROLLER3_SSH="ssh -i ** root@**" +endif::[] ---- +* For a multi-cell deployment, specify the overcloud and cell controllers IPs instead, for example: ++ +[subs=+quotes] +---- +ifeval::["{build}" != "downstream"] +CONTROLLER1_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.103" +CONTROLLER2_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.106" +CONTROLLER3_SSH="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.109" +endif::[] +ifeval::["{build}" == "downstream"] +CONTROLLER1_SSH="ssh -i ** root@**" <2> +CONTROLLER2_SSH="ssh -i ** root@**" +CONTROLLER3_SSH="ssh -i ** root@**" +# ... +endif::[] +---- +ifeval::["{build}" == "downstream"] +<1> Replace `` with the path to your SSH key. +<2> Replace `` with IP addresses of all controllers, includinig cell controllers. +endif::[] .Procedure diff --git a/tests/roles/backend_services/tasks/main.yaml b/tests/roles/backend_services/tasks/main.yaml index 238a4d94d..d1010f602 100644 --- a/tests/roles/backend_services/tasks/main.yaml +++ b/tests/roles/backend_services/tasks/main.yaml @@ -105,27 +105,28 @@ vars: deploy_ctlplane_ospdo: true -- name: Deploy the podified control plane +- name: deploy the OpenStackControlPlane CR ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} oc apply -f ../config/tmp/test_deployment.yaml when: not ospdo_src| bool -- name: wait for services to start up +- name: verify that MariaDB and RabbitMQ are running, for all defined cells ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} - oc wait pod --for condition=Ready {{ item }} - register: service_running_result - until: service_running_result is success - retries: 150 + {{ cells_env }} + oc get pod openstack-galera-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running + oc get pod rabbitmq-server-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running + for CELL in $(echo $RENAMED_CELLS); do + oc get pod openstack-$CELL-galera-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running + oc get pod rabbitmq-$CELL-server-0 -o jsonpath='{.status.phase}{"\n"}' | grep Running + done + register: mariadb_running_result + until: mariadb_running_result is success + retries: 60 delay: 2 - loop: - - openstack-galera-0 - - openstack-cell1-galera-0 - - rabbitmq-server-0 - - rabbitmq-cell1-server-0 - name: Patch openstack upstream dns server to set the correct value for the environment when: upstream_dns is defined diff --git a/tests/roles/backend_services/tasks/ospdo_backend_services.yaml b/tests/roles/backend_services/tasks/ospdo_backend_services.yaml index 71273976f..a21e90a50 100644 --- a/tests/roles/backend_services/tasks/ospdo_backend_services.yaml +++ b/tests/roles/backend_services/tasks/ospdo_backend_services.yaml @@ -1,6 +1,7 @@ - name: get service passwords ansible.builtin.shell: | oc get secret tripleo-passwords -o jsonpath='{.data.*}' | base64 -d>~/tripleo-standalone-passwords.yaml + cp ~/tripleo-standalone-passwords.yaml ~/overcloud-passwords.yaml when: get_svc_pass| default(false) | bool - name: get OSPdO storage storageClass diff --git a/tests/roles/backend_services/templates/openstack_control_plane.j2 b/tests/roles/backend_services/templates/openstack_control_plane.j2 index 53054f443..6930a7631 100644 --- a/tests/roles/backend_services/templates/openstack_control_plane.j2 +++ b/tests/roles/backend_services/templates/openstack_control_plane.j2 @@ -81,7 +81,16 @@ spec: openstack-cell1: secret: osp-secret replicas: 1 - storageRequest: 1Gi + storageRequest: 250M + # TODO(bogdando): iterate based on renamed_cells value in kustomization.yaml + openstack-cell2: + secret: osp-secret + replicas: 1 + storageRequest: 250M + openstack-cell3: + secret: osp-secret + replicas: 1 + storageRequest: 250M memcached: enabled: true @@ -121,6 +130,8 @@ spec: rabbitmq: templates: rabbitmq: + persistence: + storage: 200M override: service: metadata: @@ -130,6 +141,8 @@ spec: spec: type: LoadBalancer rabbitmq-cell1: + persistence: + storage: 200M override: service: metadata: @@ -138,7 +151,29 @@ spec: metallb.universe.tf/loadBalancerIPs: 172.17.0.86 spec: type: LoadBalancer - + # TODO(bogdando): iterate based on renamed_cells value in kustomization.yaml + rabbitmq-cell2: + persistence: + storage: 200M + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.87 + spec: + type: LoadBalancer + rabbitmq-cell3: + persistence: + storage: 200M + override: + service: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.88 + spec: + type: LoadBalancer telemetry: enabled: false diff --git a/tests/roles/common_defaults/defaults/main.yaml b/tests/roles/common_defaults/defaults/main.yaml index 035d86f46..44699d2b0 100644 --- a/tests/roles/common_defaults/defaults/main.yaml +++ b/tests/roles/common_defaults/defaults/main.yaml @@ -1,3 +1,15 @@ +# Whether to use no_log on tasks which may output potentially +# sensitive data. +use_no_log: false + +# Whether the adopted node will host compute services +compute_adoption: true + +# The names of cells on the target cloud +renamed_cells: "{{ [default_cell_name] + cells | difference('default') }}" + +## Env headers (should be the same as provided in the docs, as the best effort) + # Header to apply in 'shell' module invocations. If overriding, at # least 'set -e' should be kept to ensure that scripts that fail # somewhere in the middle will also fail the whole Ansible run as @@ -5,12 +17,102 @@ shell_header: | set -euxo pipefail -# Whether to use no_log on tasks which may output potentially -# sensitive data. -use_no_log: false +# Snippet to get the desired 'oc' command onto $PATH. +oc_header: | + eval $(crc oc-env) -# Whether the adopted node will host compute services -compute_adoption: true +# Header for a Compute cells names evaluation for the source and destination cloud +# and renaming of the default cell from tripleo +# NOTE: 'super' is not a cell, but a reference for the top-scope "upcall" API on main controllers +cells_env: | + CELLS="{{ cells | join(' ') }}" + DEFAULT_CELL_NAME={{ default_cell_name }} + RENAMED_CELLS="{{ renamed_cells | join(' ') }}" + +# Headers for DB client CLI image +mariadb_image_env: | + MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified + +# Header for the source databases initial clusters members health check +mariadb_members_env: |- + {{ shell_header }} + {{ cells_env }} + + {% for cell in cells %} + declare -A SOURCE_GALERA_MEMBERS_{{ cell.upper() }} + SOURCE_GALERA_MEMBERS_{{ cell.upper() }}=( + {% for n in source_galera_members[cell] | default([]) %} + [{{ n.name|quote }}]={{ n.ip|quote }} + {% endfor %} + ) + {% endfor %} + +# Headers for the destination databases access passwords +mariadb_passwords_env: |- + {{ shell_header }} + {{ cells_env }} + + declare -A PODIFIED_DB_ROOT_PASSWORD + for CELL in $(echo "super $RENAMED_CELLS"); do + PODIFIED_DB_ROOT_PASSWORD[$CELL]=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) + done + +# Header for the source database access +mariadb_copy_shell_vars_src: |- + {{ shell_header }} + {{ mariadb_image_env }} + {{ cells_env }} + MARIADB_CLIENT_ANNOTATIONS='--annotations=k8s.v1.cni.cncf.io/networks=internalapi' + STORAGE_CLASS={{ storage_class_name }} + + declare -A TRIPLEO_PASSWORDS + for CELL in $(echo $CELLS); do + if [ "$CELL" = "default" ]; then + TRIPLEO_PASSWORDS[default]="$HOME/overcloud-passwords.yaml" + else + # in a split-stack source cloud, it should take a stack-specific passwords file instead + TRIPLEO_PASSWORDS[$CELL]="$HOME/overcloud-passwords.yaml" + fi + done + + declare -A SOURCE_DB_ROOT_PASSWORD + for CELL in $(echo $CELLS); do + SOURCE_DB_ROOT_PASSWORD[$CELL]=$(cat ${TRIPLEO_PASSWORDS[$CELL]} | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') + done + + declare -A SOURCE_MARIADB_IP + {% for cell in cells %} + SOURCE_MARIADB_IP[{{ cell }}]={{ source_mariadb_ip[cell] }} + {% endfor %} + +# Header for the destination database access +mariadb_copy_shell_vars_dst: | + {{ shell_header }} + {{ oc_header }} + {{ mariadb_image_env }} + {{ cells_env }} + + # The CHARACTER_SET and collation should match the source DB + # if the do not then it will break foreign key relationships + # for any tables that are created in the future as part of db sync + CHARACTER_SET=utf8 + COLLATION=utf8_general_ci + + {{ mariadb_passwords_env }} + + declare -A PODIFIED_MARIADB_IP + for CELL in $(echo "super $RENAMED_CELLS"); do + if [ "$CELL" = "super" ]; then + PODIFIED_MARIADB_IP[$CELL]=$(oc get svc --selector "mariadb/name=openstack" -ojsonpath='{.items[0].spec.clusterIP}') + else + PODIFIED_MARIADB_IP[$CELL]=$(oc get svc --selector "mariadb/name=openstack-$CELL" -ojsonpath='{.items[0].spec.clusterIP}') + fi + done + +pull_openstack_configuration_ssh_shell_vars: | + CONTROLLER1_SSH="{{ controller1_ssh }}" + CONTROLLER2_SSH="{{ controller2_ssh }}" + CONTROLLER3_SSH="{{ controller3_ssh }}" # Whether to adopt Octavia enable_octavia: true diff --git a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml index 33227f258..c88404c5f 100644 --- a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml +++ b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml @@ -1,8 +1,3 @@ -- name: get dst database service environment variables - ansible.builtin.include_role: - name: mariadb_copy - tasks_from: env_vars_dst.yaml - - name: wait for cell1 Nova compute EDPM services version updated ansible.builtin.shell: | {{ shell_header }} diff --git a/tests/roles/get_services_configuration/tasks/main.yaml b/tests/roles/get_services_configuration/tasks/main.yaml index e76b1ffb8..718638bd6 100644 --- a/tests/roles/get_services_configuration/tasks/main.yaml +++ b/tests/roles/get_services_configuration/tasks/main.yaml @@ -1,54 +1,48 @@ -# Pull nova configuration steps -- name: set shell vars pull openstack configuration ssh commands - no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - pull_openstack_configuration_ssh_shell_vars: | - CONTROLLER1_SSH="{{ controller1_ssh }}" - - name: execute alternative tasks when source env is OSPdO ansible.builtin.include_role: name: mariadb_copy tasks_from: env_vars_src_ospdo.yaml when: ospdo_src| bool -- name: get src database service environment variables - ansible.builtin.include_role: - name: mariadb_copy - tasks_from: env_vars_src.yaml - # NOTE(bogdando): env variables must be used to keep this consistent with documentation, # where the stored values need to be compared with post-adoption ones w/o using ansible specifics - name: test connection to the original DB no_log: "{{ use_no_log }}" ansible.builtin.shell: | - {{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} - export PULL_OPENSTACK_CONFIGURATION_DATABASES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;') - echo "$PULL_OPENSTACK_CONFIGURATION_DATABASES" - register: _databases_check + unset PULL_OPENSTACK_CONFIGURATION_DATABASES + declare -xA PULL_OPENSTACK_CONFIGURATION_DATABASES + for CELL in $(echo $CELLS); do + PULL_OPENSTACK_CONFIGURATION_DATABASES[$CELL]=$(oc run mariadb-client-1-$CELL ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh "${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" -e 'SHOW databases;') + done - name: run mysqlcheck on the original DB to look for things that are not OK no_log: "{{ use_no_log }}" ansible.builtin.shell: | - {{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} + unset PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK + declare -xA PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK run_mysqlcheck() { - export PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysqlcheck --all-databases -h "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" | grep -v OK) + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK=$(oc run mariadb-client-2-$1 ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysqlcheck --all-databases -h ${SOURCE_MARIADB_IP[$CELL]} -u root -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" | grep -v OK) } - run_mysqlcheck + for CELL in $(echo $CELLS); do + run_mysqlcheck $CELL + done if [ "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" != "" ]; then - # Try mysql_upgrade to fix mysqlcheck failure - MYSQL_UPGRADE=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysql_upgrade -v -h "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}") - # rerun mysqlcheck to check if problem is resolved - run_mysqlcheck + # Try mysql_upgrade to fix mysqlcheck failure + for CELL in $(echo $CELLS); do + MYSQL_UPGRADE=$(oc run mariadb-client-3-$CELL ${MARIADB_CLIENT_ANNOTATIONS} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql_upgrade -v -h ${SOURCE_MARIADB_IP[$CELL]} -u root -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}") + # rerun mysqlcheck to check if problem is resolved + run_mysqlcheck + done fi echo "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" - failed_when: _mysqlnok_check.stdout != '' + failed_when: _mysqlnok_check.stdout | trim != '' register: _mysqlnok_check - name: get source Nova services topology specific configuration @@ -56,63 +50,57 @@ block: - name: get Nova cells mappings from database ansible.builtin.shell: | - {{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} - export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + export PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS=$(oc run mariadb-client-1 ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ mysql -rsh "${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" nova_api -e \ 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') - echo "$PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS" - register: _novadb_mapped_cells_check - name: get the host names of the registered Nova compute services ansible.builtin.shell: | - {{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} - export PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES=$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysql -rsh "$SOURCE_MARIADB_IP" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" nova_api -e \ - "select host from nova.services where services.binary='nova-compute';") - echo "$PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES" - register: _nova_compute_hostnames_check + unset PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES + declare -xA PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES + for CELL in $(echo $CELLS); do + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$CELL]=$(oc run mariadb-client-4-$CELL ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh "${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" -e \ + "select host from nova.services where services.binary='nova-compute';") + done - name: get the list of mapped Nova cells ansible.builtin.shell: | {{ shell_header }} {{ pull_openstack_configuration_ssh_shell_vars }} export PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS=$($CONTROLLER1_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells) - echo "$PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS" - register: _nova_cell_mappings_check - -- name: set cached fact for pulled openstack services configuration shell headers - no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - pulled_openstack_configuration_shell_headers: | - PULL_OPENSTACK_CONFIGURATION_DATABASES="{{ _databases_check.stdout_lines | join(' ') }}" - PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK="{{ _mysqlnok_check.stdout_lines | join(' ') }}" - PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS="{{ _novadb_mapped_cells_check.stdout_lines | join(' ') }}" - PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES="{{ _nova_compute_hostnames_check.stdout_lines | join(' ') }}" - PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS="{{ _nova_cell_mappings_check.stdout_lines | join(' ') }}" - name: store exported variables for future use no_log: "{{ use_no_log }}" ansible.builtin.shell: | - {{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} {{ pull_openstack_configuration_ssh_shell_vars }} - - cat > ~/.source_cloud_exported_variables << EOF - PULL_OPENSTACK_CONFIGURATION_DATABASES="$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysql -rsh $SOURCE_MARIADB_IP -uroot -p$SOURCE_DB_ROOT_PASSWORD -e 'SHOW databases;')" - PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK="$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root -p$SOURCE_DB_ROOT_PASSWORD | grep -v OK)" - PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS="$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysql -rsh $SOURCE_MARIADB_IP -uroot -p$SOURCE_DB_ROOT_PASSWORD nova_api -e \ - 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;')" - PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES="$(oc run mariadb-client ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ - mysql -rsh $SOURCE_MARIADB_IP -uroot -p$SOURCE_DB_ROOT_PASSWORD nova_api -e \ + unset PULL_OPENSTACK_CONFIGURATION_DATABASES + unset PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK + unset PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES + declare -xA PULL_OPENSTACK_CONFIGURATION_DATABASES + declare -xA PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK + declare -xA PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES + for CELL in $(echo $CELLS); do + cat > ~/.source_cloud_exported_variables_$CELL << EOF + PULL_OPENSTACK_CONFIGURATION_DATABASES[$CELL]="$(oc run mariadb-client-5-$CELL ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh ${SOURCE_MARIADB_IP[$CELL]} -uroot -p${SOURCE_DB_ROOT_PASSWORD[$CELL]} -e 'SHOW databases;')" + PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK[$CELL]="$(oc run mariadb-client-6-$CELL ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysqlcheck --all-databases -h ${SOURCE_MARIADB_IP[$CELL]} -u root -p${SOURCE_DB_ROOT_PASSWORD[$CELL]} | grep -v OK)" + PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$CELL]="$(oc run mariadb-client-7-$CELL ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh ${SOURCE_MARIADB_IP[$CELL]} -uroot -p${SOURCE_DB_ROOT_PASSWORD[$CELL]} -e \ "select host from nova.services where services.binary='nova-compute';")" - PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS="$($CONTROLLER1_SSH sudo podman exec -it nova_api nova-manage cell_v2 list_cells)" EOF - chmod 0600 ~/.source_cloud_exported_variables + done + cat >> ~/.source_cloud_exported_variables_default << EOF + PULL_OPENSTACK_CONFIGURATION_NOVADB_MAPPED_CELLS="$(oc run mariadb-client-2 ${MARIADB_CLIENT_ANNOTATIONS} {{ mysql_client_override }} -q --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -rsh ${SOURCE_MARIADB_IP['default']} -uroot -p${SOURCE_DB_ROOT_PASSWORD['default']} -e \ + 'select uuid,name,transport_url,database_connection,disabled from nova_api.cell_mappings;' || echo None)" + PULL_OPENSTACK_CONFIGURATION_NOVAMANAGE_CELL_MAPPINGS="$($CONTROLLER1_SSH sudo podman exec -it nova_conductor nova-manage cell_v2 list_cells)" + EOF + chmod 0600 ~/.source_cloud_exported_variables* diff --git a/tests/roles/mariadb_copy/defaults/main.yaml b/tests/roles/mariadb_copy/defaults/main.yaml index 155f76e4e..7b094179b 100644 --- a/tests/roles/mariadb_copy/defaults/main.yaml +++ b/tests/roles/mariadb_copy/defaults/main.yaml @@ -1,8 +1,6 @@ edpm_node_hostname: standalone.localdomain mariadb_copy_tmp_dir: tmp/mariadb storage_reclaim_policy: delete -source_galera_members: |- - ["{{ edpm_node_hostname }}"]={{ source_mariadb_ip|default(external_mariadb_ip) }} dpa_dir: "../.." dpa_tests_dir: "{{ dpa_dir }}/tests" diff --git a/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml b/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml deleted file mode 100644 index f1538fd59..000000000 --- a/tests/roles/mariadb_copy/tasks/env_vars_dst.yaml +++ /dev/null @@ -1,27 +0,0 @@ -- name: get podified MariaDB service cluster IP - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - oc get svc --selector "mariadb/name=openstack" -n {{ rhoso_namespace }} -ojsonpath='{.items[0].spec.clusterIP}' - register: podified_mariadb_ip_result - -- name: get podified cell1 MariaDB IP - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - oc get svc --selector "mariadb/name=openstack-cell1" -n {{ rhoso_namespace }} -ojsonpath='{.items[0].spec.clusterIP}' - register: podified_cell1_mariadb_ip_result - -- name: set MariaDB copy shell vars - no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - mariadb_copy_shell_vars_dst: | - PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} - PODIFIED_CELL1_MARIADB_IP={{ podified_cell1_mariadb_ip_result.stdout }} - PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" - - # The CHARACTER_SET and collation should match the source DB - # if the do not then it will break foreign key relationships - # for any tables that are created in the future as part of db sync - CHARACTER_SET=utf8 - COLLATION=utf8_general_ci diff --git a/tests/roles/mariadb_copy/tasks/env_vars_src.yaml b/tests/roles/mariadb_copy/tasks/env_vars_src.yaml deleted file mode 100644 index 378e3840b..000000000 --- a/tests/roles/mariadb_copy/tasks/env_vars_src.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- name: set src MariaDB copy shell vars - no_log: "{{ use_no_log }}" - ansible.builtin.set_fact: - mariadb_copy_shell_vars_src: | - MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified - STORAGE_CLASS={{ storage_class_name }} - # TODO: remove the default(external_...) when CI is transitioned to use 'source_...' - SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }} - declare -A SOURCE_GALERA_MEMBERS - SOURCE_GALERA_MEMBERS=( - {{ source_galera_members }} - ) - SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}" - MARIADB_CLIENT_ANNOTATIONS='--annotations=k8s.v1.cni.cncf.io/networks=internalapi' - when: not ospdo_src| bool diff --git a/tests/roles/mariadb_copy/tasks/env_vars_src_ospdo.yaml b/tests/roles/mariadb_copy/tasks/env_vars_src_ospdo.yaml index fc03a54e4..6ba631dbd 100644 --- a/tests/roles/mariadb_copy/tasks/env_vars_src_ospdo.yaml +++ b/tests/roles/mariadb_copy/tasks/env_vars_src_ospdo.yaml @@ -23,12 +23,14 @@ mariadb_copy_shell_vars_src: | MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified STORAGE_CLASS=host-nfs-storageclass - SOURCE_MARIADB_IP={{ source_mariadb_ip.stdout }} - declare -A SOURCE_GALERA_MEMBERS - SOURCE_GALERA_MEMBERS=( - ["standalone.localdomain"]={{ source_mariadb_ip.stdout }} + declare -A SOURCE_MARIADB_IP + SOURCE_MARIADB_IP['default']={{ source_mariadb_ip.stdout }} + declare -A SOURCE_GALERA_MEMBERS_DEFAULT + SOURCE_GALERA_MEMBERS_DEFAULT=( + ['standalone.localdomain']='{{ source_mariadb_ip.stdout }}' ) - SOURCE_DB_ROOT_PASSWORD={{ source_db_root_pass.stdout }} + declare -A SOURCE_DB_ROOT_PASSWORD + SOURCE_DB_ROOT_PASSWORD['default']={{ source_db_root_pass.stdout }} MARIADB_CLIENT_ANNOTATIONS="-n {{ org_namespace }}" CONTROLLER_NODE={{ controller_node.stdout }} diff --git a/tests/roles/mariadb_copy/tasks/main.yaml b/tests/roles/mariadb_copy/tasks/main.yaml index af853164c..4223c3a42 100644 --- a/tests/roles/mariadb_copy/tasks/main.yaml +++ b/tests/roles/mariadb_copy/tasks/main.yaml @@ -1,20 +1,11 @@ -- name: get the source database service environment variables - ansible.builtin.include_tasks: - file: env_vars_src.yaml - - name: execute alternative tasks when source env is OSPdO ansible.builtin.include_role: name: mariadb_copy tasks_from: env_vars_src_ospdo.yaml when: ospdo_src| bool -- name: get the destination database service environment variables - ansible.builtin.include_tasks: - file: env_vars_dst.yaml - - name: start an adoption mariadb helper pod ansible.builtin.shell: |- - {{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} @@ -77,30 +68,32 @@ retries: 25 delay: 2 -- name: check that the Galera database cluster members are online and synced +- name: check that the Galera database cluster(s) members are online and synced, for all cells no_log: "{{ use_no_log }}" ansible.builtin.shell: | - {{ shell_header }} {{ oc_header }} + {{ mariadb_members_env }} {{ mariadb_copy_shell_vars_src }} - for i in "${!SOURCE_GALERA_MEMBERS[@]}"; do - echo "Checking for the database node $i WSREP status Synced" - oc rsh -n {{ org_namespace }} mariadb-copy-data mysql \ - -h "${SOURCE_GALERA_MEMBERS[$i]}" -uroot -p"$SOURCE_DB_ROOT_PASSWORD" \ - -e "show global status like 'wsrep_local_state_comment'" | \ - grep -qE "\bSynced\b" + for CELL in $(echo $CELLS); do + MEMBERS=SOURCE_GALERA_MEMBERS_$(echo ${CELL}|tr '[:lower:]' '[:upper:]')[@] + for i in "${!MEMBERS}"; do + echo "Checking for the database node $i WSREP status Synced" + oc rsh -n {{ org_namespace }} mariadb-copy-data mysql \ + -h "$i" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" \ + -e "show global status like 'wsrep_local_state_comment'" | \ + grep -qE "\bSynced\b" + done done - name: Get the count of not-OK source databases no_log: "{{ use_no_log }}" ansible.builtin.shell: | - {% if pulled_openstack_configuration_shell_headers is defined %} - {{ pulled_openstack_configuration_shell_headers }} - {% else %} - . ~/.source_cloud_exported_variables - {% endif %} - - test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" || [ "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" = " " ] && echo "OK" || echo "CHECK FAILED" + for CELL in $(echo $CELLS); do + set +u + . ~/.source_cloud_exported_variables_$CELL + set -u + done + test -z "$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" || [ "x$PULL_OPENSTACK_CONFIGURATION_MYSQLCHECK_NOK" = "x " ] && echo "OK" || echo "CHECK FAILED" register: result failed_when: result.rc != 0 or 'CHECK FAILED' in result.stdout diff --git a/tests/roles/mariadb_copy/tasks/mariadb_verify.yaml b/tests/roles/mariadb_copy/tasks/mariadb_verify.yaml index 9a3462428..7571a24e4 100644 --- a/tests/roles/mariadb_copy/tasks/mariadb_verify.yaml +++ b/tests/roles/mariadb_copy/tasks/mariadb_verify.yaml @@ -4,14 +4,6 @@ tasks_from: env_vars_src_ospdo.yaml when: ospdo_src| bool -- name: get the source database service environment variables - ansible.builtin.include_tasks: - file: env_vars_src.yaml - -- name: get the destination database service environment variables - ansible.builtin.include_tasks: - file: env_vars_dst.yaml - - name: MariaDB checks no_log: "{{ use_no_log }}" ansible.builtin.shell: diff --git a/tests/roles/mariadb_copy/templates/dump_dbs.bash b/tests/roles/mariadb_copy/templates/dump_dbs.bash index 45ec654e8..dd693c984 100755 --- a/tests/roles/mariadb_copy/templates/dump_dbs.bash +++ b/tests/roles/mariadb_copy/templates/dump_dbs.bash @@ -1,18 +1,20 @@ #!/bin/bash -{{ shell_header }} {{ oc_header }} {{ mariadb_copy_shell_vars_src }} +# Create a dump of the original databases # Note Filter the information and performance schema tables # Gnocchi is no longer used as a metric store, skip dumping gnocchi database as well # Migrating Aodh alarms from previous release is not supported, hence skip aodh database -oc rsh -n "{{ org_namespace }}" mariadb-copy-data << EOF - mysql -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \ - -N -e "show databases" | grep -E -v "schema|mysql|gnocchi|aodh" | \ - while read dbname; do - echo "Dumping \${dbname}"; - mysqldump -h"${SOURCE_MARIADB_IP}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD}" \ - --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \ - "\${dbname}" > /backup/"\${dbname}".sql; - done +for CELL in $(echo $CELLS); do + oc rsh -n "{{ org_namespace }}" mariadb-copy-data << EOF + mysql -h"${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" \ + -N -e "show databases" | grep -E -v "schema|mysql|gnocchi|aodh" | \ + while read dbname; do + echo "Dumping $CELL cell \${dbname}"; + mysqldump -h"${SOURCE_MARIADB_IP[$CELL]}" -uroot -p"${SOURCE_DB_ROOT_PASSWORD[$CELL]}" \ + --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \ + "\${dbname}" > /backup/"${CELL}.\${dbname}".sql; + done EOF +done diff --git a/tests/roles/mariadb_copy/templates/post_checks.bash b/tests/roles/mariadb_copy/templates/post_checks.bash index d475ddb7c..ce0991a98 100755 --- a/tests/roles/mariadb_copy/templates/post_checks.bash +++ b/tests/roles/mariadb_copy/templates/post_checks.bash @@ -1,26 +1,20 @@ -{{ shell_header }} -{{ oc_header }} {{ mariadb_copy_shell_vars_dst }} -{% if pulled_openstack_configuration_shell_headers is defined %} -{{ pulled_openstack_configuration_shell_headers }} -{% else %} -. ~/.source_cloud_exported_variables -{% endif %} - +# Check that the databases were imported correctly # use 'oc exec' and 'mysql -rs' to maintain formatting -dbs=$(oc exec openstack-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;') + +set +u +. ~/.source_cloud_exported_variables_default +set -u + +dbs=$(oc exec openstack-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot -p"${PODIFIED_DB_ROOT_PASSWORD['super']}" -e 'SHOW databases;') echo $dbs | grep -Eq '\bkeystone\b' && echo "OK" || echo "CHECK FAILED" # ensure neutron db is renamed from ovs_neutron echo $dbs | grep -Eq '\bneutron\b' -echo $PULL_OPENSTACK_CONFIGURATION_DATABASES | grep -Eq '\bovs_neutron\b' && echo "OK" || echo "CHECK FAILED" +echo "${PULL_OPENSTACK_CONFIGURATION_DATABASES[@]}" | grep -Eq '\bovs_neutron\b' && echo "OK" || echo "CHECK FAILED" -# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 -c1dbs=$(oc exec openstack-cell1-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;') -echo $c1dbs | grep -Eq '\bnova_cell1\b' && echo "OK" || echo "CHECK FAILED" - -# ensure default cell renamed to cell1, and the cell UUIDs retained intact -novadb_mapped_cells=$(oc exec openstack-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ +# ensure default cell is renamed to $DEFAULT_CELL_NAME, and the cell UUIDs retained intact +novadb_mapped_cells=$(oc exec openstack-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot -p"${PODIFIED_DB_ROOT_PASSWORD['super']}" \ nova_api -e 'select uuid,name,transport_url,database_connection,disabled from cell_mappings;') uuidf='\S{8,}-\S{4,}-\S{4,}-\S{4,}-\S{12,}' left_behind=$(comm -23 \ @@ -32,9 +26,21 @@ changed=$(comm -13 \ test $(grep -Ec ' \S+$' <<<$left_behind) -eq 1 && echo "OK" || echo "CHECK FAILED" default=$(grep -E ' default$' <<<$left_behind) test $(grep -Ec ' \S+$' <<<$changed) -eq 1 && echo "OK" || echo "CHECK FAILED" -grep -qE " $(awk '{print $1}' <<<$default) cell1$" <<<$changed && echo "OK" || echo "CHECK FAILED" +grep -qE " $(awk '{print $1}' <<<$default) ${DEFAULT_CELL_NAME}$" <<<$changed && echo "OK" || echo "CHECK FAILED" # ensure the registered Compute service name has not changed -novadb_svc_records=$(oc exec openstack-cell1-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" \ - nova_cell1 -e "select host from services where services.binary='nova-compute' order by host asc;") -diff -Z <(echo $novadb_svc_records) <(echo $PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES) && echo "OK" || echo "CHECK FAILED" +for CELL in $(echo $CELLS); do + set +u + . ~/.source_cloud_exported_variables_$CELL + set -u + RCELL=$CELL + [ "$CELL" = "default" ] && RCELL=$DEFAULT_CELL_NAME + # ensure nova cells' db are extracted to separate db servers and renamed from nova to nova_cell + c1dbs=$(oc exec openstack-$RCELL-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot -p${PODIFIED_DB_ROOT_PASSWORD[$RCELL]} -e 'SHOW databases;') + echo $c1dbs | grep -Eq "\bnova_${RCELL}\b" && echo "OK" || echo "CHECK FAILED" + + # ensure the registered Compute service name has not changed + novadb_svc_records=$(oc exec openstack-$RCELL-galera-0 -n {{ rhoso_namespace }} -c galera -- mysql -rs -uroot -p${PODIFIED_DB_ROOT_PASSWORD[$RCELL]} \ + nova_$RCELL -e "select host from services where services.binary='nova-compute' order by host asc;") + diff -Z <(echo $novadb_svc_records) <(echo ${PULL_OPENSTACK_CONFIGURATION_NOVA_COMPUTE_HOSTNAMES[$RCELL]}) && echo "OK" || echo "CHECK FAILED" +done diff --git a/tests/roles/mariadb_copy/templates/pre_checks.bash b/tests/roles/mariadb_copy/templates/pre_checks.bash index 19aa24fbc..2f7e089bc 100755 --- a/tests/roles/mariadb_copy/templates/pre_checks.bash +++ b/tests/roles/mariadb_copy/templates/pre_checks.bash @@ -1,11 +1,8 @@ #!/bin/bash -{{ shell_header }} -{{ oc_header }} -{{ mariadb_copy_shell_vars_src }} {{ mariadb_copy_shell_vars_dst }} -# Test connection to podified DBs (show databases) -oc run mariadb-client -n {{ rhoso_namespace }} ${MARIADB_CLIENT_ANNOTATIONS} --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -rsh "$PODIFIED_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' -oc run mariadb-client -n {{ rhoso_namespace }} ${MARIADB_CLIENT_ANNOTATIONS} --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -rsh "$PODIFIED_CELL1_MARIADB_IP" -uroot -p"$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' +# Test the connection to the control plane "upcall" and cells' databases +for CELL in $(echo "super $RENAMED_CELLS"); do + oc run mariadb-client -n {{ rhoso_namespace }} --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -rsh "${PODIFIED_MARIADB_IP[$CELL]}" -uroot -p"${PODIFIED_DB_ROOT_PASSWORD[$CELL]}" -e 'SHOW databases;' +done diff --git a/tests/roles/mariadb_copy/templates/restore_dbs.bash b/tests/roles/mariadb_copy/templates/restore_dbs.bash index e45eb5a10..da189ca25 100755 --- a/tests/roles/mariadb_copy/templates/restore_dbs.bash +++ b/tests/roles/mariadb_copy/templates/restore_dbs.bash @@ -1,51 +1,78 @@ #!/bin/bash -{{ shell_header }} -{{ oc_header }} {{ mariadb_copy_shell_vars_src }} {{ mariadb_copy_shell_vars_dst }} -oc rsh -n {{ rhoso_namespace }} mariadb-copy-data << EOF - # db schemas to rename on import - declare -A db_name_map - db_name_map['nova']='nova_cell1' - db_name_map['ovs_neutron']='neutron' - db_name_map['ironic-inspector']='ironic_inspector' +# Restore the databases from .sql files into the control plane MariaDB - # db servers to import into - declare -A db_server_map - db_server_map['default']=${PODIFIED_MARIADB_IP} - db_server_map['nova_cell1']=${PODIFIED_CELL1_MARIADB_IP} +for CELL in $(echo $CELLS); do + RCELL=$CELL + [ "$CELL" = "default" ] && RCELL=$DEFAULT_CELL_NAME + oc rsh -n {{ rhoso_namespace }} mariadb-copy-data << EOF + # db schemas to rename on import + declare -A db_name_map + db_name_map['nova']="nova_$RCELL" + db_name_map['ovs_neutron']='neutron' + db_name_map['ironic-inspector']='ironic_inspector' - # db server root password map - declare -A db_server_password_map - db_server_password_map['default']=${PODIFIED_DB_ROOT_PASSWORD} - db_server_password_map['nova_cell1']=${PODIFIED_DB_ROOT_PASSWORD} + # cells' db schemas to import for cells + declare -A db_cell_map + db_cell_map['nova']="nova_$DEFAULT_CELL_NAME" + db_cell_map["nova_$RCELL"]="nova_$RCELL" + # Omit importing cells' cell0 DBs as we cannot consolidate them + # db_cell_map["nova_cell0"]="nova_$RCELL" - cd /backup - for db_file in \$(ls *.sql); do - db_name=\$(echo \${db_file} | awk -F'.' '{ print \$1; }') - if [[ -v "db_name_map[\${db_name}]" ]]; then - echo "renaming \${db_name} to \${db_name_map[\${db_name}]}" - db_name=\${db_name_map[\${db_name}]} - fi - db_server=\${db_server_map["default"]} - if [[ -v "db_server_map[\${db_name}]" ]]; then - db_server=\${db_server_map[\${db_name}]} - fi - db_password=\${db_server_password_map['default']} - if [[ -v "db_server_password_map[\${db_name}]" ]]; then - db_password=\${db_server_password_map[\${db_name}]} - fi - echo "creating \${db_name} in \${db_server}" - mysql -h"\${db_server}" -uroot -p"\${db_password}" -e \ - "CREATE DATABASE IF NOT EXISTS \${db_name} DEFAULT \ - CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};" - echo "importing \${db_name} into \${db_server}" - mysql -h "\${db_server}" -uroot -p"\${db_password}" "\${db_name}" < "\${db_file}" - done + # db servers to import into + declare -A db_server_map + db_server_map['default']=${PODIFIED_MARIADB_IP['super']} + db_server_map["nova"]=${PODIFIED_MARIADB_IP[$DEFAULT_CELL_NAME]} + db_server_map["nova_$RCELL"]=${PODIFIED_MARIADB_IP[$RCELL]} - mysql -h "\${db_server_map['default']}" -uroot -p"\${db_server_password_map['default']}" -e \ - "update nova_api.cell_mappings set name='cell1' where name='default';" - mysql -h "\${db_server_map['nova_cell1']}" -uroot -p"\${db_server_password_map['nova_cell1']}" -e \ - "delete from nova_cell1.services where host not like '%nova-cell1-%' and services.binary != 'nova-compute';" + # db server root password map + declare -A db_server_password_map + db_server_password_map['default']=${PODIFIED_DB_ROOT_PASSWORD['super']} + db_server_password_map["nova"]=${PODIFIED_DB_ROOT_PASSWORD[$DEFAULT_CELL_NAME]} + db_server_password_map["nova_$RCELL"]=${PODIFIED_DB_ROOT_PASSWORD[$RCELL]} + + cd /backup + for db_file in \$(ls ${CELL}.*.sql); do + db_name=\$(echo \${db_file} | awk -F'.' '{ print \$2; }') + # Only import cells' DBs and omit everything else + [[ "$CELL" != "default" && ! -v "db_cell_map[\${db_name}]" ]] && continue + # Route databases for importing, when extracting cell's / non-cell's DBs from 'default' cell + if [[ "$CELL" == "default" && -v "db_cell_map[\${db_name}]" ]] ; then + target=$DEFAULT_CELL_NAME + elif [[ "$CELL" == "default" && ! -v "db_cell_map[\${db_name}]" ]] ; then + target=super # 'upcall' + else + target=$RCELL + fi + renamed_db_file="\${target}_new.\${db_name}.sql" + mv -f \${db_file} \${renamed_db_file} + if [[ -v "db_name_map[\${db_name}]" ]]; then + echo "renaming $CELL cell \${db_name} to \$target \${db_name_map[\${db_name}]}" + db_name=\${db_name_map[\${db_name}]} + fi + db_server=\${db_server_map["default"]} + if [[ -v "db_server_map[\${db_name}]" ]]; then + db_server=\${db_server_map[\${db_name}]} + fi + db_password=\${db_server_password_map['default']} + if [[ -v "db_server_password_map[\${db_name}]" ]]; then + db_password=\${db_server_password_map[\${db_name}]} + fi + echo "creating $CELL cell \${db_name} in \$target \${db_server}" + mysql -h"\${db_server}" -uroot "-p\${db_password}" -e \ + "CREATE DATABASE IF NOT EXISTS \${db_name} DEFAULT \ + CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};" + echo "importing $CELL cell \${db_name} into \$target \${db_server} from \${renamed_db_file}" + mysql -h "\${db_server}" -uroot "-p\${db_password}" "\${db_name}" < "\${renamed_db_file}" + done + + if [ "$CELL" = "default" ] ; then + mysql -h "\${db_server_map['default']}" -uroot -p"\${db_server_password_map['default']}" -e \ + "update nova_api.cell_mappings set name='$DEFAULT_CELL_NAME' where name='default';" + fi + mysql -h "\${db_server_map["nova_$RCELL"]}" -uroot -p"\${db_server_password_map["nova_$RCELL"]}" -e \ + "delete from nova_${RCELL}.services where host not like '%nova_${RCELL}-%' and services.binary != 'nova-compute';" EOF +done diff --git a/tests/roles/pcp_cleanup/tasks/main.yaml b/tests/roles/pcp_cleanup/tasks/main.yaml index dfbfb1f88..dd997e312 100644 --- a/tests/roles/pcp_cleanup/tasks/main.yaml +++ b/tests/roles/pcp_cleanup/tasks/main.yaml @@ -48,4 +48,9 @@ {{ oc_header }} cd {{ install_yamls_path }} for i in {1..3}; do make crc_storage_cleanup crc_storage && break || sleep 5; done + {{ cells_env }} + for CELL in $(echo $RENAMED_CELLS); do + oc delete pvc mysql-db-openstack-$CELL-galera-0 --ignore-not-found=true + oc delete pvc persistence-rabbitmq-$CELL-server-0 --ignore-not-found=true + done when: reset_crc_storage|bool diff --git a/tests/secrets.sample.yaml b/tests/secrets.sample.yaml index e8f4a34c2..6ab9bb790 100644 --- a/tests/secrets.sample.yaml +++ b/tests/secrets.sample.yaml @@ -2,40 +2,40 @@ oc_login_command: | oc login -u kubeadmin -p {{ admin_password }} -tripleo_passwords: ~/tripleo-standalone-passwords.yaml #CUSTOMIZE_THIS +# Enumerated in terms of TripleO architecture, where 'default' cell may change its name after adoption +tripleo_passwords: #CUSTOMIZE_THIS + # This default has nothing to the real special Nova cell0, just an unfortunate iota iterator + default: ~/overcloud-passwords.yaml + #cell1: ~/cell1-passwords.yaml + #cell2: ~/cell2-passwords.yaml # Adopted OpenStack admin password. Matching the install_yamls default # to reduce developer confusion. admin_password: 12345678 #CUSTOMIZE_THIS -# DB root passwords. Source password needs to be set based on the -# original environment, podified can be customized, it matches the -# install_yamls default to reduce developer confusion. -source_db_root_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.MysqlRootPassword') | first }}" -podified_db_root_password: 12345678 - # Service account passwords (not DB passwords). -aodh_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.AodhPassword') | first }}" -barbican_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.BarbicanPassword') | first }}" -ceilometer_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.CeilometerPassword') | first }}" -cinder_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.CinderPassword') | first }}" -glance_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.GlancePassword') | first }}" -ironic_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.IronicPassword') | first }}" -manila_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.ManilaPassword') | first }}" -neutron_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.NeutronPassword') | first }}" -heat_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.HeatPassword') | first }}" -heat_stack_domain_admin_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.HeatStackDomainAdminPassword') | first }}" -heat_auth_encryption_key: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.HeatAuthEncryptionKey') | first }}" -nova_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.NovaPassword') | first }}" -octavia_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.OctaviaPassword') | first }}" -placement_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.PlacementPassword') | first }}" -swift_password: "{{ lookup('file', tripleo_passwords) | from_yaml | community.general.json_query('*.SwiftPassword') | first }}" +aodh_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.AodhPassword') | first }}" +barbican_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.BarbicanPassword') | first }}" +ceilometer_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.CeilometerPassword') | first }}" +cinder_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.CinderPassword') | first }}" +glance_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.GlancePassword') | first }}" +ironic_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.IronicPassword') | first }}" +manila_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.ManilaPassword') | first }}" +neutron_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.NeutronPassword') | first }}" +heat_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.HeatPassword') | first }}" +heat_stack_domain_admin_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.HeatStackDomainAdminPassword') | first }}" +heat_auth_encryption_key: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.HeatAuthEncryptionKey') | first }}" +nova_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.NovaPassword') | first }}" +octavia_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.OctaviaPassword') | first }}" +placement_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.PlacementPassword') | first }}" +swift_password: "{{ lookup('file', tripleo_passwords['default']) | from_yaml | community.general.json_query('*.SwiftPassword') | first }}" # FreeIPA SSH connection strings for importing the CA certificate and key ipa_ssh: "ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 podman exec -ti freeipa-server-container" # Controller SSH connection strings for the MariaDB copy procedure. # Use ":" for controler 2 and 3 if you are testing with a single controller. +# Also specify connection strings for all cells controllers controller1_ssh: "ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100" # CUSTOMIZE THIS controller2_ssh: ":" controller3_ssh: ":" diff --git a/tests/vars.sample.yaml b/tests/vars.sample.yaml index 0067d220b..4d08c07ef 100644 --- a/tests/vars.sample.yaml +++ b/tests/vars.sample.yaml @@ -4,6 +4,18 @@ install_yamls_path: ~/install_yamls #CUSTOMIZE_THIS # This flag signifies if TLS Everywhere is enabled on the source cloud enable_tlse: false +# Source MariaDB Galera cluster members {name:IP} pairs (also in additional cells) for pre-adoption checks. +# Defaults provided for a single-cell case. Complete the lists for an HA multi-cell adoption. +source_galera_members: + default: + - name: standalone.localdomain + ip: 172.17.0.100 #CUSTOMIZE_THIS + +# Source MariaDB Galera cluster VIP(s) for DB exports of all cells +# Defaults provided for a single-cell case. +source_mariadb_ip: + default: 172.17.0.2 #CUSTOMIZE_THIS + # To enable TLS-E, the standalone hostname must be set to standalone.ooo.test edpm_node_hostname: standalone.localdomain # TODO: There is no reason to change the domain depending on the type of @@ -30,8 +42,14 @@ storage_reclaim_policy: delete # or retain oc_header: | eval $(crc oc-env) -# Source MariaDB Galera cluster VIP for DB exports. -source_mariadb_ip: 172.17.0.2 #CUSTOMIZE_THIS +# Source cloud Nova compute v2 cells to adopt (all must be listed, cannot adopt cells partially) +cells: + - default + +# A cell name for the 'default' cell to take after adoption. +# Must be renamed for a single-cell deployment. Can remain 'default' for a multi-cell one. +# Defaults provided for a single-cell case. For a mult-cell, use the latest 'cells' element index + 1 +default_cell_name: cell1 # Source OS diff config ip for Tripleo source_os_diff_config_ip: 192.168.122.100 @@ -56,6 +74,10 @@ ironic_adoption: false # Run pre-adoption validation before the deploying run_pre_adoption_validation: true +# Adopt source cloud with additional compute cells v2. +# Defaults provided for a single-cell case. Enable for a multi-cell adoption. +multi_cell: false + # Supported storage backends for Cinder supported_volume_backends: #CUSTOMIZE_THIS - ceph