Fix series-upgrade tests (openstack-charmers#406)

A couple of changes here:

 * Ensure that the post-upgrade-hook runs BEFORE the config-changed to
   set the openstack-origin/source back to distro.  The former behaviour
   breaks keystone quite badly.
 * Ensure that the charm name is used, as discovered from the model, for
   rabbitmq-server and percona-cluster to cope with different names for
   the application in the model vs the charm name from the charm-store.
 * Check whether the machine needs to be rebooted after the dist-upgrade
   (before the do-release-upgrade), and reboot the machine first.
   Otherwise, do-release-upgrade will fail.

requirements.txt

@@ -1,7 +1,7 @@
 aiounittest
 async_generator
 boto3
-juju
+juju!=2.8.3  # blacklist 2.8.3 as it appears to have a connection bug
 juju_wait
 PyYAML<=4.2,>=3.0
 flake8>=2.2.4

zaza/openstack/charm_tests/series_upgrade/tests.py

@@ -81,7 +81,7 @@ def test_200_run_series_upgrade(self):
                 logging.info(
                     "Running complete-cluster-series-upgrade action on leader")
                 model.run_action_on_leader(
-                    'rabbitmq-server',
+                    charm_name,
                     'complete-cluster-series-upgrade',
                     action_params={})
                 model.block_until_all_units_idle()
@@ -90,7 +90,7 @@ def test_200_run_series_upgrade(self):
                 logging.info(
                     "Running complete-cluster-series-upgrade action on leader")
                 model.run_action_on_leader(
-                    'mysql',
+                    charm_name,
                     'complete-cluster-series-upgrade',
                     action_params={})
                 model.block_until_all_units_idle()

zaza/openstack/utilities/series_upgrade.py

@@ -14,11 +14,13 @@
 
 """Collection of functions for testing series upgrade."""
 
+import asyncio
 import collections
 import copy
 import concurrent
 import logging
 import os
+import time
 
 from zaza import model
 from zaza.charm_lifecycle import utils as cl_utils
@@ -642,14 +644,14 @@ def series_upgrade(unit_name, machine_num,
     model.block_until_unit_wl_status(unit_name, "blocked")
     logging.info("Waiting for model idleness")
     model.block_until_all_units_idle()
+    logging.info("Complete series upgrade on {}".format(machine_num))
+    model.complete_series_upgrade(machine_num)
+    model.block_until_all_units_idle()
     logging.info("Set origin on {}".format(application))
     # Allow for charms which have neither source nor openstack-origin
     if origin:
         os_utils.set_origin(application, origin)
     model.block_until_all_units_idle()
-    logging.info("Complete series upgrade on {}".format(machine_num))
-    model.complete_series_upgrade(machine_num)
-    model.block_until_all_units_idle()
     logging.info("Running run_post_upgrade_functions {}".format(
         post_upgrade_functions))
     run_post_upgrade_functions(post_upgrade_functions)
@@ -882,6 +884,21 @@ def dist_upgrade(unit_name):
         """-o "Dpkg::Options::=--force-confdef" """
         """-o "Dpkg::Options::=--force-confold" dist-upgrade""")
     model.run_on_unit(unit_name, dist_upgrade_cmd)
+    rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required")
+    if "Stdout" in rdict and "restart" in rdict["Stdout"].lower():
+        logging.info("dist-upgrade required reboot {}".format(unit_name))
+        os_utils.reboot(unit_name)
+        logging.info("Waiting for workload status 'unknown' on {}"
+                     .format(unit_name))
+        model.block_until_unit_wl_status(unit_name, "unknown")
+        logging.info("Waiting for workload status to return to normal on {}"
+                     .format(unit_name))
+        model.block_until_unit_wl_status(
+            unit_name, "unknown", negate_match=True)
+        logging.info("Waiting for model idleness")
+        # pause for a big
+        time.sleep(5.0)
+        model.block_until_all_units_idle()
 
 
 async def async_dist_upgrade(unit_name):
@@ -902,6 +919,21 @@ async def async_dist_upgrade(unit_name):
         """-o "Dpkg::Options::=--force-confdef" """
         """-o "Dpkg::Options::=--force-confold" dist-upgrade""")
     await model.async_run_on_unit(unit_name, dist_upgrade_cmd)
+    rdict = await model.async_run_on_unit(unit_name,
+                                          "cat /var/run/reboot-required")
+    if "Stdout" in rdict and "restart" in rdict["Stdout"].lower():
+        logging.info("dist-upgrade required reboot {}".format(unit_name))
+        await os_utils.async_reboot(unit_name)
+        logging.info("Waiting for workload status 'unknown' on {}"
+                     .format(unit_name))
+        await model.async_block_until_unit_wl_status(unit_name, "unknown")
+        logging.info("Waiting for workload status to return to normal on {}"
+                     .format(unit_name))
+        await model.async_block_until_unit_wl_status(
+            unit_name, "unknown", negate_match=True)
+        logging.info("Waiting for model idleness")
+        await asyncio.sleep(5.0)
+        await model.async_block_until_all_units_idle()
 
 
 def do_release_upgrade(unit_name):