Add tempest to openstack-upgrade tests

tests/charm-upgrade/tests/tests.yaml

@@ -39,12 +39,11 @@ tests_options:
     default:
       smoke: True
       exclude-list:
-        # NOTE(lourot): these test cases occasionally fail (at least on
-        # focal-ussuri distro) already before charm upgrade. Blocklisting for
-        # now:
+        - octavia_tempest_plugin  # workaround for zaza-openstack-tests#603
+        # NOTE(lourot): these not-so-important test cases occasionally fail (at
+        # least on focal-ussuri distro) already before charm upgrade.
+        # Blocklisting for now. See charmed-openstack-tester#55
         - tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_create_get_update_delete
         - tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_role_create_update_show_list
         - tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_update_get_service
         - tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_too_many_objects
-      exclude-regex:
-        - octavia_tempest_plugin  # workaround for zaza-openstack-tests#603

tests/openstack-upgrade/tests/bundles/kitchen-sink-bionic-queens.yaml.j2

@@ -0,0 +1 @@
+../../../shared/kitchen-sink-bionic-queens.yaml.j2

tests/openstack-upgrade/tests/bundles/kitchen-sink-focal-victoria.yaml.j2

@@ -0,0 +1 @@
+../../../shared/kitchen-sink-focal-victoria.yaml.j2

tests/openstack-upgrade/tests/tests.yaml

@@ -1,24 +1,64 @@
 before_deploy:
-  - kitchen-sink-focal-ussuri:
+  - queens-to-rocky: &pre_deploy
     - zaza.openstack.configure.pre_deploy_certs.set_cidr_certs
+  - ussuri-to-victoria: *pre_deploy
+  - victoria-to-wallaby: *pre_deploy
 
 configure:
-  - kitchen-sink-focal-ussuri:
+  - queens-to-rocky:
+    - zaza.openstack.charm_tests.ceilometer.setup.basic_setup
+    - zaza.openstack.charm_tests.glance.setup.add_cirros_image
+    - zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image
+    - zaza.openstack.charm_tests.glance.setup.add_lts_image
+    - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
+    - zaza.openstack.charm_tests.nova.setup.create_flavors
+    - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
+    - zaza.openstack.charm_tests.keystone.setup.add_demo_user
+    - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
+  - ussuri-to-victoria: &configure_with_vault
     - zaza.openstack.charm_tests.ceilometer.setup.basic_setup
     - zaza.openstack.charm_tests.vault.setup.auto_initialize
     - zaza.openstack.charm_tests.glance.setup.add_cirros_image
+    - zaza.openstack.charm_tests.glance.setup.add_cirros_alt_image
     - zaza.openstack.charm_tests.glance.setup.add_lts_image
     - zaza.openstack.charm_tests.neutron.setup.basic_overcloud_network
     - zaza.openstack.charm_tests.nova.setup.create_flavors
     - zaza.openstack.charm_tests.nova.setup.manage_ssh_key
+    - zaza.openstack.charm_tests.keystone.setup.add_demo_user
+    - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles
+  - victoria-to-wallaby: *configure_with_vault
 
 tests:
-  - kitchen-sink-focal-ussuri:
+  - queens-to-rocky:
+    - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+    # NOTE(lourot): the openstack-upgrade action on `keystone` fails on
+    # `keystone-manage bootstrap ...`. See lp:1901982
+    # - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+
+  - ussuri-to-victoria:
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.WaitForMySQL
+    - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+    # NOTE(lourot) the pause action on `ceilometer-hacluster` occasionally fails
+    # with `Couldn't pause: Resources still running on unit`. See
+    # charmed-openstack-tester#56
+    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # NOTE(lourot): test cases involving floating IPs fail at this point. See
+    # charmed-openstack-tester#57
+    # - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+
+  - victoria-to-wallaby:
     - zaza.openstack.charm_tests.openstack_upgrade.tests.WaitForMySQL
-    - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsFocalUssuri
+    - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
+    # NOTE(lourot) the pause action on `swift-proxy` fails and exits 1. See
+    # lp:1947559
+    # - zaza.openstack.charm_tests.openstack_upgrade.tests.OpenStackUpgradeTestsByOption
+    # - zaza.openstack.charm_tests.tempest.tests.TempestTestWithKeystoneV3
 
 smoke_bundles:
-  - kitchen-sink-focal-ussuri: kitchen-sink-focal-ussuri
+  - queens-to-rocky: kitchen-sink-bionic-queens
+  - ussuri-to-victoria: kitchen-sink-focal-ussuri
+  - victoria-to-wallaby: kitchen-sink-focal-victoria
 
 target_deploy_status:
   easyrsa:
@@ -31,3 +71,37 @@ target_deploy_status:
   ceilometer:
     workload-status: blocked
     workload-status-message: "Run the ceilometer-upgrade action on the leader to initialize ceilometer and gnocchi"
+tests_options:
+  openstack-upgrade:
+    detect-using-charm: keystone
+  tempest:
+    queens-to-rocky:
+      smoke: True
+      exclude-list:
+        - octavia_tempest_plugin  # workaround for zaza-openstack-tests#603
+        # NOTE(lourot): these not-so-important test cases occasionally fail
+        # already before openstack upgrade. Blocklisting for now. See
+        # charmed-openstack-tester#55
+        - tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_create_get_update_delete
+        - tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_role_create_update_show_list
+        - tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_update_get_service
+        - tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_too_many_objects
+        # NOTE(lourot): Some volume-related test cases are failing already
+        # before openstack upgrade against bionic-queens. See
+        # charmed-openstack-tester#58
+        - tempest.api.volume
+        - tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume
+    ussuri-to-victoria:
+      smoke: True
+      exclude-list: &common_tempest_exclude_list
+        - octavia_tempest_plugin  # workaround for zaza-openstack-tests#603
+        # NOTE(lourot): these not-so-important test cases occasionally fail
+        # already before openstack upgrade. Blocklisting for now. See
+        # charmed-openstack-tester#55
+        - tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_create_get_update_delete
+        - tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_role_create_update_show_list
+        - tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_update_get_service
+        - tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_too_many_objects
+    victoria-to-wallaby:
+      smoke: True
+      exclude-list: *common_tempest_exclude_list