Tests get imageRef from nova

autoscale_cloudcafe/autoscale_fixtures/config.py

@@ -10,6 +10,13 @@ class AutoscaleConfig(ConfigSectionInterface):
     """
     SECTION_NAME = 'autoscale'
 
+    # Image ID to be used in group launch config
+    # This is not really in config any more but it still here as some
+    # tests use this instead of fetched imaged ids in AutoscaleFixture.
+    # See comment in AutoscaleFixture.setupClass
+    lc_image_ref = None
+    lc_image_ref_alt = None
+
     @property
     def tenant_id(self):
         """
@@ -73,20 +80,6 @@ def lc_flavor_ref(self):
         """
         return self.get('lc_flavor_ref')
 
-    @property
-    def lc_image_ref(self):
-        """
-        launch configuration server image id
-        """
-        return self.get('lc_image_ref')
-
-    @property
-    def lc_image_ref_alt(self):
-        """
-        Alternate launch configuration server image id
-        """
-        return self.get('lc_image_ref_alt')
-
     @property
     def sp_name(self):
         """

autoscale_cloudroast/test_repo/autoscale/fixtures.py

@@ -9,22 +9,23 @@
 from functools import partial
 from unittest import skip
 
+from autoscale_fixtures.behaviors import AutoscaleBehaviors
+from autoscale_fixtures.client import (
+    AutoscalingAPIClient, LbaasAPIClient, RackConnectV3APIClient
+)
+from autoscale_fixtures.config import AutoscaleConfig
+from autoscale_fixtures.otter_constants import OtterConstants
+
 from cafe.drivers.unittest.fixtures import BaseTestFixture
 
 from cloudcafe.auth.config import UserAuthConfig, UserConfig
 from cloudcafe.auth.provider import AuthProvider
 
 from cloudcafe.common.resources import ResourcePool
 from cloudcafe.common.tools.datagen import rand_name
+from cloudcafe.compute.images_api.client import ImagesClient
 from cloudcafe.compute.servers_api.client import ServersClient
 
-from autoscale_fixtures.behaviors import AutoscaleBehaviors
-from autoscale_fixtures.client import (
-    AutoscalingAPIClient, LbaasAPIClient, RackConnectV3APIClient
-)
-from autoscale_fixtures.config import AutoscaleConfig
-from autoscale_fixtures.otter_constants import OtterConstants
-
 
 def _make_client(access_data, service_name, region, client_cls, debug_name):
     """
@@ -89,6 +90,13 @@ def _set_up_clients():
         ServersClient,
         "Nova Compute")
 
+    _images_client = _make_client(
+        access_data,
+        autoscale_config.server_endpoint_name,
+        autoscale_config.region,
+        ImagesClient,
+        "Nova images")
+
     _lbaas_client = _make_client(
         access_data,
         autoscale_config.load_balancer_endpoint_name,
@@ -113,7 +121,8 @@ def _set_up_clients():
         raise Exception(
             "Unable to instantiate all necessary clients.")
 
-    return (_autoscale_client, _server_client, _lbaas_client, _rcv3_client)
+    return (_autoscale_client, _server_client, _images_client, _lbaas_client,
+            _rcv3_client)
 
 
 # Global testing state - unfortunate, but only needs to be done once and also
@@ -132,7 +141,26 @@ def _set_up_clients():
 
 _rcv3_cloud_network = autoscale_config.rcv3_cloud_network
 
-autoscale_client, server_client, lbaas_client, rcv3_client = _set_up_clients()
+(autoscale_client, server_client, images_client,
+ lbaas_client, rcv3_client) = _set_up_clients()
+
+
+def image_ids_with_and_without_name(images_client, name="Ubuntu"):
+    """
+    Fetch image IDs, from Nova that can be used as imageRef in tests
+    Note: Serves the same purpose as integration.lib.nova.fetch_ubuntu_image_id
+    in trial integration tests
+    """
+    images = images_client.list_images().entity
+    base, other = None, None
+    for image in images:
+        if name in image.name:
+            base = image.id
+        else:
+            other = image.id
+    if base is None:
+        raise Exception("Couldn't get {} image".format(name))
+    return base, other
 
 
 def only_run_if_mimic_is(should_mimic_be_available):
@@ -173,8 +201,6 @@ def setUpClass(cls):
         cls.gc_max_entities = int(cls.autoscale_config.gc_max_entities)
         cls.lc_name = cls.autoscale_config.lc_name
         cls.lc_flavor_ref = cls.autoscale_config.lc_flavor_ref
-        cls.lc_image_ref = cls.autoscale_config.lc_image_ref
-        cls.lc_image_ref_alt = cls.autoscale_config.lc_image_ref_alt
         cls.sp_name = rand_name(cls.autoscale_config.sp_name)
         cls.sp_cooldown = int(cls.autoscale_config.sp_cooldown)
         cls.sp_change = int(cls.autoscale_config.sp_change)
@@ -214,6 +240,15 @@ def setUpClass(cls):
         cls.lbaas_client = lbaas_client
         cls.rcv3_client = rcv3_client
 
+        # ImageRefs of ununtu and non-ubuntu images that will be used
+        # when creating groups
+        image_refs = image_ids_with_and_without_name(images_client)
+        cls.lc_image_ref, cls.lc_image_ref_alt = image_refs
+        # Unfortunately some of the tests use imageRef from config instead of
+        # this class. So, storing the same in config too
+        autoscale_config.lc_image_ref = cls.lc_image_ref
+        autoscale_config.lc_image_ref_alt = cls.lc_image_ref_alt
+
         cls.rcv3_load_balancer_pool = _rcv3_load_balancer_pool
         cls.rcv3_cloud_network = _rcv3_cloud_network
 

otter/integration/lib/autoscale.py

@@ -12,13 +12,15 @@
 
 from testtools.matchers import MatchesPredicateWithParams
 
+from toolz.dicttoolz import get_in, update_in
+
 import treq
 
 from twisted.internet import reactor
 from twisted.internet.defer import gatherResults, inlineCallbacks, returnValue
 from twisted.python.log import msg
 
-from otter.integration.lib.nova import NovaServer
+from otter.integration.lib.nova import NovaServer, fetch_ubuntu_image_id
 from otter.integration.lib.utils import diagnose
 
 from otter.util.deferredutils import retry_and_timeout
@@ -59,8 +61,7 @@ def create_scaling_group_dict(
     payload.  Note: this function does NOT create a scaling group.
 
     :param str image_ref: An OpenStack image reference ID (typically a UUID).
-        If not provided, the content of the AS_IMAGE_REF environment variable
-        will be taken as default.  If that doesn't exist, "" will be used.
+        If not provided, "" will be used.
     :param str flavor_ref: As with image_ref above, but for the launch config's
         flavor setting.
     :param int min_entities: The minimum number of servers to bring up when
@@ -87,8 +88,6 @@ def create_scaling_group_dict(
         flavor and image IDs.
     """
 
-    if not image_ref:
-        image_ref = os.environ['AS_IMAGE_REF']
     if not flavor_ref:
         flavor_ref = os.environ['AS_FLAVOR_REF']
     if not name:
@@ -100,7 +99,7 @@ def create_scaling_group_dict(
             "args": {
                 "server": {
                     "flavorRef": flavor_ref,
-                    "imageRef": image_ref,
+                    "imageRef": "" if image_ref is None else image_ref,
                     "networks": [
                         {
                             "uuid": "11111111-1111-1111-1111-111111111111"
@@ -360,6 +359,14 @@ def debug_print(resp_tuple):
             .addCallback(debug_print)
         )
 
+    def _image_ref(self):
+        """
+        Get group config's imageRef
+        """
+        return get_in(
+            self.group_config,
+            ["launchConfiguration", "args", "server", "imageRef"])
+
     @diagnose("AS", "Creating scaling group")
     def start(self, rcs, test):
         """Create a scaling group.
@@ -380,17 +387,26 @@ def record_results(resp):
                 pp.pprint(rcs.groups)
             return rcs
 
-        return (
-            self.treq.post(
+        def send(image_id):
+            if image_id is not None:
+                self.group_config = update_in(
+                    self.group_config,
+                    ["launchConfiguration", "args", "server", "imageRef"],
+                    lambda _: image_id)
+            d = self.treq.post(
                 "%s/groups" % str(rcs.endpoints["otter"]),
                 json.dumps(self.group_config),
                 headers=headers(str(rcs.token)),
-                pool=self.pool
-            )
-            .addCallback(check_success, [201])
-            .addCallback(self.treq.json_content)
-            .addCallback(record_results)
-        )
+                pool=self.pool)
+            d.addCallback(check_success, [201])
+            d.addCallback(self.treq.json_content)
+            return d.addCallback(record_results)
+
+        # If image_ref is not configured, get it from nova
+        if self._image_ref() is None:
+            return fetch_ubuntu_image_id(rcs, self.pool).addCallback(send)
+        else:
+            return send(None)
 
     @diagnose("AS+Nova", "Getting the servicenet IPs of the active servers")
     def get_servicenet_ips(self, rcs, server_ids=None):

otter/integration/lib/nova.py

@@ -141,6 +141,36 @@ def list_servers(rcs, pool, _treq=treq):
     ).addCallback(check_success, [200]).addCallback(_treq.json_content)
 
 
+@diagnose("nova", "Listing all images")
+def list_images(rcs, pool, _treq=treq):
+    """
+    Get a list of all images.
+    """
+    params = {'limit': 10000}
+    return _treq.get(
+        "{}/images".format(rcs.endpoints['nova']),
+        params=params,
+        headers=headers(str(rcs.token)),
+        pool=pool
+    ).addCallback(check_success, [200]).addCallback(_treq.json_content)
+
+
+@inlineCallbacks
+def fetch_ubuntu_image_id(rcs, pool):
+    """
+    Get image ID from nova that can be used in creating servers or
+    scaling group
+    Note: Serves the same purpose as fixtures.image_ids_with_and_without_name
+    in cloudcafe tests
+    """
+    images_resp = yield list_images(rcs, pool)
+    for image in images_resp["images"]:
+        if image["name"].startswith("Ubuntu"):
+            returnValue(image["id"])
+    else:
+        returnValue(images_resp["images"][0]["id"])
+
+
 @diagnose("nova", "Creating server")
 def create_server(rcs, pool, server_args, _treq=treq):
     """

otter/integration/lib/trial_tools.py

@@ -38,6 +38,7 @@
 from otter.integration.lib.nova import (
     create_server,
     delete_servers,
+    fetch_ubuntu_image_id,
     wait_for_servers
 )
 
@@ -57,7 +58,6 @@
 password = os.environ['AS_PASSWORD']
 endpoint = os.environ['AS_IDENTITY']
 flavor_ref = os.environ['AS_FLAVOR_REF']
-image_ref = os.environ['AS_IMAGE_REF']
 region = os.environ['AS_REGION']
 scheduler_interval = float(os.environ.get("AS_SCHEDULER_INTERVAL", "10"))
 otter_build_timeout = float(os.environ.get("AS_BUILD_TIMEOUT_SECONDS", "30"))
@@ -216,7 +216,6 @@ def create_group(self, **kwargs):
                 'use_lbs',
                 [clb.scaling_group_spec() for clb in self.clbs])
 
-        kwargs.setdefault("image_ref", image_ref)
         kwargs.setdefault("flavor_ref", flavor_ref)
         kwargs.setdefault("min_entities", 0)
 
@@ -293,8 +292,9 @@ def create_servers(self, rcs, num, wait_for=None):
 
         :return: an iterable of server details JSON of the created servers.
         """
+        image_id = yield fetch_ubuntu_image_id(rcs, self.pool)
         as_args = create_scaling_group_dict(
-            image_ref=image_ref,
+            image_ref=image_id,
             flavor_ref=flavor_ref)
         server_args = as_args['launchConfiguration']['args']
         server_args['server']['name'] = "autogenerated-non-as-test-server"

otter/integration/tests/test_convergence.py

@@ -1028,7 +1028,6 @@ def test_error_reasons_are_updated(self):
             server_name_prefix="no-such-key",
             key_name="invalid_key_name"
         )
-        launch_config = deepcopy(group.group_config['launchConfiguration'])
 
         if not not_mimic():
             # if this is mimic, we have to make server creation fail.
@@ -1048,6 +1047,7 @@ def test_error_reasons_are_updated(self):
 
         # group should go into error
         yield group.start(self.rcs, self)
+        launch_config = deepcopy(group.group_config['launchConfiguration'])
         yield group.wait_for_state(
             self.rcs,
             ContainsDict({

requirements/mimic.txt

@@ -0,0 +1 @@
+git+https://github.com/rackerlabs/mimic.git@b4ffe4b066430404f5363d7506a6228fcc2d5b4f