Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Watcher zed #1178

Open
wants to merge 9 commits into
base: stable/zed
Choose a base branch
from
Open
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ python-neutronclient
python-novaclient
python-octaviaclient
python-swiftclient
python-watcherclient
tenacity
paramiko

Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
'python-ceilometerclient',
'python-cinderclient<6.0.0',
'python-swiftclient<3.9.0',
'python-watcherclient',
# 'zaza@git+https://github.com/openstack-charmers/zaza.git#egg=zaza',
'zaza',
]
Expand Down
2 changes: 1 addition & 1 deletion zaza/openstack/charm_tests/tempest/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest'
TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon',
'ironic', 'manila', 'neutron', 'nova', 'octavia',
'sahara', 'swift', 'trove', 'zaqar']
'sahara', 'swift', 'trove', 'watcher', 'zaqar']


def render_tempest_config_keystone_v2():
Expand Down
15 changes: 15 additions & 0 deletions zaza/openstack/charm_tests/watcher/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Copyright 2023 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Collection of code for setting up and testing Watcher."""
146 changes: 146 additions & 0 deletions zaza/openstack/charm_tests/watcher/tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
# Copyright 2023 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Encapsulate Cinder testing."""

import logging
import tenacity

import zaza.openstack.charm_tests.test_utils as test_utils
import zaza.openstack.utilities.openstack as openstack_utils
import zaza.openstack.configure.guest as guest
import watcherclient.common.apiclient.exceptions as watcherclient_exceptions

logger = logging.getLogger(__name__)


class WatcherTests(test_utils.OpenStackBaseTest):
"""Encapsulate Watcher tests."""

AUDIT_TEMPLATE_NAME = 'zaza-at1'
AUDIT_TEMPLATE_GOAL = 'server_consolidation'
AUDIT_TEMPLATE_STRATEGY = 'vm_workload_consolidation'
AUDIT_TYPE = 'ONESHOT'

BLOCK_SECS = 600

@classmethod
def setUpClass(cls):
"""Configure Watcher tests class."""
super().setUpClass()
cls.watcher_client = openstack_utils.get_watcher_session_client(
cls.keystone_session,
)

def test_server_consolidation(self):
"""Test server consolidation policy."""
for i, attempt in enumerate(tenacity.Retrying(
wait=tenacity.wait_fixed(2),
retry=tenacity.retry_if_exception_type(AssertionError),
reraise=True,
stop=tenacity.stop_after_attempt(4))):
with attempt:
logger.info('Attempt number %d', i + 1)
self._check_server_consolidation()

def _check_server_consolidation(self):
try:
at = self.watcher_client.audit_template.get(
self.AUDIT_TEMPLATE_NAME
)
logger.info('Re-using audit template: %s (%s)', at.name, at.uuid)
except watcherclient_exceptions.NotFound:
at = self.watcher_client.audit_template.create(
name=self.AUDIT_TEMPLATE_NAME,
goal=self.AUDIT_TEMPLATE_GOAL,
strategy=self.AUDIT_TEMPLATE_STRATEGY,
)
logger.info('Audit template created: %s (%s)', at.name, at.uuid)

hypervisors_before = {
'enabled': [],
'disabled': [],
}
for i, hypervisor in enumerate(self.nova_client.hypervisors.list()):
hypervisors_before[hypervisor.status].append(
hypervisor.hypervisor_hostname
)
# There is a need to have instances running to allow Watcher not
# fail when calling gnocchi for cpu_util metric measures.
logger.info('Launching instance on hypervisor %s',
hypervisor.hypervisor_hostname)
guest.launch_instance(
'cirros',
vm_name='zaza-watcher-%s' % i,
perform_connectivity_check=False,
host=hypervisor.hypervisor_hostname,
nova_api_version='2.74',
)

audit = self.watcher_client.audit.create(
audit_template_uuid=at.uuid,
audit_type=self.AUDIT_TYPE,
parameters={'period': 600, 'granularity': 300},
)
logger.info('Audit created: %s', audit.uuid)

openstack_utils.resource_reaches_status(self.watcher_client.audit,
audit.uuid,
msg='audit',
resource_attribute='state',
expected_status='SUCCEEDED',
wait_iteration_max_time=180,
stop_after_attempt=30,
stop_status='FAILED')
action_plans = self.watcher_client.action_plan.list(audit=audit.uuid)
assert len(action_plans) == 1
action_plan = action_plans[0]
actions = self.watcher_client.action.list(action_plan=action_plan.uuid)

for action in actions:
logger.info('Action %s: %s %s',
action.uuid, action.state, action.action_type)
self.assertEqual(action.state, 'PENDING',
'Action %s state %s != PENDING' % (action.uuid,
action.state))

self.watcher_client.action_plan.start(action_plan.uuid)

openstack_utils.resource_reaches_status(
self.watcher_client.action_plan,
action_plan.uuid,
resource_attribute='state',
expected_status='SUCCEEDED',
wait_iteration_max_time=180,
stop_after_attempt=30,
)
# get fresh list of action objects
actions = self.watcher_client.action.list(action_plan=action_plan.uuid)
for action in actions:
logger.info('Action %s: %s %s',
action.uuid, action.state, action.action_type)
self.assertEqual(
action.state, 'SUCCEEDED',
'Action %s state %s != SUCCEEDED' % (action.uuid,
action.state),
)

hypervisors_after = {
'enabled': [],
'disabled': [],
}
for i, hypervisor in enumerate(self.nova_client.hypervisors.list()):
hypervisors_after[hypervisor.status].append(
hypervisor.hypervisor_hostname
)
self.assertNotEqual(hypervisors_before, hypervisors_after)
24 changes: 20 additions & 4 deletions zaza/openstack/configure/guest.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
private_network_name=None, image_name=None,
flavor_name=None, external_network_name=None, meta=None,
userdata=None, attach_to_external_network=False,
keystone_session=None):
keystone_session=None, perform_connectivity_check=True,
host=None, nova_api_version=None
):
"""Launch an instance.

:param instance_key: Key to collect associated config data with.
Expand Down Expand Up @@ -123,13 +125,22 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
:type attach_to_external_network: bool
:param keystone_session: Keystone session to use.
:type keystone_session: Optional[keystoneauth1.session.Session]
:param perform_connectivity_check: Whether to perform a connectivity check.
:type perform_connectivity_check: bool
:param host: Requested host to create servers
:type host: str
:param nova_api_version: Nova API version to use
:type nova_api_version: str | None
:returns: the created instance
:rtype: novaclient.Server
"""
if not keystone_session:
keystone_session = openstack_utils.get_overcloud_keystone_session()

nova_client = openstack_utils.get_nova_session_client(keystone_session)
nova_client = openstack_utils.get_nova_session_client(
keystone_session,
version=nova_api_version,
)
neutron_client = openstack_utils.get_neutron_session_client(
keystone_session)

Expand Down Expand Up @@ -177,7 +188,9 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
key_name=nova_utils.KEYPAIR_NAME,
meta=meta,
nics=nics,
userdata=userdata)
userdata=userdata,
host=host,
)

# Test Instance is ready.
logging.info('Checking instance is active')
Expand All @@ -188,7 +201,10 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
# NOTE(lourot): in some models this may sometimes take more than 15
# minutes. See lp:1945991
wait_iteration_max_time=120,
stop_after_attempt=16)
stop_after_attempt=16,
stop_status='ERROR',
msg='instance',
)

logging.info('Checking cloud init is complete')
openstack_utils.cloud_init_complete(
Expand Down
6 changes: 6 additions & 0 deletions zaza/openstack/utilities/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,3 +220,9 @@ class LoadBalancerUnrecoverableError(Exception):
"""The LoadBalancer has reached to an unrecoverable error state."""

pass


class StatusError(Exception):
"""The resource status is in error state."""

pass
48 changes: 41 additions & 7 deletions zaza/openstack/utilities/openstack.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
v3,
v2,
)
from watcherclient import client as watcher_client
import zaza.openstack.utilities.cert as cert
import zaza.utilities.deployment_env as deployment_env
import zaza.utilities.juju as juju_utils
Expand Down Expand Up @@ -347,16 +348,18 @@ def get_designate_session_client(**kwargs):
**kwargs)


def get_nova_session_client(session, version=2):
def get_nova_session_client(session, version=None):
"""Return novaclient authenticated by keystone session.

:param session: Keystone session object
:type session: keystoneauth1.session.Session object
:param version: Version of client to request.
:type version: float
:type version: float | str | None
:returns: Authenticated novaclient
:rtype: novaclient.Client object
"""
if not version:
version = 2
return novaclient_client.Client(version, session=session)


Expand Down Expand Up @@ -497,6 +500,15 @@ def get_manila_session_client(session, version='2'):
return manilaclient.Client(session=session, client_version=version)


def get_watcher_session_client(session):
"""Return Watcher client authenticated by keystone session.

:param session: Keystone session object
:returns: Authenticated watcher client
"""
return watcher_client.get_client(session=session, api_version='1')


def get_keystone_scope(model_name=None):
"""Return Keystone scope based on OpenStack release of the overcloud.

Expand Down Expand Up @@ -2394,7 +2406,8 @@ def download_image(image_url, target_file):
def _resource_reaches_status(resource, resource_id,
expected_status='available',
msg='resource',
resource_attribute='status'):
resource_attribute='status',
stop_status=None):
"""Wait for an openstack resources status to reach an expected status.

Wait for an openstack resources status to reach an expected status
Expand All @@ -2412,11 +2425,26 @@ def _resource_reaches_status(resource, resource_id,
:type msg: str
:param resource_attribute: Resource attribute to check against
:type resource_attribute: str
:param stop_status: Stop retrying when this status is reached
:type stop_status: str
:raises: AssertionError
:raises: StatusError
"""
resource_status = getattr(resource.get(resource_id), resource_attribute)
try:
res_object = resource.get(resource_id)
resource_status = getattr(res_object, resource_attribute)
except AttributeError:
logging.error('attributes available: %s' % str(dir(res_object)))
raise

logging.info("{}: resource {} in {} state, waiting for {}".format(
msg, resource_id, resource_status, expected_status))
if stop_status:
if isinstance(stop_status, list) and resource_status in stop_status:
raise exceptions.StatusError(resource_status, expected_status)
elif isinstance(stop_status, str) and resource_status == stop_status:
raise exceptions.StatusError(resource_status, expected_status)

assert resource_status == expected_status


Expand All @@ -2428,6 +2456,7 @@ def resource_reaches_status(resource,
wait_exponential_multiplier=1,
wait_iteration_max_time=60,
stop_after_attempt=8,
stop_status=None,
):
"""Wait for an openstack resources status to reach an expected status.

Expand All @@ -2452,23 +2481,28 @@ def resource_reaches_status(resource,
:param wait_iteration_max_time: Wait a max of wait_iteration_max_time
between retries.
:type wait_iteration_max_time: int
:param stop_after_attempt: Stop after stop_after_attempt retires.
:param stop_after_attempt: Stop after stop_after_attempt retries
:type stop_after_attempt: int
:raises: AssertionError
:raises: StatusError
"""
retryer = tenacity.Retrying(
wait=tenacity.wait_exponential(
multiplier=wait_exponential_multiplier,
max=wait_iteration_max_time),
reraise=True,
stop=tenacity.stop_after_attempt(stop_after_attempt))
stop=tenacity.stop_after_attempt(stop_after_attempt),
retry=tenacity.retry_if_exception_type(AssertionError),
)
retryer(
_resource_reaches_status,
resource,
resource_id,
expected_status,
msg,
resource_attribute)
resource_attribute,
stop_status,
)


def _resource_removed(resource, resource_id, msg="resource"):
Expand Down
Loading