From 348e7211cd8837f34db3f378c59c5bf3163eea0d Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 29 Oct 2018 11:32:43 -0700 Subject: [PATCH 1/3] Explicit Model Timeout Exception When we timeout waiting for the model to be idle and have specific workload statuses make that explicit. --- unit_tests/test_zaza_model.py | 34 ++++++++++++++++-- zaza/model.py | 67 +++++++++++++++++++++-------------- 2 files changed, 73 insertions(+), 28 deletions(-) diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index 2b80080..2cfe202 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -14,6 +14,7 @@ import aiounittest import asyncio.futures +import concurrent import mock import unit_tests.utils as ut_utils @@ -386,8 +387,14 @@ class TestModel(ut_utils.BaseTestCase): def _application_states_setup(self, setup, units_idle=True): self.system_ready = True + self._block_until_calls = 0 - async def _block_until(f, timeout=None): + async def _block_until(f, timeout=0): + # Mimic timeouts + timeout = timeout + self._block_until_calls + self._block_until_calls += 1 + if timeout == -1: + raise concurrent.futures._base.TimeoutError result = f() if not result: self.system_ready = False @@ -539,7 +546,7 @@ class TestModel(ut_utils.BaseTestCase): timeout=1) self.assertTrue(self.system_ready) - def test_wait_for_application_states_bespoke_msg_bloked_ok(self): + def test_wait_for_application_states_bespoke_msg_blocked_ok(self): self._application_states_setup({ 'workload-status': 'blocked', 'workload-status-message': 'Sure, I could do something'}) @@ -551,6 +558,29 @@ class TestModel(ut_utils.BaseTestCase): timeout=1) self.assertTrue(self.system_ready) + def test_wait_for_application_states_idle_timeout(self): + self._application_states_setup({ + 'agent-status': 'executing', + 'workload-status': 'blocked', + 'workload-status-message': 'Sure, I could do something'}) + with self.assertRaises(model.ModelTimeout) as timeout: + model.wait_for_application_states('modelname', timeout=-2) + self.assertEqual( + timeout.exception.args[0], + "Zaza has timed out waiting on the model to reach idle state.") + + def test_wait_for_application_states_timeout(self): + self._application_states_setup({ + 'agent-status': 'executing', + 'workload-status': 'blocked', + 'workload-status-message': 'Sure, I could do something'}) + with self.assertRaises(model.ModelTimeout) as timeout: + model.wait_for_application_states('modelname', timeout=-3) + self.assertEqual( + timeout.exception.args[0], + "Zaza has timed out waiting on the model to reach expected " + "workload statuses.") + def test_get_current_model(self): self.patch_object(model, 'Model') self.Model.return_value = self.Model_mock diff --git a/zaza/model.py b/zaza/model.py index 8a3efff..e1a8c34 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -27,6 +27,7 @@ import subprocess import tempfile import yaml from oslo_config import cfg +import concurrent from juju.errors import JujuError from juju.model import Model @@ -36,6 +37,12 @@ from zaza import sync_wrapper CURRENT_MODEL = None +class ModelTimeout(Exception): + """Model timeout exception.""" + + pass + + def set_juju_model(model_name): """Point environment at the given model. @@ -725,32 +732,40 @@ async def async_wait_for_application_states(model_name=None, states=None, lambda: len(model.units) > 0 ) logging.info("Waiting for all units to be idle") - await model.block_until( - lambda: model.all_units_idle(), timeout=timeout) - for application in model.applications: - check_info = states.get(application, {}) - for unit in model.applications[application].units: - logging.info("Checking workload status of {}".format( - unit.entity_id)) - await model.block_until( - lambda: check_unit_workload_status( - model, - unit, - check_info.get('workload-status', 'active')), - timeout=timeout) - check_msg = check_info.get('workload-status-message') - logging.info("Checking workload status message of {}".format( - unit.entity_id)) - if check_msg is not None: - prefixes = (check_msg) - else: - prefixes = approved_message_prefixes - await model.block_until( - lambda: check_unit_workload_status_message( - model, - unit, - prefixes=prefixes), - timeout=timeout) + try: + await model.block_until( + lambda: model.all_units_idle(), timeout=timeout) + except concurrent.futures._base.TimeoutError as e: + raise ModelTimeout("Zaza has timed out waiting on the model to " + "reach idle state.") + try: + for application in model.applications: + check_info = states.get(application, {}) + for unit in model.applications[application].units: + logging.info("Checking workload status of {}".format( + unit.entity_id)) + await model.block_until( + lambda: check_unit_workload_status( + model, + unit, + check_info.get('workload-status', 'active')), + timeout=timeout) + check_msg = check_info.get('workload-status-message') + logging.info("Checking workload status message of {}" + .format(unit.entity_id)) + if check_msg is not None: + prefixes = (check_msg) + else: + prefixes = approved_message_prefixes + await model.block_until( + lambda: check_unit_workload_status_message( + model, + unit, + prefixes=prefixes), + timeout=timeout) + except concurrent.futures._base.TimeoutError as e: + raise ModelTimeout("Zaza has timed out waiting on the model to " + "reach expected workload statuses.") wait_for_application_states = sync_wrapper(async_wait_for_application_states) From 1d709bbdddbf7d40859bb593fb08aac9271965d1 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 31 Oct 2018 07:39:38 -0700 Subject: [PATCH 2/3] Review requests Pep8 fixes. --- unit_tests/test_zaza_model.py | 2 +- zaza/model.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/test_zaza_model.py b/unit_tests/test_zaza_model.py index 2cfe202..c8d0ec2 100644 --- a/unit_tests/test_zaza_model.py +++ b/unit_tests/test_zaza_model.py @@ -394,7 +394,7 @@ class TestModel(ut_utils.BaseTestCase): timeout = timeout + self._block_until_calls self._block_until_calls += 1 if timeout == -1: - raise concurrent.futures._base.TimeoutError + raise concurrent.futures._base.TimeoutError("Timeout", 1) result = f() if not result: self.system_ready = False diff --git a/zaza/model.py b/zaza/model.py index e1a8c34..7e36b0b 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -735,7 +735,7 @@ async def async_wait_for_application_states(model_name=None, states=None, try: await model.block_until( lambda: model.all_units_idle(), timeout=timeout) - except concurrent.futures._base.TimeoutError as e: + except concurrent.futures._base.TimeoutError: raise ModelTimeout("Zaza has timed out waiting on the model to " "reach idle state.") try: @@ -763,7 +763,7 @@ async def async_wait_for_application_states(model_name=None, states=None, unit, prefixes=prefixes), timeout=timeout) - except concurrent.futures._base.TimeoutError as e: + except concurrent.futures._base.TimeoutError: raise ModelTimeout("Zaza has timed out waiting on the model to " "reach expected workload statuses.") From 1fb704507b9c2aa781176471351c9860ea76e98b Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 31 Oct 2018 08:24:47 -0700 Subject: [PATCH 3/3] Use model.applications as a dict --- zaza/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/model.py b/zaza/model.py index 7e36b0b..dc391b1 100644 --- a/zaza/model.py +++ b/zaza/model.py @@ -739,9 +739,9 @@ async def async_wait_for_application_states(model_name=None, states=None, raise ModelTimeout("Zaza has timed out waiting on the model to " "reach idle state.") try: - for application in model.applications: + for application, app_data in model.applications.items(): check_info = states.get(application, {}) - for unit in model.applications[application].units: + for unit in app_data.units: logging.info("Checking workload status of {}".format( unit.entity_id)) await model.block_until(