From 9fb488efef20f28501c5de931c311ff2bf33470a Mon Sep 17 00:00:00 2001 From: Bas de Bruijne <43334573+Basdbruijne@users.noreply.github.com> Date: Mon, 23 Jan 2023 08:39:10 -0400 Subject: [PATCH] Update series upgrade automation (#952) * update series upgrade test * bug fixes * bug fixes --- .../utilities/test_zaza_utilities_generic.py | 31 +++++++- ..._zaza_utilities_parallel_series_upgrade.py | 64 +++++++++++++---- .../test_zaza_utilities_series_upgrade.py | 27 +++++-- .../series_upgrade/parallel_tests.py | 26 +++++-- zaza/openstack/utilities/generic.py | 37 +++++----- .../utilities/parallel_series_upgrade.py | 42 +++++------ zaza/openstack/utilities/series_upgrade.py | 71 +++++++------------ 7 files changed, 183 insertions(+), 115 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index f1a648f..4dc742f 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import mock import unit_tests.utils as ut_utils from zaza.openstack.utilities import generic as generic_utils @@ -190,7 +191,13 @@ class TestGenericUtils(ut_utils.BaseTestCase): def test_set_origin(self): "application, origin='openstack-origin', pocket='distro'):" - self.patch_object(generic_utils.model, "set_application_config") + self.patch_object(generic_utils.model, "async_set_application_config", + new_callable=mock.MagicMock(), + name="set_application_config") + future = asyncio.Future() + future.set_result(None) + self.set_application_config.return_value = future + _application = "application" _origin = "source" _pocket = "cloud:fake-cloud" @@ -198,6 +205,28 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.set_application_config.assert_called_once_with( _application, {_origin: _pocket}) + def test_set_origin_auto(self): + self.patch_object(generic_utils.model, "async_set_application_config", + new_callable=mock.MagicMock(), + name="set_application_config") + set_future = asyncio.Future() + set_future.set_result(None) + self.set_application_config.return_value = set_future + + self.patch_object(generic_utils.model, "async_get_application_config", + new_callable=mock.MagicMock(), + name="get_application_config") + get_future = asyncio.Future() + get_future.set_result({"source": {"value": "distro"}}) + self.get_application_config.return_value = get_future + + _application = "application" + _origin = "auto" + _pocket = "cloud:fake-cloud" + generic_utils.set_origin(_application, origin=_origin, pocket=_pocket) + self.set_application_config.assert_called_once_with( + _application, {"source": _pocket}) + def test_set_dpkg_non_interactive_on_unit(self): self.patch_object(generic_utils, "model") _unit_name = "app/1" diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 70ffd3b..b4aaf26 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -19,6 +19,7 @@ import unit_tests.utils as ut_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.series_upgrade as series_upgrade import zaza.openstack.utilities.parallel_series_upgrade as upgrade_utils +import zaza FAKE_STATUS = { 'can-upgrade-to': '', @@ -88,7 +89,7 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): def test_app_config_openstack_charm(self): expected = { - 'origin': 'openstack-origin', + 'origin': 'auto', 'pause_non_leader_subordinate': True, 'pause_non_leader_primary': True, 'post_upgrade_functions': [], @@ -189,7 +190,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') @mock.patch.object(upgrade_utils, 'maybe_pause_things') @mock.patch.object(upgrade_utils, 'series_upgrade_machine') - async def test_parallel_series_upgrade_mongo( + async def test_async_parallel_series_upgrade_mongo( self, mock_series_upgrade_machine, mock_maybe_pause_things, @@ -200,7 +201,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): self.juju_status.return_value.applications.__getitem__.return_value = \ FAKE_STATUS_MONGO upgrade_config = upgrade_utils.app_config('mongodb') - await upgrade_utils.parallel_series_upgrade( + await upgrade_utils.async_parallel_series_upgrade( 'mongodb', from_series='trusty', to_series='xenial', @@ -249,7 +250,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') @mock.patch.object(upgrade_utils, 'maybe_pause_things') @mock.patch.object(upgrade_utils, 'series_upgrade_machine') - async def test_serial_series_upgrade_mongo( + async def test_async_serial_series_upgrade_mongo( self, mock_series_upgrade_machine, mock_maybe_pause_things, @@ -260,7 +261,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): self.juju_status.return_value.applications.__getitem__.return_value = \ FAKE_STATUS_MONGO upgrade_config = upgrade_utils.app_config('mongodb') - await upgrade_utils.serial_series_upgrade( + await upgrade_utils.async_serial_series_upgrade( 'mongodb', from_series='trusty', to_series='xenial', @@ -306,7 +307,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') @mock.patch.object(upgrade_utils, 'maybe_pause_things') @mock.patch.object(upgrade_utils, 'series_upgrade_machine') - async def test_parallel_series_upgrade( + async def test_async_parallel_series_upgrade( self, mock_series_upgrade_machine, mock_maybe_pause_things, @@ -314,7 +315,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): mock_async_prepare_series_upgrade, mock_post_application_upgrade_functions, ): - await upgrade_utils.parallel_series_upgrade( + await upgrade_utils.async_parallel_series_upgrade( 'app', from_series='trusty', to_series='xenial', @@ -361,7 +362,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') @mock.patch.object(upgrade_utils, 'maybe_pause_things') @mock.patch.object(upgrade_utils, 'series_upgrade_machine') - async def test_serial_series_upgrade( + async def test_async_serial_series_upgrade( self, mock_series_upgrade_machine, mock_maybe_pause_things, @@ -369,7 +370,7 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): mock_async_prepare_series_upgrade, mock_post_application_upgrade_functions, ): - await upgrade_utils.serial_series_upgrade( + await upgrade_utils.async_serial_series_upgrade( 'app', from_series='trusty', to_series='xenial', @@ -472,53 +473,86 @@ class TestParallelSeriesUpgrade(ut_utils.AioTestCase): mock_remove_confdef_file.assert_called_once_with('1') mock_add_confdef_file.assert_called_once_with('1') + @mock.patch.object(zaza.model, "async_run_action") + @mock.patch.object(zaza.model, "async_get_application") @mock.patch("asyncio.gather") - async def test_maybe_pause_things_primary(self, mock_gather): + async def test_maybe_pause_things_primary( + self, mock_gather, mock_async_get_application, mock_async_run_action + ): + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + async def _gather(*args): for f in args: await f + mock_app = mock.AsyncMock() + mock_app.get_actions.return_value = ["pause", "resume"] + mock_async_get_application.return_value = mock_app + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], pause_non_leader_subordinate=False, pause_non_leader_primary=True) - self.async_run_action.assert_has_calls([ + mock_async_run_action.assert_has_calls([ mock.call('app/1', "pause", action_params={}), mock.call('app/2', "pause", action_params={}), ]) + @mock.patch.object(zaza.model, "async_run_action") + @mock.patch.object(zaza.model, "async_get_application") @mock.patch("asyncio.gather") - async def test_maybe_pause_things_subordinates(self, mock_gather): + async def test_maybe_pause_things_subordinates( + self, mock_gather, mock_async_get_application, mock_async_run_action + ): + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + async def _gather(*args): for f in args: await f + mock_app = mock.AsyncMock() + mock_app.get_actions.return_value = ["pause", "resume"] + mock_async_get_application.return_value = mock_app + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], pause_non_leader_subordinate=True, pause_non_leader_primary=False) - self.async_run_action.assert_has_calls([ + mock_async_run_action.assert_has_calls([ mock.call('app-hacluster/1', "pause", action_params={}), mock.call('app-hacluster/2', "pause", action_params={}), ]) + @mock.patch.object(zaza.model, "async_run_action") + @mock.patch.object(zaza.model, "async_get_application") @mock.patch("asyncio.gather") - async def test_maybe_pause_things_all(self, mock_gather): + async def test_maybe_pause_things_all( + self, mock_gather, mock_async_get_application, mock_async_run_action + ): + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + async def _gather(*args): for f in args: await f + mock_app = mock.AsyncMock() + mock_app.get_actions.return_value = ["pause", "resume"] + mock_async_get_application.return_value = mock_app + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], pause_non_leader_subordinate=True, pause_non_leader_primary=True) - self.async_run_action.assert_has_calls([ + mock_async_run_action.assert_has_calls([ mock.call('app-hacluster/1', "pause", action_params={}), mock.call('app/1', "pause", action_params={}), mock.call('app-hacluster/2', "pause", action_params={}), diff --git a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py index cf0ab8e..e6d384f 100644 --- a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import mock +import sys +import unittest import unit_tests.utils as ut_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.series_upgrade as series_upgrade_utils @@ -88,9 +91,19 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): self.set_origin.assert_called_once_with(_application, _origin) self.reboot.assert_called_once_with(_unit) + def _mock_app(self): + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + mock_get_action = mock.AsyncMock() + mock_get_action.get_actions.return_value = ["pause", "resume"] + mock_app = asyncio.Future() + mock_app.set_result(mock_get_action) + return mock_app + def test_series_upgrade_application_pause_peers_and_subordinates(self): - self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils.model, "async_run_action") self.patch_object(series_upgrade_utils, "series_upgrade") + self.patch_object(series_upgrade_utils.model, "async_get_application") _application = "app" _from_series = "xenial" _to_series = "bionic" @@ -108,6 +121,8 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): mock.call("{}/2".format(_application), "pause", action_params={}), ] _series_upgrade_calls = [] + self.async_get_application.return_value = self._mock_app() + self.async_run_action.return_value = self._mock_app() for machine_num in ("0", "1", "2"): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), @@ -125,12 +140,13 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): pause_non_leader_subordinate=True, completed_machines=_completed_machines, workaround_script=_workaround_script, files=_files), - self.run_action.assert_has_calls(_run_action_calls) + self.async_run_action.assert_has_calls(_run_action_calls) self.series_upgrade.assert_has_calls(_series_upgrade_calls) def test_series_upgrade_application_pause_subordinates(self): - self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils.model, "async_run_action") self.patch_object(series_upgrade_utils, "series_upgrade") + self.patch_object(series_upgrade_utils.model, "async_get_application") _application = "app" _from_series = "xenial" _to_series = "bionic" @@ -146,7 +162,8 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): "pause", action_params={}), ] _series_upgrade_calls = [] - + self.async_get_application.return_value = self._mock_app() + self.async_run_action.return_value = self._mock_app() for machine_num in ("0", "1", "2"): _series_upgrade_calls.append( mock.call("{}/{}".format(_application, machine_num), @@ -164,7 +181,7 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): pause_non_leader_subordinate=True, completed_machines=_completed_machines, workaround_script=_workaround_script, files=_files), - self.run_action.assert_has_calls(_run_action_calls) + self.async_run_action.assert_has_calls(_run_action_calls) self.series_upgrade.assert_has_calls(_series_upgrade_calls) def test_series_upgrade_application_no_pause(self): diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 4dd2432..98477dc 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -22,6 +22,7 @@ import os import sys import unittest import juju +import zaza from zaza import model from zaza.openstack.utilities import ( @@ -66,6 +67,7 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): cls.from_series = None cls.to_series = None cls.workaround_script = None + cls.vault_unsealer = None cls.files = [] def test_200_run_series_upgrade(self): @@ -77,6 +79,7 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): target_series=self.to_series) from_series = self.from_series to_series = self.to_series + vault_unsealer = self.vault_unsealer completed_machines = [] workaround_script = None files = [] @@ -102,9 +105,14 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): # unstable if all the applications are done at the same time. sem = asyncio.Semaphore(4) for charm_name in apps: + if applications[charm_name]["series"] == to_series: + logging.warn("{} already has series {}, skipping".format( + charm_name, to_series)) + continue charm = applications[charm_name]['charm'] name = upgrade_utils.extract_charm_name_from_url(charm) - upgrade_config = parallel_series_upgrade.app_config(name) + upgrade_config = parallel_series_upgrade.app_config( + name, vault_unsealer) upgrade_functions.append( wrap_coroutine_with_sem( sem, @@ -116,8 +124,7 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): completed_machines=completed_machines, workaround_script=workaround_script, files=files))) - asyncio.get_event_loop().run_until_complete( - asyncio.gather(*upgrade_functions)) + zaza.run(asyncio.gather(*upgrade_functions)) model.block_until_all_units_idle() logging.info("Finished {}".format(group_name)) logging.info("Done!") @@ -191,12 +198,23 @@ class BionicFocalSeriesUpgrade(OpenStackParallelSeriesUpgrade): @classmethod def setUpClass(cls): - """Run setup for Xenial to Bionic Series Upgrades.""" + """Run setup for Bionic to Focal Series Upgrades.""" super(BionicFocalSeriesUpgrade, cls).setUpClass() cls.from_series = "bionic" cls.to_series = "focal" +class FocalJammySeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Bionic to FocalSeries Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Focal to Jammy Series Upgrades.""" + super(BionicFocalSeriesUpgrade, cls).setUpClass() + cls.from_series = "focal" + cls.to_series = "jammy" + + class UbuntuLiteParallelSeriesUpgrade(unittest.TestCase): """ubuntu Lite Parallel Series Upgrade.""" diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 277dcd4..fa33771 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -23,7 +23,7 @@ import telnetlib import tempfile import yaml -from zaza import model +from zaza import model, sync_wrapper from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE from zaza.utilities import juju as juju_utils @@ -234,24 +234,6 @@ def get_yaml_config(config_file): return yaml.safe_load(open(config_file, 'r').read()) -def set_origin(application, origin='openstack-origin', pocket='distro'): - """Set the configuration option for origin source. - - :param application: Name of application to upgrade series - :type application: str - :param origin: The configuration setting variable name for changing origin - source. (openstack-origin or source) - :type origin: str - :param pocket: Origin source cloud pocket. - i.e. 'distro' or 'cloud:xenial-newton' - :type pocket: str - :returns: None - :rtype: None - """ - logging.info("Set origin on {} to {}".format(application, origin)) - model.set_application_config(application, {origin: pocket}) - - async def async_set_origin(application, origin='openstack-origin', pocket='distro'): """Set the configuration option for origin source. @@ -259,7 +241,8 @@ async def async_set_origin(application, origin='openstack-origin', :param application: Name of application to upgrade series :type application: str :param origin: The configuration setting variable name for changing origin - source. (openstack-origin or source) + source. (openstack-origin or source). Use "auto" to + automatically detect origin variable name. :type origin: str :param pocket: Origin source cloud pocket. i.e. 'distro' or 'cloud:xenial-newton' @@ -267,9 +250,21 @@ async def async_set_origin(application, origin='openstack-origin', :returns: None :rtype: None """ - logging.info("Set origin on {} to {}".format(application, origin)) + if origin == "auto": + config = await model.async_get_application_config(application) + for origin in ("openstack-origin", "source"): + if config.get(origin): + break + else: + logging.warn("Failed to set origin for {} to {}, no origin config " + "found".format(application, origin)) + return + + logging.info("Set origin on {} to {}".format(application, pocket)) await model.async_set_application_config(application, {origin: pocket}) +set_origin = sync_wrapper(async_set_origin) + def run_via_ssh(unit_name, cmd): """Run command on unit via ssh. diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 610496d..fdd89d5 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -23,16 +23,14 @@ import copy import logging import subprocess -from zaza import model +from zaza import model, sync_wrapper from zaza.charm_lifecycle import utils as cl_utils import zaza.openstack.utilities.generic as os_utils import zaza.openstack.utilities.series_upgrade as series_upgrade_utils -from zaza.openstack.utilities.series_upgrade import ( - SUBORDINATE_PAUSE_RESUME_BLACKLIST, -) +from zaza.openstack.utilities.series_upgrade import async_pause_helper -def app_config(charm_name): +def app_config(charm_name, vault_unsealer=None): """Return a dict with the upgrade config for an application. :param charm_name: Name of the charm about to upgrade @@ -43,7 +41,7 @@ def app_config(charm_name): :rtype: Dict """ default = { - 'origin': 'openstack-origin', + 'origin': 'auto', 'pause_non_leader_subordinate': True, 'pause_non_leader_primary': True, 'post_upgrade_functions': [], @@ -95,6 +93,7 @@ def app_config(charm_name): 'pause_non_leader_primary': False, 'pause_non_leader_subordinate': True, 'post_upgrade_functions': [ + vault_unsealer or ('zaza.openstack.charm_tests.vault.setup.' 'async_mojo_or_default_unseal_by_unit')] }, @@ -119,7 +118,7 @@ def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'): """ completed_machines = [] asyncio.get_event_loop().run_until_complete( - parallel_series_upgrade( + async_parallel_series_upgrade( 'ubuntu-lite', pause_non_leader_primary=False, pause_non_leader_subordinate=False, from_series=from_series, to_series=to_series, @@ -127,7 +126,7 @@ def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'): ) -async def parallel_series_upgrade( +async def async_parallel_series_upgrade( application, from_series='xenial', to_series='bionic', @@ -229,6 +228,8 @@ async def parallel_series_upgrade( await run_post_application_upgrade_functions( post_application_upgrade_functions) +parallel_series_upgrade = sync_wrapper(async_parallel_series_upgrade) + async def wait_for_idle_then_prepare_series_upgrade( machine, to_series, model_name=None): @@ -251,7 +252,7 @@ async def wait_for_idle_then_prepare_series_upgrade( await prepare_series_upgrade(machine, to_series=to_series) -async def serial_series_upgrade( +async def async_serial_series_upgrade( application, from_series='xenial', to_series='bionic', @@ -379,6 +380,8 @@ async def serial_series_upgrade( post_application_upgrade_functions) logging.info("Done series upgrade for: {}".format(application)) +serial_series_upgrade = sync_wrapper(async_serial_series_upgrade) + async def series_upgrade_machine( machine, @@ -472,6 +475,9 @@ async def run_post_upgrade_functions(post_upgrade_functions): """ if post_upgrade_functions: for func in post_upgrade_functions: + if callable(func): + func() + return logging.info("Running {}".format(func)) m = cl_utils.get_class(func) await m() @@ -512,26 +518,14 @@ async def maybe_pause_things( if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - unit_pauses.append( - _pause_helper("subordinate", subordinate)) + unit_pauses.append(async_pause_helper( + "subordinate", subordinate)) if pause_non_leader_primary: - unit_pauses.append(_pause_helper("leader", unit)) + unit_pauses.append(async_pause_helper("leader", unit)) if unit_pauses: await asyncio.gather(*unit_pauses) -async def _pause_helper(_type, unit): - """Pause helper to ensure that the log happens nearer to the action.""" - logging.info("Pausing ({}) {}".format(_type, unit)) - await model.async_run_action(unit, "pause", action_params={}) - logging.info("Finished Pausing ({}) {}".format(_type, unit)) - - def get_leader_and_non_leaders(status): """Get the leader and non-leader Juju units. diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 42683f6..09ae9a0 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -22,16 +22,11 @@ import logging import os import time -from zaza import model +from zaza import model, sync_wrapper from zaza.charm_lifecycle import utils as cl_utils import zaza.openstack.utilities.generic as os_utils -SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ - "cinder-ceph", -] - - def app_config(charm_name, is_async=True): """Return a dict with the upgrade config for an application. @@ -166,17 +161,9 @@ def series_upgrade_non_leaders_first( if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - logging.info("Pausing {}".format(subordinate)) - model.run_action( - subordinate, "pause", action_params={}) + pause_helper("subordinate", subordinate) if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - model.run_action(unit, "pause", action_params={}) + pause_helper("leader", unit) # Series upgrade the non-leaders first for unit in non_leaders: @@ -270,14 +257,7 @@ async def async_series_upgrade_non_leaders_first( if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - logging.info("Pausing {}".format(subordinate)) - await model.async_run_action( - subordinate, "pause", action_params={}) + pause_helper("subordinate", subordinate) if pause_non_leader_primary: logging.info("Pausing {}".format(unit)) await model.async_run_action(unit, "pause", action_params={}) @@ -375,17 +355,9 @@ def series_upgrade_application(application, pause_non_leader_primary=True, if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - logging.info("Pausing {}".format(subordinate)) - model.run_action( - subordinate, "pause", action_params={}) + pause_helper("subordinate", subordinate) if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - model.run_action(unit, "pause", action_params={}) + pause_helper("leader", unit) machine = status["units"][leader]["machine"] # Series upgrade the leader @@ -491,23 +463,14 @@ async def async_series_upgrade_application( leader = unit else: non_leaders.append(unit) - # Pause the non-leaders for unit in non_leaders: if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - logging.info("Pausing {}".format(subordinate)) - await model.async_run_action( - subordinate, "pause", action_params={}) + pause_helper("subordinate", subordinate) if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - await model.async_run_action(unit, "pause", action_params={}) + pause_helper("leader", unit) machine = status["units"][leader]["machine"] # Series upgrade the leader @@ -970,3 +933,21 @@ async def async_do_release_upgrade(unit_name): 'DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive', raise_exceptions=True) + + +async def async_pause_helper(_type, unit): + """Pause helper to ensure that the log happens nearer to the action. + + :param _type: Type of unit (subordinate/leader), used for logging only + :param unit: Unit to pause + """ + logging.info("Pausing ({}) {}".format(_type, unit)) + app = await model.async_get_application(unit.split('/')[0]) + if "pause" not in await app.get_actions(): + logging.info("Skipping pausing {} - no pause action" + .format(unit)) + return + await model.async_run_action(unit, "pause", action_params={}) + logging.info("Finished Pausing ({}) {}".format(_type, unit)) + +pause_helper = sync_wrapper(async_pause_helper)