From 71a3c68d516bfe52339d5db89f1cd28e6a184f6a Mon Sep 17 00:00:00 2001 From: Mark Gurevich Date: Wed, 28 Feb 2018 16:26:15 -0500 Subject: [PATCH 1/2] Better Python login action ConnectionError exception handling --- .../lib/python/agent/common/exceptions.py | 6 +- .../lib/python/agent/common/rest.py | 56 +++++++++++++++++-- .../agent/hwctl/executor/openbmc_power.py | 3 +- .../lib/python/agent/hwctl/openbmc_client.py | 21 +++++-- 4 files changed, 76 insertions(+), 10 deletions(-) diff --git a/xCAT-openbmc-py/lib/python/agent/common/exceptions.py b/xCAT-openbmc-py/lib/python/agent/common/exceptions.py index 15af285c4..6f5dad33c 100644 --- a/xCAT-openbmc-py/lib/python/agent/common/exceptions.py +++ b/xCAT-openbmc-py/lib/python/agent/common/exceptions.py @@ -1,7 +1,11 @@ #!/usr/bin/env python class SelfServerException(Exception) : - pass + def __init__(self, message, code=0, detail_msg= "", host_and_port="") : + super(Exception, self).__init__(message) + self.code = code + self.host_and_port = host_and_port + self.detail_msg = detail_msg class SelfClientException(Exception) : def __init__(self, message, code) : diff --git a/xCAT-openbmc-py/lib/python/agent/common/rest.py b/xCAT-openbmc-py/lib/python/agent/common/rest.py index 250fea527..bc52f325b 100644 --- a/xCAT-openbmc-py/lib/python/agent/common/rest.py +++ b/xCAT-openbmc-py/lib/python/agent/common/rest.py @@ -26,17 +26,65 @@ class RestSession(object): headers=headers, verify=False, timeout=timeout) - except requests.exceptions.ConnectionError: - raise xcat_exception.SelfServerException('Error: Failed to connect to server.') + except requests.exceptions.ConnectionError as e: + # Extract real reason for the exception and host/port from ConnectionError message + # Sometimes e.message is a list, sometimes is a string. Look for different patterns + # to extract the data needed. + causing_error = "n/a" + host_and_port = "n/a" + if "]" in e.message[0]: + causing_error_part1 = e.message[0].split("]")[1] + causing_error = causing_error_part1.split("'")[0] + causing_error = causing_error.strip() + host_and_port = self.extract_server_and_port(e.message[0], "STRING") - except requests.exceptions.Timeout: - raise xcat_exception.SelfServerException('Error: Timeout to connect to server') + if "Connection aborted." in e.message[0]: + causing_error = "Connection reset by peer" + host_and_port = self.extract_server_and_port(url, "URL") + + if "connect timeout=" in e.message[0]: + causing_error = "timeout" + host_and_port = self.extract_server_and_port(e.message[0], "STRING") + + message = 'Error: Failed to connect to server.' + # message = '\n\n--> {0} \n\n'.format(e.message[0]) + raise xcat_exception.SelfServerException(message, "500", '({0})'.format(causing_error), host_and_port) + + except requests.exceptions.Timeout as e: + causing_error = "timeout" + host_and_port = self.extract_server_and_port(e.message[0], "STRING") + + message = 'Error: Timeout to connect to server' + raise xcat_exception.SelfServerException(message, "500", '({0})'.format(causing_error), host_and_port) if not self.cookies: self.cookies = requests.utils.dict_from_cookiejar(self.session.cookies) return response + def extract_server_and_port(self, message_string, format="STRING"): + # Extract hostip and port number from ConnectionError message + # If format="STRING" look for host='IP' and port=xxxx pattern + # If format="URL" look for https://IP/login pattern + if format == "STRING": + start = "host='" + end = "'," + host_ip = message_string[message_string.find(start)+len(start):message_string.find(end)] + start = "port=" + end = "):" + port = message_string[message_string.find(start)+len(start):message_string.find(end)] + host_and_port = host_ip + ":" + port + elif format == "URL": + start = "https://" + end = "/login" + host_ip = message_string[message_string.find(start)+len(start):message_string.find(end)] + host_and_port = host_ip + else: + host_and_port = "n/a" + + return host_and_port + + def request_download(self, method, url, headers, file_path, using_curl=True): if using_curl: diff --git a/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py b/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py index c400ee13d..4bab39788 100644 --- a/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py +++ b/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py @@ -83,7 +83,8 @@ class OpenBMCPowerTask(ParallelNodesCommand): except SelfServerException, SelfClientException: # There is no response when BMC is not ready - result = '%s: %s' % (node, openbmc.RPOWER_STATES[bmc_not_ready]) + # Do not print bmc_state, instead, error messages from login failure will be displayed + return bmc_state self.callback.info(result) return bmc_state diff --git a/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py b/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py index 29169ff8a..d73e17b05 100644 --- a/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py +++ b/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py @@ -249,6 +249,15 @@ class OpenBMCRest(object): self.messager.info(localtime + ' ' + log) logger.debug(log) + def _print_record (self, msg, cmd, type="I"): + + log = self.name + ': ' + msg + logger.debug(log) + if type == "E": + self.messager.error(log) + else: + self.messager.info(log) + def _log_request (self, method, url, headers, data=None, files=None, file_path=None, cmd=''): header_str = ' '.join([ "%s: %s" % (k, v) for k,v in headers.items() ]) @@ -276,7 +285,7 @@ class OpenBMCRest(object): if code != requests.codes.ok: description = ''.join(data['data']['description']) error = 'Error: [%d] %s' % (code, description) - self._print_record_log(error, cmd) + self._print_record(error, cmd, "E") raise SelfClientException(error, code) self._print_record_log(data['message'], cmd) @@ -298,9 +307,13 @@ class OpenBMCRest(object): response = self.session.request(method, url, httpheaders, data=data) return self.handle_response(response, cmd=cmd) except SelfServerException as e: - e.message = 'Error: BMC did not respond. ' \ - 'Validate BMC configuration and retry the command.' - self._print_record_log(e.message, cmd) + message = 'Error: BMC did not respond. ' \ + 'Validate BMC configuration and retry the command.' + if cmd == 'login': + login_message = "Error: [{0}] Login to BMC failed: {1} Can't connect to {2} {3}.".format(e.code, e.code, e.host_and_port, e.detail_msg) + self._print_record(login_message, cmd, "I") + + self._print_record(message, cmd, "E") raise except ValueError: error = 'Error: Received wrong format response: %s' % response From 02523e89715f80867585bf7ede2b10ac086dc2f9 Mon Sep 17 00:00:00 2001 From: Mark Gurevich Date: Thu, 1 Mar 2018 13:48:37 -0500 Subject: [PATCH 2/2] Changes for review comments --- .../lib/python/agent/common/exceptions.py | 3 +-- .../lib/python/agent/common/rest.py | 8 +++---- .../agent/hwctl/executor/openbmc_power.py | 10 +++++++-- .../lib/python/agent/hwctl/openbmc_client.py | 21 ++++--------------- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/xCAT-openbmc-py/lib/python/agent/common/exceptions.py b/xCAT-openbmc-py/lib/python/agent/common/exceptions.py index 6f5dad33c..88e80e59a 100644 --- a/xCAT-openbmc-py/lib/python/agent/common/exceptions.py +++ b/xCAT-openbmc-py/lib/python/agent/common/exceptions.py @@ -1,9 +1,8 @@ #!/usr/bin/env python class SelfServerException(Exception) : - def __init__(self, message, code=0, detail_msg= "", host_and_port="") : + def __init__(self, message, detail_msg= "", host_and_port="") : super(Exception, self).__init__(message) - self.code = code self.host_and_port = host_and_port self.detail_msg = detail_msg diff --git a/xCAT-openbmc-py/lib/python/agent/common/rest.py b/xCAT-openbmc-py/lib/python/agent/common/rest.py index bc52f325b..88c836998 100644 --- a/xCAT-openbmc-py/lib/python/agent/common/rest.py +++ b/xCAT-openbmc-py/lib/python/agent/common/rest.py @@ -46,16 +46,16 @@ class RestSession(object): causing_error = "timeout" host_and_port = self.extract_server_and_port(e.message[0], "STRING") - message = 'Error: Failed to connect to server.' + message = 'Failed to connect to server.' # message = '\n\n--> {0} \n\n'.format(e.message[0]) - raise xcat_exception.SelfServerException(message, "500", '({0})'.format(causing_error), host_and_port) + raise xcat_exception.SelfServerException(message, '({0})'.format(causing_error), host_and_port) except requests.exceptions.Timeout as e: causing_error = "timeout" host_and_port = self.extract_server_and_port(e.message[0], "STRING") - message = 'Error: Timeout to connect to server' - raise xcat_exception.SelfServerException(message, "500", '({0})'.format(causing_error), host_and_port) + message = 'Timeout to connect to server' + raise xcat_exception.SelfServerException(message, '({0})'.format(causing_error), host_and_port) if not self.cookies: self.cookies = requests.utils.dict_from_cookiejar(self.session.cookies) diff --git a/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py b/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py index 4bab39788..567d6b825 100644 --- a/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py +++ b/xCAT-openbmc-py/lib/python/agent/hwctl/executor/openbmc_power.py @@ -73,6 +73,13 @@ class OpenBMCPowerTask(ParallelNodesCommand): bmc_not_ready = bmc_state = 'NotReady' try: obmc.login() + except SelfServerException as e: + # Special exception handling for login failure + login_message = "Login to BMC failed: Can't connect to {0} {1}.".format(e.host_and_port, e.detail_msg) + self.callback.error(login_message, node) + return bmc_state + + try: state = obmc.get_bmc_state() bmc_state = state.get('bmc') @@ -83,8 +90,7 @@ class OpenBMCPowerTask(ParallelNodesCommand): except SelfServerException, SelfClientException: # There is no response when BMC is not ready - # Do not print bmc_state, instead, error messages from login failure will be displayed - return bmc_state + result = '%s: %s' % (node, openbmc.RPOWER_STATES[bmc_not_ready]) self.callback.info(result) return bmc_state diff --git a/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py b/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py index d73e17b05..29169ff8a 100644 --- a/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py +++ b/xCAT-openbmc-py/lib/python/agent/hwctl/openbmc_client.py @@ -249,15 +249,6 @@ class OpenBMCRest(object): self.messager.info(localtime + ' ' + log) logger.debug(log) - def _print_record (self, msg, cmd, type="I"): - - log = self.name + ': ' + msg - logger.debug(log) - if type == "E": - self.messager.error(log) - else: - self.messager.info(log) - def _log_request (self, method, url, headers, data=None, files=None, file_path=None, cmd=''): header_str = ' '.join([ "%s: %s" % (k, v) for k,v in headers.items() ]) @@ -285,7 +276,7 @@ class OpenBMCRest(object): if code != requests.codes.ok: description = ''.join(data['data']['description']) error = 'Error: [%d] %s' % (code, description) - self._print_record(error, cmd, "E") + self._print_record_log(error, cmd) raise SelfClientException(error, code) self._print_record_log(data['message'], cmd) @@ -307,13 +298,9 @@ class OpenBMCRest(object): response = self.session.request(method, url, httpheaders, data=data) return self.handle_response(response, cmd=cmd) except SelfServerException as e: - message = 'Error: BMC did not respond. ' \ - 'Validate BMC configuration and retry the command.' - if cmd == 'login': - login_message = "Error: [{0}] Login to BMC failed: {1} Can't connect to {2} {3}.".format(e.code, e.code, e.host_and_port, e.detail_msg) - self._print_record(login_message, cmd, "I") - - self._print_record(message, cmd, "E") + e.message = 'Error: BMC did not respond. ' \ + 'Validate BMC configuration and retry the command.' + self._print_record_log(e.message, cmd) raise except ValueError: error = 'Error: Received wrong format response: %s' % response