diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index a8a4412b..ce792fcb 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -73,7 +73,7 @@ import sys import yaml pluginmap = {} -dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos') +dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos', b'enos', u'enos') PluginCollection = plugin.PluginCollection diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index dfb50b9f..7b94154a 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -74,6 +74,7 @@ import confluent.discovery.handlers.tsm as tsm import confluent.discovery.handlers.pxe as pxeh import confluent.discovery.handlers.smm as smm import confluent.discovery.handlers.xcc as xcc +import confluent.discovery.handlers.megarac as megarac import confluent.exceptions as exc import confluent.log as log import confluent.messages as msg @@ -113,6 +114,7 @@ nodehandlers = { 'service:lenovo-smm': smm, 'service:lenovo-smm2': smm, 'lenovo-xcc': xcc, + 'megarac-bmc': megarac, 'service:management-hardware.IBM:integrated-management-module2': imm, 'pxe-client': pxeh, 'onie-switch': None, @@ -132,6 +134,7 @@ servicenames = { 'service:lenovo-smm2': 'lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', + 'megarac-bmc': 'megarac-bmc', #'openbmc': 'openbmc', 'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2', 'service:io-device.Lenovo:management-module': 'lenovo-switch', @@ -147,6 +150,7 @@ servicebyname = { 'lenovo-smm2': 'service:lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', + 'megarac-bmc': 'megarac-bmc', 'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2', 'lenovo-switch': 'service:io-device.Lenovo:management-module', 'thinkagile-storage': 'service:thinkagile-storagebmc', @@ -453,7 +457,7 @@ def iterate_addrs(addrs, countonly=False): yield 1 return yield addrs - + def _parameterize_path(pathcomponents): listrequested = False childcoll = True @@ -542,7 +546,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents): if len(pathcomponents) > 2: raise Exception('TODO') currsubs = get_subscriptions() - return [msg.ChildCollection(x) for x in currsubs] + return [msg.ChildCollection(x) for x in currsubs] elif operation == 'retrieve': return handle_read_api_request(pathcomponents) elif (operation in ('update', 'create') and @@ -1703,3 +1707,4 @@ if __name__ == '__main__': start_detection() while True: eventlet.sleep(30) + diff --git a/confluent_server/confluent/discovery/handlers/megarac.py b/confluent_server/confluent/discovery/handlers/megarac.py new file mode 100644 index 00000000..d7d8786a --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/megarac.py @@ -0,0 +1,51 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.redfishbmc as redfishbmc +import eventlet.support.greendns + + +getaddrinfo = eventlet.support.greendns.getaddrinfo + + +class NodeHandler(redfishbmc.NodeHandler): + + def get_firmware_default_account_info(self): + return ('admin', 'admin') + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]]} + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) + diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py new file mode 100644 index 00000000..eed401de --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/redfishbmc.py @@ -0,0 +1,269 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.generic as generic +import confluent.exceptions as exc +import confluent.netutil as netutil +import confluent.util as util +import eventlet +import eventlet.support.greendns +import json +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode + +getaddrinfo = eventlet.support.greendns.getaddrinfo + +webclient = eventlet.import_patched('pyghmi.util.webclient') + +def get_host_interface_urls(wc, mginfo): + returls = [] + hifurl = mginfo.get('HostInterfaces', {}).get('@odata.id', None) + if not hifurl: + return None + hifinfo = wc.grab_json_response(hifurl) + hifurls = hifinfo.get('Members', []) + for hifurl in hifurls: + hifurl = hifurl['@odata.id'] + hifinfo = wc.grab_json_response(hifurl) + acturl = hifinfo.get('ManagerEthernetInterface', {}).get('@odata.id', None) + if acturl: + returls.append(acturl) + return returls + + +class NodeHandler(generic.NodeHandler): + devname = 'BMC' + + def __init__(self, info, configmanager): + self.trieddefault = None + self.targuser = None + self.curruser = None + self.currpass = None + self.targpass = None + self.nodename = None + self.csrftok = None + self.channel = None + self.atdefault = True + super(NodeHandler, self).__init__(info, configmanager) + + def get_firmware_default_account_info(self): + raise Exception('This must be subclassed') + + def scan(self): + c = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) + i = c.grab_json_response('/redfish/v1/') + uuid = i.get('UUID', None) + if uuid: + self.info['uuid'] = uuid.lower() + + def validate_cert(self, certificate): + # broadly speaking, merely checks consistency moment to moment, + # but if https_cert gets stricter, this check means something + fprint = util.get_fingerprint(self.https_cert) + return util.cert_matches(fprint, certificate) + + def _get_wc(self): + defuser, defpass = self.get_firmware_default_account_info() + wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) + wc.set_basic_credentials(defuser, defpass) + wc.set_header('Content-Type', 'application/json') + authmode = 0 + if not self.trieddefault: + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status == 403: + self.trieddefault = True + chgurl = None + rsp = json.loads(rsp) + currerr = rsp.get('error', {}) + ecode = currerr.get('code', None) + if ecode.endswith('PasswordChangeRequired'): + for einfo in currerr.get('@Message.ExtendedInfo', []): + if einfo.get('MessageId', None).endswith('PasswordChangeRequired'): + for msgarg in einfo.get('MessageArgs'): + chgurl = msgarg + break + if chgurl: + if self.targpass == defpass: + raise Exception("Must specify a non-default password to onboard this BMC") + wc.set_header('If-Match', '*') + cpr = wc.grab_json_response_with_status(chgurl, {'Password': self.targpass}, method='PATCH') + if cpr[1] >= 200 and cpr[1] < 300: + self.curruser = defuser + self.currpass = self.targpass + wc.set_basic_credentials(self.curruser, self.currpass) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + tries = 10 + while status >= 300 and tries: + eventlet.sleep(1) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + return wc + + if status > 400: + self.trieddefault = True + if status == 401: + wc.set_basic_credentials(self.DEFAULT_USER, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status == 200: # Default user still, but targpass + self.currpass = self.targpass + self.curruser = defuser + return wc + elif self.targuser != defuser: + wc.set_basic_credentials(self.targuser, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + raise Exception("Target BMC does not recognize firmware default credentials nor the confluent stored credential") + else: + self.curruser = defuser + self.currpass = defpass + return wc + if self.curruser: + wc.set_basic_credentials(self.curruser, self.currpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + return None + return wc + wc.set_basic_credentials(self.targuser, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + return None + self.curruser = self.targuser + self.currpass = self.targpass + return wc + + def config(self, nodename): + self.nodename = nodename + creds = self.configmanager.get_node_attributes( + nodename, ['secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword', + 'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'], + True) + cd = creds.get(nodename, {}) + defuser, defpass = self.get_firmware_default_account_info() + user, passwd, _ = self.get_node_credentials( + nodename, creds, defuser, defpass) + user = util.stringify(user) + passwd = util.stringify(passwd) + self.targuser = user + self.targpass = passwd + wc = self._get_wc() + srvroot, status = wc.grab_json_response_with_status('/redfish/v1/') + curruserinfo = {} + authupdate = {} + wc.set_header('Content-Type', 'application/json') + if user != self.curruser: + authupdate['UserName'] = user + if passwd != self.currpass: + authupdate['Password'] = passwd + if authupdate: + targaccturl = None + asrv = srvroot.get('AccountService', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(asrv) + accts = rsp.get('Accounts', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(accts) + accts = rsp.get('Members', []) + for accturl in accts: + accturl = accturl.get('@odata.id', '') + if accturl: + rsp, status = wc.grab_json_response_with_status(accturl) + if rsp.get('UserName', None) == self.curruser: + targaccturl = accturl + break + else: + raise Exception("Unable to identify Account URL to modify on this BMC") + rsp, status = wc.grab_json_response_with_status(targaccturl, authupdate, method='PATCH') + if status >= 300: + raise Exception("Failed attempting to update credentials on BMC") + wc.set_basic_credentials(user, passwd) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + tries = 10 + while tries and status >= 300: + tries -= 1 + eventlet.sleep(1.0) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if ('hardwaremanagement.manager' in cd and + cd['hardwaremanagement.manager']['value'] and + not cd['hardwaremanagement.manager']['value'].startswith( + 'fe80::')): + newip = cd['hardwaremanagement.manager']['value'] + newip = newip.split('/', 1)[0] + newipinfo = getaddrinfo(newip, 0)[0] + newip = newipinfo[-1][0] + if ':' in newip: + raise exc.NotImplementedException('IPv6 remote config TODO') + mgrs = srvroot['Managers']['@odata.id'] + rsp = wc.grab_json_response(mgrs) + if len(rsp['Members']) != 1: + raise Exception("Can not handle multiple Managers") + mgrurl = rsp['Members'][0]['@odata.id'] + mginfo = wc.grab_json_response(mgrurl) + hifurls = get_host_interface_urls(wc, mginfo) + mgtnicinfo = mginfo['EthernetInterfaces']['@odata.id'] + mgtnicinfo = wc.grab_json_response(mgtnicinfo) + mgtnics = [x['@odata.id'] for x in mgtnicinfo.get('Members', [])] + actualnics = [] + for candnic in mgtnics: + if candnic in hifurls: + continue + actualnics.append(candnic) + if len(actualnics) != 1: + raise Exception("Multi-interface BMCs are not supported currently") + currnet = wc.grab_json_response(actualnics[0]) + netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip) + newconfig = { + "Address": newip, + "SubnetMask": netutil.cidr_to_mask(netconfig['prefix']), + } + newgw = netconfig['ipv4_gateway'] + if newgw: + newconfig['Gateway'] = newgw + else: + newconfig['Gateway'] = newip # required property, set to self just to have a value + for net in currnet.get("IPv4Addresses", []): + if net["Address"] == newip and net["SubnetMask"] == newconfig['SubnetMask'] and (not newgw or newconfig['Gateway'] == newgw): + break + else: + wc.set_header('If-Match', '*') + rsp, status = wc.grab_json_response_with_status(actualnics[0], {'IPv4StaticAddresses': [newconfig]}, method='PATCH') + elif self.ipaddr.startswith('fe80::'): + self.configmanager.set_node_attributes( + {nodename: {'hardwaremanagement.manager': self.ipaddr}}) + else: + raise exc.TargetEndpointUnreachable( + 'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)') + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]] } + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index ff7ca042..49fe1e87 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -605,7 +605,10 @@ class NodeHandler(immhandler.NodeHandler): statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway'] elif not netutil.address_is_local(newip): raise exc.InvalidArgumentException('Will not remotely configure a device with no gateway') - wc.grab_json_response('/api/dataset', statargs) + netset, status = wc.grab_json_response_with_status('/api/dataset', statargs) + print(repr(netset)) + print(repr(status)) + elif self.ipaddr.startswith('fe80::'): self.configmanager.set_node_attributes( {nodename: {'hardwaremanagement.manager': self.ipaddr}}) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 4a39654f..133d8abd 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -315,9 +315,9 @@ def proxydhcp(handler, nodeguess): optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4 except ValueError: continue - hwlen = rq[2] - opts, disco = opts_to_dict(rq, optidx, 3) - disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]]) + hwlen = rqv[2] + opts, disco = opts_to_dict(rqv, optidx, 3) + disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rqv[28:28+hwlen]]) node = None if disco.get('hwaddr', None) in macmap: node = macmap[disco['hwaddr']] diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index b5847965..ec8275f1 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -60,6 +60,7 @@ def active_scan(handler, protocol=None): known_peers = set([]) for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::service:affluent']): for addr in scanned['addresses']: + addr = addr[0:1] + addr[2:] if addr in known_peers: break hwaddr = neighutil.get_hwaddr(addr[0]) @@ -79,8 +80,14 @@ def scan(services, target=None): def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler): - if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']: - peerbymacaddress[mac]['addresses'].append(peer) + if mac in peerbymacaddress: + normpeer = peer[0:1] + peer[2:] + for currpeer in peerbymacaddress[mac]['addresses']: + currnormpeer = currpeer[0:1] + peer[2:] + if currnormpeer == normpeer: + break + else: + peerbymacaddress[mac]['addresses'].append(peer) else: peerdata = { 'hwaddr': mac, @@ -108,16 +115,18 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha elif header == 'LOCATION': if '/eth' in value and value.endswith('.xml'): targurl = '/redfish/v1/' + targtype = 'megarac-bmc' continue # MegaRAC redfish elif value.endswith('/DeviceDescription.json'): targurl = '/DeviceDescription.json' + targtype = 'megarac-bmc' else: return if handler and targurl: - eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl) + eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype) -def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl): - retdata = check_fish((targurl, peerdata)) +def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype): + retdata = check_fish((targurl, peerdata, targtype)) if retdata: known_peers.add(peer) newmacs.add(mac) @@ -328,7 +337,7 @@ def _find_service(service, target): host = '[{0}]'.format(host) msg = smsg.format(host, service) if not isinstance(msg, bytes): - msg = msg.encode('utf8') + msg = msg.encode('utf8') net6.sendto(msg, addr[4]) else: net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) @@ -416,11 +425,11 @@ def _find_service(service, target): if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()): continue if '/DeviceDescription.json' in peerdata[nid]['urls']: - pooltargs.append(('/DeviceDescription.json', peerdata[nid])) + pooltargs.append(('/DeviceDescription.json', peerdata[nid], 'lenovo-xcc')) else: for targurl in peerdata[nid]['urls']: if '/eth' in targurl and targurl.endswith('.xml'): - pooltargs.append(('/redfish/v1/', peerdata[nid])) + pooltargs.append(('/redfish/v1/', peerdata[nid], 'megarac-bmc')) # For now, don't interrogate generic redfish bmcs # This is due to a need to deduplicate from some supported SLP # targets (IMM, TSM, others) @@ -435,7 +444,7 @@ def _find_service(service, target): def check_fish(urldata, port=443, verifycallback=None): if not verifycallback: verifycallback = lambda x: True - url, data = urldata + url, data, targtype = urldata try: wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5) peerinfo = wc.grab_json_response(url) @@ -457,7 +466,7 @@ def check_fish(urldata, port=443, verifycallback=None): peerinfo = wc.grab_json_response('/redfish/v1/') if url == '/redfish/v1/': if 'UUID' in peerinfo: - data['services'] = ['service:redfish-bmc'] + data['services'] = [targtype] data['uuid'] = peerinfo['UUID'].lower() return data return None @@ -476,7 +485,12 @@ def _parse_ssdp(peer, rsp, peerdata): if code == b'200': if nid in peerdata: peerdatum = peerdata[nid] - if peer not in peerdatum['addresses']: + normpeer = peer[0:1] + peer[2:] + for currpeer in peerdatum['addresses']: + currnormpeer = currpeer[0:1] + peer[2:] + if currnormpeer == normpeer: + break + else: peerdatum['addresses'].append(peer) else: peerdatum = { @@ -511,5 +525,7 @@ def _parse_ssdp(peer, rsp, peerdata): if __name__ == '__main__': def printit(rsp): - print(repr(rsp)) + pass # print(repr(rsp)) active_scan(printit) + + diff --git a/confluent_server/confluent/plugins/hardwaremanagement/enos.py b/confluent_server/confluent/plugins/hardwaremanagement/enos.py new file mode 100644 index 00000000..f568fae2 --- /dev/null +++ b/confluent_server/confluent/plugins/hardwaremanagement/enos.py @@ -0,0 +1,347 @@ + +# Copyright 2019 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#Noncritical: +# - One or more temperature sensors is in the warning range; +#Critical: +# - One or more temperature sensors is in the failure range; +# - One or more fans are running < 100 RPM; +# - One power supply is off. + +import re +import eventlet +import eventlet.queue as queue +import confluent.exceptions as exc +webclient = eventlet.import_patched("pyghmi.util.webclient") +import confluent.messages as msg +import confluent.util as util +import confluent.plugins.shell.ssh as ssh + + +class SwitchSensor(object): + def __init__(self, name, states=None, units=None, value=None, health=None): + self.name = name + self.value = value + self.states = states + self.health = health + self.units = units + + +def _run_method(method, workers, results, configmanager, nodes, element): + creds = configmanager.get_node_attributes( + nodes, ["switchuser", "switchpass", "secret.hardwaremanagementpassword", + "secret.hardwaremanagementuser"], decrypt=True) + for node in nodes: + workers.add(eventlet.spawn(method, configmanager, creds, + node, results, element)) + + +def enos_login(node, configmanager, creds): + try: + ukey = "switchuser" + upass = "switchpass" + if ukey not in creds and "secret.hardwaremanagementuser" in creds[node]: + ukey = "secret.hardwaremanagementuser" + upass = "secret.hardwaremanagementpassword" + + if ukey not in creds[node]: + raise exc.TargetEndpointBadCredentials("Unable to authenticate - switchuser or secret.hardwaremanagementuser not set") + user = creds[node][ukey]["value"] + if upass not in creds[node]: + passwd = None + else: + passwd = creds[node][upass]["value"] + nssh = ssh.SshConn(node=node, config=configmanager, username=user, password=passwd) + nssh.do_logon() + return nssh + except Exception as e: + raise exc.TargetEndpointBadCredentials(f"Unable to authenticate {e}") + + +def enos_version(ssh): + sshStdout, sshStderr = ssh.exec_command(cmd="show", cmdargs=["version"]) + return sshStdout + + +def update(nodes, element, configmanager, inputdata): + for node in nodes: + yield msg.ConfluentNodeError(node, "Not Implemented") + + +def delete(nodes, element, configmanager, inputdata): + for node in nodes: + yield msg.ConfluentNodeError(node, "Not Implemented") + + +def create(nodes, element, configmanager, inputdata): + for node in nodes: + yield msg.ConfluentNodeError(node, "Not Implemented") + + +def retrieve(nodes, element, configmanager, inputdata): + results = queue.LightQueue() + workers = set([]) + if element == ["power", "state"]: + for node in nodes: + yield msg.PowerState(node=node, state="on") + return + elif element == ["health", "hardware"]: + _run_method(retrieve_health, workers, results, configmanager, nodes, element) + elif element[:3] == ["inventory", "hardware", "all"]: + _run_method(retrieve_inventory, workers, results, configmanager, nodes, element) + elif element[:3] == ["inventory", "firmware", "all"]: + _run_method(retrieve_firmware, workers, results, configmanager, nodes, element) + elif element[:3] == ["sensors", "hardware", "all"]: + _run_method(retrieve_sensors, workers, results, configmanager, nodes, element) + else: + for node in nodes: + yield msg.ConfluentNodeError(node, f"Not Implemented: {element}") + return + currtimeout = 10 + while workers: + try: + datum = results.get(10) + while datum: + if datum: + yield datum + datum = results.get_nowait() + except queue.Empty: + pass + eventlet.sleep(0.001) + for t in list(workers): + if t.dead: + workers.discard(t) + try: + while True: + datum = results.get_nowait() + if datum: + yield datum + except queue.Empty: + pass + + +def retrieve_inventory(configmanager, creds, node, results, element): + if len(element) == 3: + results.put(msg.ChildCollection("all")) + results.put(msg.ChildCollection("system")) + return + + switch = gather_data(configmanager, creds, node) + invinfo = switch["inventory"] + + for fan, data in switch["fans"].items(): + invinfo["inventory"][0]["information"][f"Fan #{fan}"] = data["state"] + + for psu, data in switch["psus"].items(): + invinfo["inventory"][0]["information"][f"PSU #{psu}"] = data["state"] + + results.put(msg.KeyValueData(invinfo, node)) + + +def gather_data(configmanager, creds, node): + nssh = enos_login(node=node, configmanager=configmanager, creds=creds) + switch_lines = enos_version(ssh=nssh) + switch_data = {} + sysinfo = {"Product name": {"regex": ".*RackSwitch (\w+)"}, + "Serial Number": {"regex": "ESN\s*\w*\s*: ([\w-]+)"}, + "Board Serial Number": {"regex": "Switch Serial No: (\w+)"}, + "Model": {"regex": "MTM\s*\w*\s*: ([\w-]+)"}, + "FRU Number": {"regex": "Hardware Part\s*\w*\s*: (\w+)"}, + "Airflow": {"regex": "System Fan Airflow\s*\w*\s*: ([\w-]+)"}, + } + + invinfo = { + "inventory": [{ + "name": "System", + "present": True, + "information": { + "Manufacturer": "Lenovo", + } + }] + } + + switch_data["sensors"] = [] + + switch_data["fans"] = gather_fans(switch_lines) + for fan, data in switch_data["fans"].items(): + if "rpm" in data: + health = "ok" + if int(data["rpm"]) < 100: + health = "critical" + switch_data["sensors"].append(SwitchSensor(name=f"Fan {fan}", value=data['rpm'], + units="RPM", health=health)) + + switch_data["psus"] = gather_psus(switch_lines) + + # Hunt for the temp limits + phylimit = {"warn": None, "shut": None} + templimit = {"warn": None, "shut": None} + for line in switch_lines: + match = re.match(r"([\w\s]+)Warning[\w\s]+\s(\d+)[\sA-Za-z\/]+\s(\d+)[\s\w\/]+\s(\d*)", line) + if match: + if "System" in match.group(1): + templimit["warn"] = int(match.group(2)) + templimit["shut"] = int(match.group(3)) + elif "PHYs" in match.group(1): + phylimit["warn"] = int(match.group(2)) + phylimit["shut"] = int(match.group(3)) + if not phylimit["warn"]: + phylimit = templimit + + for line in switch_lines: + # match the inventory data + for key in sysinfo.keys(): + match = re.match(re.compile(sysinfo[key]["regex"]), line) + if match: + invinfo["inventory"][0]["information"][key] = match.group(1).strip() + + # match temp sensors logging where failed + match = re.match(r"Temperature\s+([\d\s\w]+)\s*:\s*(\d+)+\s+([CF])+", line) + if match: + health = "ok" + temp = int(match.group(2)) + name = f"{match.group(1).strip()} Temp" + if "Phy" in name: + if temp > phylimit["warn"]: + health = "warning" + if temp > phylimit["shut"]: + health = "critical" + else: + if temp > templimit["warn"]: + health = "warning" + if temp > templimit["shut"]: + health = "critical" + switch_data["sensors"].append(SwitchSensor(name=name, + value=temp, units=f"°{match.group(3)}", health=health)) + match = re.match(r"\s*(\w+) Faults\s*:\s+(.+)", line) + if match and match.group(2) not in ["()", "None"]: + switch_data["sensors"].append(SwitchSensor(name=f"{match.group(1)} Fault", + value=match.group(2).strip(), units="", health="critical")) + + switch_data["inventory"] = invinfo + + sysfw = {"Software Version": "Unknown", "Boot kernel": "Unknown"} + for line in switch_lines: + for key in sysfw.keys(): + regex = f"{key}\s*\w*\s* ([0-9.]+)" + match = re.match(re.compile(regex), line) + if match: + sysfw[key] = match.group(1) + switch_data["firmware"] = sysfw + + return switch_data + + +def gather_psus(data): + psus = {} + for line in data: + # some switches are: + # Power Supply 1: Back-To-Front + # others are: + # Internal Power Supply: On + if "Power Supply" in line: + match = re.match(re.compile(f"Power Supply (\d)+.*"), line) + if match: + psu = match.group(1) + if psu not in psus: + psus[psu] = {} + m = re.match(r".+\s+(\w+\-\w+\-\w+)\s*\[*.*$", line) + if m: + psus[psu]["airflow"] = m.group(1) + psus[psu]["state"] = "Present" + else: + psus[psu]["state"] = "Not installed" + else: + for psu in range(1, 10): + if "Power Supply" in line and psu not in psus: + if psu not in psus: + psus[psu] = {} + if "Not Installed" in line: + psus[psu]["state"] = "Not installed" + break + else: + psus[psu]["state"] = "Present" + break + return psus + + +def gather_fans(data): + fans = {} + for line in data: + # look for presence of fans + if "Fan" in line: + match = re.match(re.compile(f"Fan (\d)+.*"), line) + if match: + fan = match.group(1) + if match: + if fan not in fans: + fans[fan] = {} + if "rpm" in line or "RPM" in line: + if "Module" in line: + m = re.search(r"Module\s+(\d)+:", line) + if m: + fans[fan]["Module"] = m.group(1) + fans[fan]["state"] = "Present" + m = re.search(r"(\d+)\s*:\s+(RPM=)*(\d+)(rpm)*", line) + if m: + fans[fan]["rpm"] = m.group(3) + + m = re.search(r"\s+(PWM=)*(\d+)(%|pwm)+", line) + if m: + fans[fan]["pwm"] = m.group(2) + + m = re.search(r"(.+)\s+(\w+\-\w+\-\w+)$", line) + if m: + fans[fan]["airflow"] = m.group(1) + else: + fans[fan]["state"] = "Not installed" + return fans + + +def retrieve_firmware(configmanager, creds, node, results, element): + if len(element) == 3: + results.put(msg.ChildCollection("all")) + return + sysinfo = gather_data(configmanager, creds, node)["firmware"] + items = [{ + "Software": {"version": sysinfo["Software Version"]}, + }, + { + "Boot kernel": {"version": sysinfo["Boot kernel"]}, + }] + results.put(msg.Firmware(items, node)) + + +def retrieve_health(configmanager, creds, node, results, element): + switch = gather_data(configmanager, creds, node) + badreadings = [] + summary = "ok" + sensors = gather_data(configmanager, creds, node)["sensors"] + + for sensor in sensors: + if sensor.health not in ["ok"]: + if sensor.health in ["critical"]: + summary = "critical" + elif summary in ["ok"] and sensor.health in ["warning"]: + summary = "warning" + badreadings.append(sensor) + results.put(msg.HealthSummary(summary, name=node)) + results.put(msg.SensorReadings(badreadings, name=node)) + + +def retrieve_sensors(configmanager, creds, node, results, element): + sensors = gather_data(configmanager, creds, node)["sensors"] + results.put(msg.SensorReadings(sensors, node)) diff --git a/confluent_server/confluent/plugins/shell/ssh.py b/confluent_server/confluent/plugins/shell/ssh.py index 2a6b65ec..f802f842 100644 --- a/confluent_server/confluent/plugins/shell/ssh.py +++ b/confluent_server/confluent/plugins/shell/ssh.py @@ -43,7 +43,6 @@ if cryptography and cryptography.__version__.split('.') < ['1', '5']: paramiko.transport.Transport._preferred_keys) - class HostKeyHandler(paramiko.client.MissingHostKeyPolicy): def __init__(self, configmanager, node): @@ -112,7 +111,7 @@ class SshShell(conapi.Console): # that would rather not use the nodename as anything but an opaque # identifier self.datacallback = callback - if self.username is not b'': + if self.username != b'': self.logon() else: self.inputmode = 0 @@ -259,6 +258,115 @@ class SshShell(conapi.Console): self.ssh.close() self.datacallback = None + def create(nodes, element, configmanager, inputdata): if len(nodes) == 1: return SshShell(nodes[0], configmanager) + + +class SshConn(): + + def __init__(self, node, config, username=b'', password=b''): + self.node = node + self.ssh = None + self.datacallback = None + self.nodeconfig = config + self.username = username + self.password = password + self.connected = False + self.inputmode = 0 # 0 = username, 1 = password... + + def __del__(self): + if self.connected: + self.close() + + def do_logon(self): + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy( + HostKeyHandler(self.nodeconfig, self.node)) + log.log({'info': f"Connecting to {self.node} by ssh"}) + try: + if self.password: + self.ssh.connect(self.node, username=self.username, + password=self.password, allow_agent=False, + look_for_keys=False) + else: + self.ssh.connect(self.node, username=self.username) + except paramiko.AuthenticationException as e: + self.ssh.close() + self.inputmode = 0 + self.username = b'' + self.password = b'' + log.log({'warn': f"Error connecting to {self.node}: {str(e)}"}) + return + except paramiko.ssh_exception.NoValidConnectionsError as e: + self.ssh.close() + self.inputmode = 0 + self.username = b'' + self.password = b'' + log.log({'warn': f"Error connecting to {self.node}: {str(e)}"}) + return + except cexc.PubkeyInvalid as pi: + self.ssh.close() + self.keyaction = b'' + self.candidatefprint = pi.fingerprint + log.log({'warn': pi.message}) + self.keyattrname = pi.attrname + log.log({'info': f"New fingerprint: {pi.fingerprint}"}) + self.inputmode = -1 + return + except paramiko.SSHException as pi: + self.ssh.close() + self.inputmode = -2 + warn = str(pi) + if warnhostkey: + warn += ' (Older cryptography package on this host only ' \ + 'works with ed25519, check ssh startup on target ' \ + 'and permissions on /etc/ssh/*key)\r\n' + log.log({'warn': warn}) + return + except Exception as e: + self.ssh.close() + self.ssh.close() + self.inputmode = 0 + self.username = b'' + self.password = b'' + log.log({'warn': f"Error connecting to {self.node}: {str(e)}"}) + return + self.inputmode = 2 + self.connected = True + log.log({'info': f"Connected by ssh to {self.node}"}) + + def exec_command(self, cmd, cmdargs): + safecmd = cmd.translate(str.maketrans({"[": r"\]", + "]": r"\]", + "?": r"\?", + "!": r"\!", + "\\": r"\\", + "^": r"\^", + "$": r"\$", + " ": r"\ ", + "*": r"\*"})) + cmds = [safecmd] + for arg in cmdargs: + arg = arg.translate(str.maketrans({"[": r"\]", + "]": r"\]", + "?": r"\?", + "!": r"\!", + "\\": r"\\", + "^": r"\^", + "$": r"\$", + " ": r"\ ", + "*": r"\*"})) + arg = "%s" % (str(arg).replace(r"'", r"'\''"),) + cmds.append(arg) + + runcmd = " ".join(cmds) + stdin, stdout, stderr = self.ssh.exec_command(runcmd) + rcode = stdout.channel.recv_exit_status() + return stdout.readlines(), stderr.readlines() + + def close(self): + if self.ssh is not None: + self.ssh.close() + log.log({'info': f"Disconnected from {self.node}"})