diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py index 0303628b..468358b1 100644 --- a/confluent_server/confluent/config/attributes.py +++ b/confluent_server/confluent/config/attributes.py @@ -188,12 +188,24 @@ node = { # 'appliesto': ['vm'], # }, 'hardwaremanagement.manager': { - 'description': 'The management address dedicated to this node', + 'description': 'The management address dedicated to this node. This ' + 'is the address of, for example, the Lenovo IMM.', }, 'hardwaremanagement.method': { 'description': 'The method used to perform operations such as power ' 'control, get sensor data, get inventory, and so on. ' }, + 'hardwaremanagement.switch': { + 'description': 'The switch to which the hardware manager is connected.' + ' Only relevant if using switch based discovery via the' + ' hardware manager (Lenovo IMMs and CMMs). Not ' + 'applicable to Lenovo Flex nodes.' + }, + 'hardwaremanagement.switchport': { + 'description': 'The port of the switch that the hardware manager is ' + 'connected. See documentation of ' + 'hardwaremanagement.switch for more detail.' + }, 'enclosure.manager': { 'description': "The management device for this node's chassis", # 'appliesto': ['system'], @@ -202,40 +214,44 @@ node = { 'description': 'The bay in the enclosure, if any', # 'appliesto': ['system'], }, + # 'enclosure.type': { # 'description': '''The type of enclosure in use (e.g. IBM BladeCenter, #IBM Flex)''', # 'appliesto': ['system'], # }, -# 'inventory.serialnumber': { +# 'id.serial': { # 'description': 'The manufacturer serial number of node', # }, -# 'inventory.uuid': { +# 'id.uuid': { # 'description': 'The UUID of the node as presented in DMI', # }, -# 'inventory.modelnumber': { +# 'id.modelnumber': { # 'description': 'The manufacturer dictated model number for the node', # }, -# 'inventory.snmpengineid': { +# 'id.modelname': { +# 'description': 'The manufacturer model label for the node', +# }, +# 'id.snmpengineid': { # 'description': 'The SNMP Engine id used by this node', # }, # 'secret.snmpuser': { # 'description': 'The user to use for SNMPv3 access to this node', # }, -# 'secret.snmppassphrase': { -# 'description': 'The passphrase to use for SNMPv3 access to this node', +# 'secret.snmppassword': { +# 'description': 'The password to use for SNMPv3 access to this node', # }, + 'secret.snmpcommunity': { + 'description': ('SNMPv1 community string, it is highly recommended to' + 'step up to SNMPv3'), + }, # 'secret.snmplocalizedkey': { # 'description': ("SNMPv3 key localized to this node's SNMP Engine id" # 'This can be used in lieu of snmppassphrase to avoid' # 'retaining the passphrase TODO: document procedure' # 'to commit passphrase to localized key'), # }, -# 'secret.snmpcommunity': { -# 'description': ('SNMPv1 community string, it is highly recommended to' -# 'step up to SNMPv3'), -# }, -# 'secret.localadminpassphrase': { +# 'secret.adminpassword': { # 'description': ('The passphrase to apply to local root/administrator ' # 'account. ' # 'If the environment is 100% Linux, the value may be ' @@ -246,35 +262,18 @@ node = { # 'AD') # }, 'secret.ipmikg': { - 'description': 'Optional Integrity key for IPMI communication' + 'description': 'Optional Integrity key for IPMI communication. This ' + 'should generally be ignored, as mutual authentication ' + 'is normally done with the password alone (which is a ' + 'shared secret in IPMI)' }, -# 'secret.ipmiuser': { -# 'description': ('The username to use to log into IPMI device related ' -# 'to the node. For setting username, default ' -# 'behavior is to randomize username, for using ' -# 'username if not set, USERID is assumed'), -# }, -# 'secret.ipmipassphrase': { -# 'description': ('The key to use to authenticate to IPMI device ' -# 'related to the node. For setting passphrase, ' -# 'default behavior is to randomize passphrase and ' -# 'store it here. If going to connect over the ' -# 'network and value is not set, PASSW0RD is attempted') -# }, 'secret.hardwaremanagementuser': { - 'description': ('Username to be set and used by protocols like SSH ' - 'and HTTP where client provides passphrase over the ' - 'network. Given the distinct security models betwen ' - 'this class of protocols and SNMP and IPMI, snmp and ' - 'ipmi utilize dedicated values.'), + 'description': ('The username to use when connecting to the hardware ' + 'manager'), }, 'secret.hardwaremanagementpassword': { - 'description': ('Passphrase to be set and used by protocols like SSH ' - 'and HTTP, where client sends passphrase over the ' - 'network. Given distinct security models between ' - 'this class of protocols, SNMP, and IPMI, SNMP and ' - 'IPMI are given their own settings with distinct ' - 'behaviors'), + 'description': ('Password to use when connecting to the hardware ' + 'manager'), }, 'pubkeys.addpolicy': { 'description': ('Policy to use when encountering unknown public ' diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 6986043b..c5043ab9 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -761,7 +761,9 @@ class ConfigManager(object): decrypt=self.decrypt) return nodeobj - def get_node_attributes(self, nodelist, attributes=()): + def get_node_attributes(self, nodelist, attributes=(), decrypt=None): + if decrypt is None: + decrypt = self.decrypt retdict = {} relattribs = attributes if isinstance(nodelist, str) or isinstance(nodelist, unicode): @@ -783,7 +785,7 @@ class ConfigManager(object): # skipped. The decryption, however, we want to do only on # demand nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj, - decrypt=self.decrypt) + decrypt=decrypt) retdict[node] = nodeobj return retdict diff --git a/confluent_server/confluent/log.py b/confluent_server/confluent/log.py index 9269c359..662294a2 100644 --- a/confluent_server/confluent/log.py +++ b/confluent_server/confluent/log.py @@ -722,3 +722,19 @@ class Logger(object): def closelog(self): self.handler.close() self.closer = None + +globaleventlog = None +tracelog = None + + +def log(logdata=None, ltype=None, event=0, eventdata=None): + if globaleventlog is None: + globaleventlog = Logger('events') + globaleventlog.log(logdata, ltype, event, eventdata) + +def logtrace(): + global tracelog + if tracelog is None: + tracelog = Logger('trace') + tracelog.log(traceback.format_exc(), ltype=DataTypes.event, + event=Events.stacktrace) \ No newline at end of file diff --git a/confluent_server/confluent/networking/__init__.py b/confluent_server/confluent/networking/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/confluent_server/confluent/networking/macmap.py b/confluent_server/confluent/networking/macmap.py new file mode 100644 index 00000000..b764929a --- /dev/null +++ b/confluent_server/confluent/networking/macmap.py @@ -0,0 +1,249 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2016 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This provides the implementation of locating MAC addresses on ethernet +# switches. It is, essentially, a port of 'MacMap.pm' to confluent. +# However, there are enhancements. +# For one, each switch interrogation is handled in an eventlet 'thread' +# For another, MAC addresses are checked in the dictionary on every +# switch return, rather than waiting for all switches to check in +# (which makes it more responsive when there is a missing or bad switch) +# Also, we track the quantity, actual ifName value, and provide a mechanism +# to detect ambiguous result (e.g. if two matches are found, can log an error +# rather than doing the wrong one, complete with the detected ifName value). +# Further, the map shall be available to all facets of the codebase, not just +# the discovery process, so that the cached data maintenance will pay off +# for direct queries + +# this module will provide mac to switch and full 'ifName' label +# This functionality is restricted to the null tenant + +import confluent.exceptions as exc +import confluent.log as log +import confluent.snmputil as snmp +from eventlet.greenpool import GreenPool +import re + +_macmap = {} +_macsbyswitch = {} +_nodesbymac = {} +_switchportmap = {} + + +_whitelistnames = ( + # 3com + re.compile(r'^RMON Port (\d+) on unit \d+'), + # Dell + re.compile(r'^Unit \d+ Port (\d+)\Z'), +) + +_blacklistnames = ( + re.compile(r'vl'), + re.compile(r'Nu'), + re.compile(r'RMON'), + re.compile(r'onsole'), + re.compile(r'Stack'), + re.compile(r'Trunk'), + re.compile(r'po\d'), + re.compile(r'XGE'), + re.compile(r'LAG'), + re.compile(r'CPU'), + re.compile(r'Management'), +) + + +def _namesmatch(switchdesc, userdesc): + if switchdesc == userdesc: + return True + try: + portnum = int(userdesc) + except ValueError: + portnum = None + if portnum is not None: + for exp in _whitelistnames: + match = exp.match(switchdesc) + if match: + snum = int(match.groups()[0]) + if snum == portnum: + return True + anymatch = re.search(r'[^0123456789]' + userdesc + r'(\.0)?\Z', switchdesc) + if anymatch: + for blexp in _blacklistnames: + if blexp.match(switchdesc): + return False + return True + return False + +def _map_switch(args): + try: + return _map_switch_backend(args) + except Exception as e: + log.logtrace() + + +def _nodelookup(switch, ifname): + """Get a nodename for a given switch and interface name + """ + for portdesc in _switchportmap.get(switch, {}): + if _namesmatch(ifname, portdesc): + return _switchportmap[switch][portdesc] + return None + + +def _map_switch_backend(args): + """Manipulate portions of mac address map relevant to a given switch + """ + + # 1.3.6.1.2.1.17.7.1.2.2.1.2 - mactoindex (qbridge - preferred) + # if not, check for cisco and if cisco, build list of all relevant vlans: + # .1.3.6.1.4.1.9.9.46.1.6.1.1.5 - trunk port vlan map (cisco only) + # .1.3.6.1.4.1.9.9.68.1.2.2.1.2 - access port vlan map (cisco only) + # if cisco, vlan community string indexed or snmpv3 contest for: + # 1.3.6.1.2.1.17.4.3.1.2 - mactoindx (bridge - low-end switches and cisco) + # .1.3.6.1.2.1.17.1.4.1.2 - bridge index to if index map + # no vlan index or context for: + # .1.3.6.1.2.1.31.1.1.1.1 - ifName... but some switches don't do it + # .1.3.6.1.2.1.2.2.1.2 - ifDescr, usually useless, but a + # fallback if ifName is empty + # + global _macmap + switch, password, user = args + haveqbridge = False + mactobridge = {} + conn = snmp.Session(switch, password, user) + for vb in conn.walk('1.3.6.1.2.1.17.7.1.2.2.1.2'): + haveqbridge = True + oid, bridgeport = vb + if not bridgeport: + continue + oid = str(oid).rsplit('.', 6) # if 7, then oid[1] would be vlan id + macaddr = '{0:02x}:{1:02x}:{2:02x}:{3:02x}:{4:02x}:{5:02x}'.format( + *([int(x) for x in oid[-6:]]) + ) + mactobridge[macaddr] = int(bridgeport) + if not haveqbridge: + raise exc.NotImplementedException('TODO: Bridge-MIB without QBRIDGE') + bridgetoifmap = {} + for vb in conn.walk('1.3.6.1.2.1.17.1.4.1.2'): + bridgeport, ifidx = vb + bridgeport = int(str(bridgeport).rsplit('.', 1)[1]) + bridgetoifmap[bridgeport] = int(ifidx) + ifnamemap = {} + havenames = False + for vb in conn.walk('1.3.6.1.2.1.31.1.1.1.1'): + ifidx, ifname = vb + if not ifname: + continue + havenames = True + ifidx = int(str(ifidx).rsplit('.', 1)[1]) + ifnamemap[ifidx] = str(ifname) + if not havenames: + for vb in conn.walk( '1.3.6.1.2.1.2.2.1.2'): + ifidx, ifname = vb + ifidx = int(str(ifidx).rsplit('.', 1)[1]) + ifnamemap[ifidx] = str(ifname) + maccounts = {} + for mac in mactobridge: + ifname = ifnamemap[bridgetoifmap[mactobridge[mac]]] + if ifname not in maccounts: + maccounts[ifname] = 1 + else: + maccounts[ifname] += 1 + _macsbyswitch[switch] = {} + for mac in mactobridge: + # We want to merge it so that when a mac appears in multiple + # places, it is captured. + ifname = ifnamemap[bridgetoifmap[mactobridge[mac]]] + if mac in _macmap: + _macmap[mac].append((switch, ifname, maccounts[ifname])) + else: + _macmap[mac] = [(switch, ifname, maccounts[ifname])] + if ifname in _macsbyswitch[switch]: + _macsbyswitch[switch][ifname].append(mac) + else: + _macsbyswitch[switch][ifname] = [mac] + nodename = _nodelookup(switch, ifname) + if nodename is not None: + if mac in _nodesbymac and _nodesbymac[mac] != nodename: + log.log({'warning': '{0} and {1} described by ambiguous' + ' switch topology values'.format(nodename, + _nodesbymac[mac] + )}) + _nodesbymac[mac] = nodename + + +def update_macmap(configmanager): + """Interrogate switches to build/update mac table + + Begin a rebuild process. This process is a generator that will yield + as each switch interrogation completes, allowing a caller to + recheck the cache as results become possible, rather + than having to wait for the process to complete to interrogate. + """ + global _macmap + global _nodesbymac + global _switchportmap + # Clear all existing entries + _macmap = {} + _nodesbymac = {} + _switchportmap = {} + if configmanager.tenant is not None: + raise exc.ForbiddenRequest('Network topology not available to tenants') + nodelocations = configmanager.get_node_attributes( + configmanager.list_nodes(), ('hardwaremanagement.switch', + 'hardwaremanagement.switchport')) + switches = set([]) + for node in nodelocations: + cfg = nodelocations[node] + if 'hardwaremanagement.switch' in cfg: + curswitch = cfg['hardwaremanagement.switch']['value'] + switches.add(curswitch) + if 'hardwaremanagement.switchport' in cfg: + portname = cfg['hardwaremanagement.switchport']['value'] + if curswitch not in _switchportmap: + _switchportmap[curswitch] = {} + if portname in _switchportmap[curswitch]: + log.log({'warning': 'Duplicate switch topology config for ' + '{0} and {1}'.format(node, + _switchportmap[ + curswitch][ + portname])}) + _switchportmap[curswitch][portname] = node + switchcfg = configmanager.get_node_attributes( + switches, ('secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword'), decrypt=True) + switchauth = [] + for switch in switches: + password = 'public' + user = None + if (switch in switchcfg and + 'secret.hardwaremanagementpassword' in switchcfg[switch]): + password = switchcfg[switch]['secret.hardwaremanagementpassword'][ + 'value'] + if 'secret.hardwaremanagementuser' in switchcfg[switch]: + user = switchcfg[switch]['secret.hardwaremanagementuser'][ + 'value'] + switchauth.append((switch, password, user)) + pool = GreenPool() + for res in pool.imap(_map_switch, switchauth): + yield res + print(repr(_macmap)) + + +if __name__ == '__main__': + # invoke as switch community + import sys + _map_switch(sys.argv[1], sys.argv[2]) diff --git a/confluent_server/confluent/snmputil.py b/confluent_server/confluent/snmputil.py new file mode 100644 index 00000000..9b5afd0d --- /dev/null +++ b/confluent_server/confluent/snmputil.py @@ -0,0 +1,103 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2016 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This provides a simplified wrapper around snmp implementation roughly +# mapping to the net-snmp commands + +# net-snmp-python was considered as the API is cleaner, but the ability to +# patch pysnmp to have it be eventlet friendly has caused it's selection +# This module simplifies the complex hlapi pysnmp interface + +import confluent.exceptions as exc +import eventlet +from eventlet.support.greendns import getaddrinfo +import socket +snmp = eventlet.import_patched('pysnmp.hlapi') + + +def _get_transport(name): + # Annoyingly, pysnmp does not automatically determine ipv6 v ipv4 + res = getaddrinfo(name, 161, 0, socket.SOCK_DGRAM) + if res[0][0] == socket.AF_INET6: + return snmp.Udp6TransportTarget(res[0][4]) + else: + return snmp.UdpTransportTarget(res[0][4]) + + +class Session(object): + + def __init__(self, server, secret, username=None, context=None): + """Create a new session to interrogate a switch + + If username is not given, it is assumed that + the secret is community string, and v2c is used. If a username given, + it'll assume SHA auth and DES privacy with the secret being the same + for both. + + :param server: The network name/address to target + :param secret: The community string or password + :param username: The username for SNMPv3 + :param context: The SNMPv3 context or index for community indexing + """ + self.server = server + self.context = context + if username is None: + # SNMP v2c + self.authdata = snmp.CommunityData(secret, mpModel=1) + else: + self.authdata = snmp.UsmUserData(username, authKey=secret, + privKey=secret) + self.eng = snmp.SnmpEngine() + + def walk(self, oid): + """Walk over children of a given OID + + This is roughly equivalent to snmpwalk. It will automatically try to + be a snmpbulkwalk if possible. + + :param oid: The SNMP object identifier + """ + # SNMP is a complicated mess of things. Will endeavor to shield caller + # from as much as possible, assuming reasonable defaults when possible. + # there may come a time where we add more parameters to override the + # automatic behavior (e.g. DES is weak, so it's likely to be + # overriden, but some devices only support DES) + tp = _get_transport(self.server) + ctx = snmp.ContextData(self.context) + if '::' in oid: + mib, field = oid.split('::') + obj = snmp.ObjectType(snmp.ObjectIdentity(mib, field)) + else: + obj = snmp.ObjectType(snmp.ObjectIdentity(oid)) + + walking = snmp.bulkCmd(self.eng, self.authdata, tp, ctx, 0, 10, obj, + lexicographicMode=False) + for rsp in walking: + errstr, errnum, erridx, answers = rsp + if errstr: + raise exc.TargetEndpointUnreachable(str(errstr)) + elif errnum: + raise exc.ConfluentException(errnum.prettyPrint()) + for ans in answers: + yield ans + + +if __name__ == '__main__': + import sys + ts = Session(sys.argv[1], 'public') + for kp in ts.walk(sys.argv[2]): + print(str(kp[0])) + print(str(kp[1]))