From 44405baaf39af36d1b695d1f8be6836319257816 Mon Sep 17 00:00:00 2001 From: weragrzeda Date: Wed, 16 Aug 2023 14:30:05 +0200 Subject: [PATCH 001/319] nodeconfig ntp servers for smms --- confluent_client/bin/nodeconfig | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/confluent_client/bin/nodeconfig b/confluent_client/bin/nodeconfig index 06d512c7..0ba8d351 100755 --- a/confluent_client/bin/nodeconfig +++ b/confluent_client/bin/nodeconfig @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/libexec/platform-python # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2017 Lenovo @@ -98,8 +98,17 @@ cfgpaths = { 'static_v6_gateway'), 'bmc.hostname': ( 'configuration/management_controller/hostname', 'hostname'), + 'bmc.ntp': ( + 'configuration/management_controller/ntp/enabled', 'state'), + 'bmc.ntpServer1': ( + 'configuration/management_controller/ntp/servers/1', 'server'), + 'bmc.ntpServer2': ( + 'configuration/management_controller/ntp/servers/2', 'server'), + 'bmc.ntpServer3': ( + 'configuration/management_controller/ntp/servers/3', 'server') } + autodeps = { 'bmc.ipv4_address': (('bmc.ipv4_method', 'static'),) } @@ -113,6 +122,7 @@ client.check_globbing(noderange) setmode = None assignment = {} queryparms = {} + printsys = [] printbmc = [] printextbmc = [] @@ -131,7 +141,6 @@ if len(args) == 1 or options.exclude: queryparms[path] = {} queryparms[path][attrib] = candidate - def _assign_value(): if key not in cfgpaths: setsys[key] = value @@ -233,6 +242,7 @@ else: parse_config_line(args[1:]) session = client.Command() rcode = 0 + if options.restoredefault: session.stop_if_noderange_over(noderange, options.maxnodes) if options.restoredefault.lower() in ( @@ -295,8 +305,10 @@ else: for path in queryparms: if options.comparedefault: continue - rcode |= client.print_attrib_path(path, session, list(queryparms[path]), - NullOpt(), queryparms[path]) + + rcode |= client.print_attrib_path(path, session, list(queryparms[path]),NullOpt(), queryparms[path]) + + if printsys == 'all' or printextbmc or printbmc or printallbmc: if printbmc or not printextbmc: rcode |= client.print_attrib_path( From 6d87d11f5e6decd854995d4b5bd7188c049c418a Mon Sep 17 00:00:00 2001 From: weragrzeda Date: Wed, 23 Aug 2023 14:28:33 +0200 Subject: [PATCH 002/319] my changes to Eaton PDU sensors --- .../plugins/hardwaremanagement/eatonpdu.py | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py index 16be5b38..da60b182 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py @@ -158,7 +158,7 @@ def get_sensor_data(element, node, configmanager): sn = _sensors_by_node.get(node, None) for outlet in sn[0]: for sensename in sn[0][outlet]: - myname = 'Outlet {0} {1}'.format(outlet, sensename) + myname = '{0} {1}'.format(outlet, sensename) measurement = sn[0][outlet][sensename] if name == 'all' or simplify_name(myname) == name: readings.append({ @@ -259,6 +259,11 @@ class PDUClient(object): url = '/config/gateway?page={}&sessionId={}&_dc={}'.format(suburl, self.sessid, int(time.time())) return wc.grab_response(url) + def do_request1(self, suburl): + wc = self.wc + url = '/config/gateway?page={}&sessionId={}'.format(suburl, self.sessid) + return wc.grab_response(url) + def logout(self): self.do_request('cgi_logout') @@ -274,29 +279,51 @@ class PDUClient(object): return def get_sensor_data(self): - rsp = self.do_request('cgi_pdu_outlets') + rsp = self.do_request1('cgi_overview') + data = sanitize_json(rsp[0]) data = json.loads(data) + data1 = data['data'][4][0][8] data = data['data'][0] sdata = {} for outdata in data: + outsense = {} - outletname = outdata[0][0] - outsense['Energy'] = { - 'value': float(outdata[11] / 1000), - 'units': 'kwh', - 'type': 'Energy' - } + outletname = outdata[3] outsense['Power'] = { - 'value': float(outdata[4]), - 'units': 'w', + 'value': outdata[5], + 'units': 'kW', 'type': 'Power', } sdata[outletname] = outsense + for outdata in data1: + + outsense = {} + outletname = outdata[0] + if type(outdata[1]) == str : + splitter = outdata[1].split(" ") + + if len(splitter) == 1: + splitter.append('w') + outsense['Power'] = { + 'value': splitter[0], + 'units': splitter[1], + 'type': 'Power', + } + elif type(outdata[1]) == list: + if type(outdata[1][1]) == float: + outletname=outletname.strip('
Since') + + outsense['Energy'] = { + 'value': outdata[1][0] / 1000, + 'units': 'kWh', + 'type': 'Energy', + } + sdata[outletname] = outsense return sdata def set_outlet(self, outlet, state): - rsp = self.do_request('cgi_pdu_outlets') + rsp = self.do_request('cgi_overview') data = sanitize_json(rsp[0]) data = json.loads(data) data = data['data'][0] From f633c93d0f6dc45af805c3830e02c596cefe74a7 Mon Sep 17 00:00:00 2001 From: weragrzeda Date: Fri, 25 Aug 2023 08:27:27 +0200 Subject: [PATCH 003/319] Geit PDU nodeinvntory Please enter the commit message for your changes. Lines starting with '' will be ignored, and an empty message aborts the commit. On branch master Your branch is up to date with 'origin/master'. Changes to be committed: modified: geist.py --- .../plugins/hardwaremanagement/geist.py | 264 +++++++++++------- 1 file changed, 166 insertions(+), 98 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/geist.py b/confluent_server/confluent/plugins/hardwaremanagement/geist.py index 3af6fa49..64bf04f3 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/geist.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/geist.py @@ -20,23 +20,28 @@ import eventlet.green.time as time import eventlet import eventlet.greenpool as greenpool + + def simplify_name(name): - return name.lower().replace(' ', '_').replace('/', '-').replace( - '_-_', '-') + return name.lower().replace(" ", "_").replace("/", "-").replace("_-_", "-") + pdupool = greenpool.GreenPool(128) + def data_by_type(indata): databytype = {} for keyname in indata: obj = indata[keyname] - objtype = obj.get('type', None) + objtype = obj.get("type", None) if not objtype: continue if objtype in databytype: - raise Exception("Multiple instances of type {} not yet supported".format(objtype)) + raise Exception( + "Multiple instances of type {} not yet supported".format(objtype) + ) databytype[objtype] = obj - obj['keyname'] = keyname + obj["keyname"] = keyname return databytype @@ -58,173 +63,227 @@ class GeistClient(object): def wc(self): if self._wc: return self._wc - targcfg = self.configmanager.get_node_attributes(self.node, - ['hardwaremanagement.manager'], - decrypt=True) + targcfg = self.configmanager.get_node_attributes( + self.node, ["hardwaremanagement.manager"], decrypt=True + ) targcfg = targcfg.get(self.node, {}) - target = targcfg.get( - 'hardwaremanagement.manager', {}).get('value', None) + target = targcfg.get("hardwaremanagement.manager", {}).get("value", None) if not target: target = self.node - target = target.split('/', 1)[0] + target = target.split("/", 1)[0] cv = util.TLSCertVerifier( - self.configmanager, self.node, - 'pubkeys.tls_hardwaremanager').verify_cert + self.configmanager, self.node, "pubkeys.tls_hardwaremanager" + ).verify_cert self._wc = wc.SecureHTTPConnection(target, port=443, verifycallback=cv) return self._wc def login(self, configmanager): - credcfg = configmanager.get_node_attributes(self.node, - ['secret.hardwaremanagementuser', - 'secret.hardwaremanagementpassword'], - decrypt=True) + credcfg = configmanager.get_node_attributes( + self.node, + ["secret.hardwaremanagementuser", "secret.hardwaremanagementpassword"], + decrypt=True, + ) credcfg = credcfg.get(self.node, {}) - username = credcfg.get( - 'secret.hardwaremanagementuser', {}).get('value', None) - passwd = credcfg.get( - 'secret.hardwaremanagementpassword', {}).get('value', None) + username = credcfg.get("secret.hardwaremanagementuser", {}).get("value", None) + passwd = credcfg.get("secret.hardwaremanagementpassword", {}).get("value", None) if not isinstance(username, str): - username = username.decode('utf8') + username = username.decode("utf8") if not isinstance(passwd, str): - passwd = passwd.decode('utf8') + passwd = passwd.decode("utf8") if not username or not passwd: - raise Exception('Missing username or password') + raise Exception("Missing username or password") self.username = username rsp = self.wc.grab_json_response( - '/api/auth/{0}'.format(username), - {'cmd': 'login', 'data': {'password': passwd}}) - token = rsp['data']['token'] + "/api/auth/{0}".format(username), + {"cmd": "login", "data": {"password": passwd}}, + ) + token = rsp["data"]["token"] return token def logout(self): if self._token: - self.wc.grab_json_response('/api/auth/{0}'.format(self.username), - {'cmd': 'logout', 'token': self.token}) + self.wc.grab_json_response( + "/api/auth/{0}".format(self.username), + {"cmd": "logout", "token": self.token}, + ) self._token = None def get_outlet(self, outlet): - rsp = self.wc.grab_json_response('/api/dev') - rsp = rsp['data'] + rsp = self.wc.grab_json_response("/api/dev") + rsp = rsp["data"] dbt = data_by_type(rsp) - if 't3hd' in dbt: - del dbt['t3hd'] + if "t3hd" in dbt: + del dbt["t3hd"] if len(dbt) != 1: - raise Exception('Multiple PDUs not supported per pdu') + raise Exception("Multiple PDUs not supported per pdu") pdutype = list(dbt)[0] - outlet = dbt[pdutype]['outlet'][str(int(outlet) - 1)] - state = outlet['state'].split('2')[-1] + outlet = dbt[pdutype]["outlet"][str(int(outlet) - 1)] + state = outlet["state"].split("2")[-1] return state def set_outlet(self, outlet, state): - rsp = self.wc.grab_json_response('/api/dev') - dbt = data_by_type(rsp['data']) - if 't3hd' in dbt: - del dbt['t3hd'] + rsp = self.wc.grab_json_response("/api/dev") + dbt = data_by_type(rsp["data"]) + if "t3hd" in dbt: + del dbt["t3hd"] if len(dbt) != 1: self.logout() - raise Exception('Multiple PDUs per endpoint not supported') - pdu = dbt[list(dbt)[0]]['keyname'] + raise Exception("Multiple PDUs per endpoint not supported") + pdu = dbt[list(dbt)[0]]["keyname"] outlet = int(outlet) - 1 rsp = self.wc.grab_json_response( - '/api/dev/{0}/outlet/{1}'.format(pdu, outlet), - {'cmd': 'control', 'token': self.token, - 'data': {'action': state, 'delay': False}}) + "/api/dev/{0}/outlet/{1}".format(pdu, outlet), + { + "cmd": "control", + "token": self.token, + "data": {"action": state, "delay": False}, + }, + ) -def process_measurement(keyname, name, enttype, entname, measurement, readings, category): - if measurement['type'] == 'realPower': - if category not in ('all', 'power'): + +def process_measurement( + keyname, name, enttype, entname, measurement, readings, category +): + if measurement["type"] == "realPower": + if category not in ("all", "power"): return - readtype = 'Real Power' - elif measurement['type'] == 'apparentPower': - if category not in ('all', 'power'): + readtype = "Real Power" + elif measurement["type"] == "apparentPower": + if category not in ("all", "power"): return - readtype = 'Apparent Power' - elif measurement['type'] == 'energy': - if category not in ('all', 'energy'): + readtype = "Apparent Power" + elif measurement["type"] == "energy": + if category not in ("all", "energy"): return - readtype = 'Energy' - elif measurement['type'] == 'voltage': - if category not in ('all',): + readtype = "Energy" + elif measurement["type"] == "voltage": + if category not in ("all",): return - readtype = 'Voltage' - elif measurement['type'] == 'temperature': - readtype = 'Temperature' - elif measurement['type'] == 'dewpoint': - readtype = 'Dewpoint' - elif measurement['type'] == 'humidity': - readtype = 'Humidity' + readtype = "Voltage" + elif measurement["type"] == "temperature": + readtype = "Temperature" + elif measurement["type"] == "dewpoint": + readtype = "Dewpoint" + elif measurement["type"] == "humidity": + readtype = "Humidity" else: return - myname = entname + ' ' + readtype - if name != 'all' and simplify_name(myname) != name: + myname = entname + " " + readtype + if name != "all" and simplify_name(myname) != name: return - readings.append({ - 'name': myname, - 'value': float(measurement['value']), - 'units': measurement['units'], - 'type': readtype.split()[-1] - }) - + readings.append( + { + "name": myname, + "value": float(measurement["value"]), + "units": measurement["units"], + "type": readtype.split()[-1], + } + ) + def process_measurements(name, category, measurements, enttype, readings): for measure in util.natural_sort(list(measurements)): - measurement = measurements[measure]['measurement'] - entname = measurements[measure]['name'] + measurement = measurements[measure]["measurement"] + entname = measurements[measure]["name"] for measureid in measurement: - process_measurement(measure, name, enttype, entname, measurement[measureid], readings, category) - + process_measurement( + measure, + name, + enttype, + entname, + measurement[measureid], + readings, + category, + ) + _sensors_by_node = {} + + def read_sensors(element, node, configmanager): category, name = element[-2:] justnames = False if len(element) == 3: # just get names category = name - name = 'all' + name = "all" justnames = True - if category in ('leds, fans', 'temperature'): + if category in ("leds, fans", "temperature"): return sn = _sensors_by_node.get(node, None) if not sn or sn[1] < time.time(): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response('/api/dev') + adev = gc.wc.grab_json_response("/api/dev") _sensors_by_node[node] = (adev, time.time() + 1) sn = _sensors_by_node.get(node, None) - dbt = data_by_type(sn[0]['data']) + dbt = data_by_type(sn[0]["data"]) readings = [] - for datatype in dbt: + for datatype in dbt: datum = dbt[datatype] - process_measurements(name, category, datum['entity'], 'entity', readings) - if 'outlet' in datum: - process_measurements(name, category, datum['outlet'], 'outlet', readings) + process_measurements(name, category, datum["entity"], "entity", readings) + if "outlet" in datum: + process_measurements(name, category, datum["outlet"], "outlet", readings) if justnames: for reading in readings: - yield msg.ChildCollection(simplify_name(reading['name'])) + yield msg.ChildCollection(simplify_name(reading["name"])) else: yield msg.SensorReadings(readings, name=node) + def get_outlet(node, configmanager, element): gc = GeistClient(node, configmanager) state = gc.get_outlet(element[-1]) return msg.PowerState(node=node, state=state) + def read_firmware(node, configmanager): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response('/api/sys') - myversion = adev['data']['version'] - yield msg.Firmware([{'PDU Firmware': {'version': myversion}}], node) + adev = gc.wc.grab_json_response("/api/sys") + myversion = adev["data"]["version"] + yield msg.Firmware([{"PDU Firmware": {"version": myversion}}], node) + + +def read_inventory(element, node, configmanager): + _inventory = {} + inventory = {} + gc = GeistClient(node, configmanager) + adev = gc.wc.grab_json_response("/api/sys") + basedata = adev["data"] + inventory["present"] = True + inventory["name"] = "PDU" + for elem in basedata.items(): + if ( + elem[0] != "component" + and elem[0] != "locale" + and elem[0] != "state" + and elem[0] != "contact" + ): # and elem[0] !='name': + _inventory[elem[0]] = elem[1] + elif elem[0] == "component": + tempname = "" + for component in basedata["component"].items(): + for item in component: + if type(item) == str: + tempname = item + else: + for entry in item.items(): + _inventory[tempname + " " + entry[0]] = entry[1] + + inventory["information"] = _inventory + + yield msg.KeyValueData({"inventory": [inventory]}, node) + def retrieve(nodes, element, configmanager, inputdata): - if 'outlets' in element: + if "outlets" in element: gp = greenpool.GreenPile(pdupool) for node in nodes: - gp.spawn(get_outlet, node, configmanager, element) + gp.spawn(get_outlet, element, node, configmanager) for res in gp: yield res - + return - elif element[0] == 'sensors': + elif element[0] == "sensors": gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_sensors, element, node, configmanager) @@ -232,21 +291,30 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum return - elif '/'.join(element).startswith('inventory/firmware/all'): + elif "/".join(element).startswith("inventory/firmware/all"): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_firmware, node, configmanager) for rsp in gp: for datum in rsp: yield datum + + elif "/".join(element).startswith("inventory/hardware/all"): + gp = greenpool.GreenPile(pdupool) + for node in nodes: + gp.spawn(read_inventory, element, node, configmanager) + for rsp in gp: + for datum in rsp: + yield datum else: for node in nodes: - yield msg.ConfluentResourceUnavailable(node, 'Not implemented') + yield msg.ConfluentResourceUnavailable(node, "Not implemented") return - + + def update(nodes, element, configmanager, inputdata): - if 'outlets' not in element: - yield msg.ConfluentResourceUnavailable(node, 'Not implemented') + if "outlets" not in element: + yield msg.ConfluentResourceUnavailable(node, "Not implemented") return for node in nodes: gc = GeistClient(node, configmanager) From c95052438688cf9481ae9f18612340ed23325393 Mon Sep 17 00:00:00 2001 From: weragrzeda Date: Thu, 7 Sep 2023 13:08:33 +0200 Subject: [PATCH 004/319] geist.py outlet state fix --- .../plugins/hardwaremanagement/geist.py | 264 ++++++++---------- 1 file changed, 119 insertions(+), 145 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/geist.py b/confluent_server/confluent/plugins/hardwaremanagement/geist.py index 64bf04f3..620a0576 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/geist.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/geist.py @@ -19,29 +19,25 @@ import confluent.exceptions as exc import eventlet.green.time as time import eventlet import eventlet.greenpool as greenpool - - +import json def simplify_name(name): - return name.lower().replace(" ", "_").replace("/", "-").replace("_-_", "-") - + return name.lower().replace(' ', '_').replace('/', '-').replace( + '_-_', '-') pdupool = greenpool.GreenPool(128) - def data_by_type(indata): databytype = {} for keyname in indata: obj = indata[keyname] - objtype = obj.get("type", None) + objtype = obj.get('type', None) if not objtype: continue if objtype in databytype: - raise Exception( - "Multiple instances of type {} not yet supported".format(objtype) - ) + raise Exception("Multiple instances of type {} not yet supported".format(objtype)) databytype[objtype] = obj - obj["keyname"] = keyname + obj['keyname'] = keyname return databytype @@ -63,227 +59,206 @@ class GeistClient(object): def wc(self): if self._wc: return self._wc - targcfg = self.configmanager.get_node_attributes( - self.node, ["hardwaremanagement.manager"], decrypt=True - ) + targcfg = self.configmanager.get_node_attributes(self.node, + ['hardwaremanagement.manager'], + decrypt=True) targcfg = targcfg.get(self.node, {}) - target = targcfg.get("hardwaremanagement.manager", {}).get("value", None) + target = targcfg.get( + 'hardwaremanagement.manager', {}).get('value', None) if not target: target = self.node - target = target.split("/", 1)[0] + target = target.split('/', 1)[0] cv = util.TLSCertVerifier( - self.configmanager, self.node, "pubkeys.tls_hardwaremanager" - ).verify_cert + self.configmanager, self.node, + 'pubkeys.tls_hardwaremanager').verify_cert self._wc = wc.SecureHTTPConnection(target, port=443, verifycallback=cv) return self._wc def login(self, configmanager): - credcfg = configmanager.get_node_attributes( - self.node, - ["secret.hardwaremanagementuser", "secret.hardwaremanagementpassword"], - decrypt=True, - ) + credcfg = configmanager.get_node_attributes(self.node, + ['secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword'], + decrypt=True) credcfg = credcfg.get(self.node, {}) - username = credcfg.get("secret.hardwaremanagementuser", {}).get("value", None) - passwd = credcfg.get("secret.hardwaremanagementpassword", {}).get("value", None) + username = credcfg.get( + 'secret.hardwaremanagementuser', {}).get('value', None) + passwd = credcfg.get( + 'secret.hardwaremanagementpassword', {}).get('value', None) if not isinstance(username, str): - username = username.decode("utf8") + username = username.decode('utf8') if not isinstance(passwd, str): - passwd = passwd.decode("utf8") + passwd = passwd.decode('utf8') if not username or not passwd: - raise Exception("Missing username or password") + raise Exception('Missing username or password') self.username = username rsp = self.wc.grab_json_response( - "/api/auth/{0}".format(username), - {"cmd": "login", "data": {"password": passwd}}, - ) - token = rsp["data"]["token"] + '/api/auth/{0}'.format(username), + {'cmd': 'login', 'data': {'password': passwd}}) + token = rsp['data']['token'] return token def logout(self): if self._token: - self.wc.grab_json_response( - "/api/auth/{0}".format(self.username), - {"cmd": "logout", "token": self.token}, - ) + self.wc.grab_json_response('/api/auth/{0}'.format(self.username), + {'cmd': 'logout', 'token': self.token}) self._token = None def get_outlet(self, outlet): - rsp = self.wc.grab_json_response("/api/dev") - rsp = rsp["data"] + rsp = self.wc.grab_json_response('/api/dev') + rsp = rsp['data'] dbt = data_by_type(rsp) - if "t3hd" in dbt: - del dbt["t3hd"] + + if 't3hd' in dbt: + del dbt['t3hd'] if len(dbt) != 1: - raise Exception("Multiple PDUs not supported per pdu") + raise Exception('Multiple PDUs not supported per pdu') pdutype = list(dbt)[0] - outlet = dbt[pdutype]["outlet"][str(int(outlet) - 1)] - state = outlet["state"].split("2")[-1] + + outlet = dbt[pdutype]['outlet'][ str(int(outlet[0]) - 1) ] + + state = outlet['state'].split('2')[-1] return state def set_outlet(self, outlet, state): - rsp = self.wc.grab_json_response("/api/dev") - dbt = data_by_type(rsp["data"]) - if "t3hd" in dbt: - del dbt["t3hd"] + rsp = self.wc.grab_json_response('/api/dev') + dbt = data_by_type(rsp['data']) + if 't3hd' in dbt: + del dbt['t3hd'] if len(dbt) != 1: self.logout() - raise Exception("Multiple PDUs per endpoint not supported") - pdu = dbt[list(dbt)[0]]["keyname"] + raise Exception('Multiple PDUs per endpoint not supported') + pdu = dbt[list(dbt)[0]]['keyname'] outlet = int(outlet) - 1 rsp = self.wc.grab_json_response( - "/api/dev/{0}/outlet/{1}".format(pdu, outlet), - { - "cmd": "control", - "token": self.token, - "data": {"action": state, "delay": False}, - }, - ) + '/api/dev/{0}/outlet/{1}'.format(pdu, outlet), + {'cmd': 'control', 'token': self.token, + 'data': {'action': state, 'delay': False}}) - -def process_measurement( - keyname, name, enttype, entname, measurement, readings, category -): - if measurement["type"] == "realPower": - if category not in ("all", "power"): +def process_measurement(keyname, name, enttype, entname, measurement, readings, category): + if measurement['type'] == 'realPower': + if category not in ('all', 'power'): return - readtype = "Real Power" - elif measurement["type"] == "apparentPower": - if category not in ("all", "power"): + readtype = 'Real Power' + elif measurement['type'] == 'apparentPower': + if category not in ('all', 'power'): return - readtype = "Apparent Power" - elif measurement["type"] == "energy": - if category not in ("all", "energy"): + readtype = 'Apparent Power' + elif measurement['type'] == 'energy': + if category not in ('all', 'energy'): return - readtype = "Energy" - elif measurement["type"] == "voltage": - if category not in ("all",): + readtype = 'Energy' + elif measurement['type'] == 'voltage': + if category not in ('all',): return - readtype = "Voltage" - elif measurement["type"] == "temperature": - readtype = "Temperature" - elif measurement["type"] == "dewpoint": - readtype = "Dewpoint" - elif measurement["type"] == "humidity": - readtype = "Humidity" + readtype = 'Voltage' + elif measurement['type'] == 'temperature': + readtype = 'Temperature' + elif measurement['type'] == 'dewpoint': + readtype = 'Dewpoint' + elif measurement['type'] == 'humidity': + readtype = 'Humidity' else: return - myname = entname + " " + readtype - if name != "all" and simplify_name(myname) != name: + myname = entname + ' ' + readtype + if name != 'all' and simplify_name(myname) != name: return - readings.append( - { - "name": myname, - "value": float(measurement["value"]), - "units": measurement["units"], - "type": readtype.split()[-1], - } - ) - + readings.append({ + 'name': myname, + 'value': float(measurement['value']), + 'units': measurement['units'], + 'type': readtype.split()[-1] + }) + def process_measurements(name, category, measurements, enttype, readings): for measure in util.natural_sort(list(measurements)): - measurement = measurements[measure]["measurement"] - entname = measurements[measure]["name"] + measurement = measurements[measure]['measurement'] + entname = measurements[measure]['name'] for measureid in measurement: - process_measurement( - measure, - name, - enttype, - entname, - measurement[measureid], - readings, - category, - ) - + process_measurement(measure, name, enttype, entname, measurement[measureid], readings, category) + _sensors_by_node = {} - - def read_sensors(element, node, configmanager): category, name = element[-2:] justnames = False if len(element) == 3: # just get names category = name - name = "all" + name = 'all' justnames = True - if category in ("leds, fans", "temperature"): + if category in ('leds, fans', 'temperature'): return sn = _sensors_by_node.get(node, None) if not sn or sn[1] < time.time(): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response("/api/dev") + adev = gc.wc.grab_json_response('/api/dev') _sensors_by_node[node] = (adev, time.time() + 1) sn = _sensors_by_node.get(node, None) - dbt = data_by_type(sn[0]["data"]) + dbt = data_by_type(sn[0]['data']) readings = [] - for datatype in dbt: + for datatype in dbt: datum = dbt[datatype] - process_measurements(name, category, datum["entity"], "entity", readings) - if "outlet" in datum: - process_measurements(name, category, datum["outlet"], "outlet", readings) + process_measurements(name, category, datum['entity'], 'entity', readings) + if 'outlet' in datum: + process_measurements(name, category, datum['outlet'], 'outlet', readings) if justnames: for reading in readings: - yield msg.ChildCollection(simplify_name(reading["name"])) + yield msg.ChildCollection(simplify_name(reading['name'])) else: yield msg.SensorReadings(readings, name=node) - -def get_outlet(node, configmanager, element): +def get_outlet(element, node, configmanager): gc = GeistClient(node, configmanager) state = gc.get_outlet(element[-1]) - return msg.PowerState(node=node, state=state) + return msg.PowerState(node=node, state=state) def read_firmware(node, configmanager): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response("/api/sys") - myversion = adev["data"]["version"] - yield msg.Firmware([{"PDU Firmware": {"version": myversion}}], node) + adev = gc.wc.grab_json_response('/api/sys') + myversion = adev['data']['version'] + yield msg.Firmware([{'PDU Firmware': {'version': myversion}}], node) def read_inventory(element, node, configmanager): _inventory = {} inventory = {} gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response("/api/sys") - basedata = adev["data"] - inventory["present"] = True - inventory["name"] = "PDU" + adev = gc.wc.grab_json_response('/api/sys') + basedata = adev['data'] + inventory['present'] = True + inventory['name'] = 'PDU' for elem in basedata.items(): - if ( - elem[0] != "component" - and elem[0] != "locale" - and elem[0] != "state" - and elem[0] != "contact" - ): # and elem[0] !='name': - _inventory[elem[0]] = elem[1] - elif elem[0] == "component": - tempname = "" - for component in basedata["component"].items(): + if elem[0] !='component' and elem[0] !='locale' and elem[0] !='state' and elem[0] !='contact': # and elem[0] !='name': + _inventory[elem[0]] = elem[1] + elif elem[0] =='component': + tempname = '' + for component in basedata['component'].items(): for item in component: if type(item) == str: tempname = item else: for entry in item.items(): - _inventory[tempname + " " + entry[0]] = entry[1] + _inventory[tempname + ' ' + entry[0]] = entry[1] + - inventory["information"] = _inventory - - yield msg.KeyValueData({"inventory": [inventory]}, node) + inventory['information']= _inventory + yield msg.KeyValueData({'inventory': [inventory]}, node) + def retrieve(nodes, element, configmanager, inputdata): - if "outlets" in element: + if 'outlets' in element: gp = greenpool.GreenPile(pdupool) for node in nodes: + print(element) gp.spawn(get_outlet, element, node, configmanager) for res in gp: yield res - + return - elif element[0] == "sensors": + elif element[0] == 'sensors': gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_sensors, element, node, configmanager) @@ -291,7 +266,7 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum return - elif "/".join(element).startswith("inventory/firmware/all"): + elif '/'.join(element).startswith('inventory/firmware/all'): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_firmware, node, configmanager) @@ -299,7 +274,7 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum - elif "/".join(element).startswith("inventory/hardware/all"): + elif '/'.join(element).startswith('inventory/hardware/all'): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_inventory, element, node, configmanager) @@ -308,13 +283,12 @@ def retrieve(nodes, element, configmanager, inputdata): yield datum else: for node in nodes: - yield msg.ConfluentResourceUnavailable(node, "Not implemented") + yield msg.ConfluentResourceUnavailable(node, 'Not implemented') return - - + def update(nodes, element, configmanager, inputdata): - if "outlets" not in element: - yield msg.ConfluentResourceUnavailable(node, "Not implemented") + if 'outlets' not in element: + yield msg.ConfluentResourceUnavailable(node, 'Not implemented') return for node in nodes: gc = GeistClient(node, configmanager) From 7b19e67bf0daeb87a0b526a0acf636c7e50bcd81 Mon Sep 17 00:00:00 2001 From: weragrzeda Date: Thu, 21 Sep 2023 06:27:12 +0200 Subject: [PATCH 005/319] amperage and outlet fix for geist.py --- .../plugins/hardwaremanagement/geist.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/geist.py b/confluent_server/confluent/plugins/hardwaremanagement/geist.py index 620a0576..a628eb34 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/geist.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/geist.py @@ -84,6 +84,7 @@ class GeistClient(object): 'secret.hardwaremanagementuser', {}).get('value', None) passwd = credcfg.get( 'secret.hardwaremanagementpassword', {}).get('value', None) + if not isinstance(username, str): username = username.decode('utf8') if not isinstance(passwd, str): @@ -94,6 +95,7 @@ class GeistClient(object): rsp = self.wc.grab_json_response( '/api/auth/{0}'.format(username), {'cmd': 'login', 'data': {'password': passwd}}) + token = rsp['data']['token'] return token @@ -113,9 +115,7 @@ class GeistClient(object): if len(dbt) != 1: raise Exception('Multiple PDUs not supported per pdu') pdutype = list(dbt)[0] - - outlet = dbt[pdutype]['outlet'][ str(int(outlet[0]) - 1) ] - + outlet = dbt[pdutype]['outlet'][ str(int(outlet.join(c for c in outlet if c.isdigit())) - 1) ] state = outlet['state'].split('2')[-1] return state @@ -128,7 +128,8 @@ class GeistClient(object): self.logout() raise Exception('Multiple PDUs per endpoint not supported') pdu = dbt[list(dbt)[0]]['keyname'] - outlet = int(outlet) - 1 + outlet = int(outlet.join(c for c in outlet if c.isdigit())) - 1 + rsp = self.wc.grab_json_response( '/api/dev/{0}/outlet/{1}'.format(pdu, outlet), {'cmd': 'control', 'token': self.token, @@ -151,6 +152,10 @@ def process_measurement(keyname, name, enttype, entname, measurement, readings, if category not in ('all',): return readtype = 'Voltage' + elif measurement['type'] == 'current': + if category not in ('all',): + return + readtype = 'Current' elif measurement['type'] == 'temperature': readtype = 'Temperature' elif measurement['type'] == 'dewpoint': @@ -196,10 +201,12 @@ def read_sensors(element, node, configmanager): _sensors_by_node[node] = (adev, time.time() + 1) sn = _sensors_by_node.get(node, None) dbt = data_by_type(sn[0]['data']) + readings = [] for datatype in dbt: datum = dbt[datatype] process_measurements(name, category, datum['entity'], 'entity', readings) + if 'outlet' in datum: process_measurements(name, category, datum['outlet'], 'outlet', readings) if justnames: @@ -249,10 +256,11 @@ def read_inventory(element, node, configmanager): def retrieve(nodes, element, configmanager, inputdata): + if 'outlets' in element: gp = greenpool.GreenPile(pdupool) for node in nodes: - print(element) + gp.spawn(get_outlet, element, node, configmanager) for res in gp: yield res From ef9083062bbb75e4e96c6206f921ce63b7247307 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 3 Oct 2023 10:13:53 -0400 Subject: [PATCH 006/319] Make multiple attempts to fetch networking configuration Since confignet runs early in startup, the networking can be a bit fickle. Tolerate outages during early use. --- .../common/profile/scripts/confignet | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index dec1808d..f2a2edff 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -435,10 +435,26 @@ if __name__ == '__main__': curridx = addr[-1] if curridx in doneidxs: continue - status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') + for tries in (1, 2 3): + try: + status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') + break + except Exception: + if tries == 3: + raise + time.sleep(1) + continue nc = json.loads(nc) if not dc: - status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') + for tries in (1, 2 3): + try: + status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') + break + except Exception: + if tries == 3: + raise + time.sleep(1) + continue dc = json.loads(dc) iname = get_interface_name(idxmap[curridx], nc.get('default', {})) if iname: From ee19386d8c1cb4e4a33b7ec79b2d7dc0fe632976 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Oct 2023 09:49:09 -0400 Subject: [PATCH 007/319] Export nodename in ubuntu pre --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 4ff1878e..5db222a7 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -35,7 +35,7 @@ echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.con echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf systemctl restart sshd mkdir -p /etc/confluent -export confluent_profile confluent_mgr +export nodename confluent_profile confluent_mgr curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions . /etc/confluent/functions run_remote_parts pre.d From 9f168aee7302412f91aaa297000645b4f02162fe Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:28:16 -0400 Subject: [PATCH 008/319] docs- batch file systax --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index b1330198..927ce5ae 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -61,7 +61,9 @@ to a blank value will allow masking a group defined attribute with an empty valu or environment variables. * `-s`, `--set`: - Set attributes using a batch file + Set attributes using a batch file rather than the command line. The attributes in the batch file + can be one line of key=value pairs or each attribute can be in its own line. Lines that start with + # sign will be read as commend. See EXAMPLES for batch file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: Prompt if trying to set attributes on more than @@ -120,6 +122,25 @@ to a blank value will allow masking a group defined attribute with an empty valu `d1: net.pxe.switch: pxeswitch1` `d1: net.switch:` +* Setting Attributes using a batch file with syntax similar to command line: + `# cat nodeattributes.batch` + `# power` + `power.psu1.outlet=3 power.psu1.pdu=pdu2` + `# nodeattrib n41 -s nodeattributes.batch` + `n41: 3` + `n41: pdu2` + +* Setting Attributes using a batch file with syntax where each attribute is in its own line: + `# cat nodeattributes.batch` + `# management` + `custom.mgt.switch=switch_main` + `custom.mgt.switch.port=swp4` + `# nodeattrib n41 -s nodeattributes.batch` + `n41: switch_main` + `n41: swp4` + + + ## SEE ALSO nodegroupattrib(8), nodeattribexpressions(5) From d299db3442f3341f348c4e560397c7a493f9eaf7 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:31:24 -0400 Subject: [PATCH 009/319] doc --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index 927ce5ae..28e37a5c 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -122,7 +122,7 @@ to a blank value will allow masking a group defined attribute with an empty valu `d1: net.pxe.switch: pxeswitch1` `d1: net.switch:` -* Setting Attributes using a batch file with syntax similar to command line: +* Setting attributes using a batch file with syntax similar to command line: `# cat nodeattributes.batch` `# power` `power.psu1.outlet=3 power.psu1.pdu=pdu2` @@ -130,7 +130,7 @@ to a blank value will allow masking a group defined attribute with an empty valu `n41: 3` `n41: pdu2` -* Setting Attributes using a batch file with syntax where each attribute is in its own line: +* Setting attributes using a batch file with syntax where each attribute is in its own line: `# cat nodeattributes.batch` `# management` `custom.mgt.switch=switch_main` From c8094276d0f9dc5129c740b32285adf66746cd11 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:34:07 -0400 Subject: [PATCH 010/319] typ0_fix --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index 28e37a5c..cacfc80f 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -62,8 +62,9 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-s`, `--set`: Set attributes using a batch file rather than the command line. The attributes in the batch file - can be one line of key=value pairs or each attribute can be in its own line. Lines that start with - # sign will be read as commend. See EXAMPLES for batch file syntax. + can be specified as one line of key=value pairs line command line or each attribute can be in + its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch + file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: Prompt if trying to set attributes on more than From ba90609f3b4893f258ac8f66b83edcd166dccaba Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:36:58 -0400 Subject: [PATCH 011/319] documentation for nodeattrib -s --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index cacfc80f..3d66a65f 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -62,8 +62,8 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-s`, `--set`: Set attributes using a batch file rather than the command line. The attributes in the batch file - can be specified as one line of key=value pairs line command line or each attribute can be in - its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch + can be specified as one line of key=value pairs simmilar to command line or each attribute can + be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: From eca1854d563e718c0bc4030d7b0076c22d12bcf1 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:37:41 -0400 Subject: [PATCH 012/319] fix to env doc --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index 3d66a65f..c71e59e7 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -54,7 +54,7 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-e`, `--environment`: Set specified attributes based on exported environment variable of matching name. Environment variable names may be lower case or all upper case. - Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE + Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE) * `-p`, `--prompt`: Request interactive prompting to provide values rather than the command line From 67f607a8f16bb534ecc1a001611b112cb55c7cf7 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 16:26:20 -0400 Subject: [PATCH 013/319] fix to synopsis --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index c71e59e7..c0316a7d 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -8,7 +8,7 @@ nodeattrib(8) -- List or change confluent nodes attributes `nodeattrib -c ...` `nodeattrib -e ...` `nodeattrib -p ...` -`nodeattrib -s ` +`nodeattrib -s ...` ## DESCRIPTION From 2e84c73baaa05cfd3840d3f18afc78926bb55bfd Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 16:27:05 -0400 Subject: [PATCH 014/319] '' --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index c0316a7d..c8127ad8 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -7,7 +7,7 @@ nodeattrib(8) -- List or change confluent nodes attributes `nodeattrib [ ...]` `nodeattrib -c ...` `nodeattrib -e ...` -`nodeattrib -p ...` +`nodeattrib -p ...` `nodeattrib -s ...` ## DESCRIPTION From 77eec1a791ee37ba15e4e2316dc9ed1369647c44 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Thu, 5 Oct 2023 11:44:04 -0400 Subject: [PATCH 015/319] missing_shlex import in nodeattrib --- confluent_client/bin/nodeattrib | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index 8c6f078d..f4b0331f 100755 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -22,6 +22,7 @@ import optparse import os import signal import sys +import shlex try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) From a4ea5e5c4b2dda518af813acdd3de61f1fdb5148 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sat, 7 Oct 2023 09:51:32 -0400 Subject: [PATCH 016/319] Abbreviate sequential nodes When we have sequential nodes, collapse to ':' delimited range. --- confluent_server/confluent/noderange.py | 112 +++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index b59bf7c6..5021d592 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -55,6 +55,92 @@ def humanify_nodename(nodename): return [int(text) if text.isdigit() else text.lower() for text in re.split(numregex, nodename)] +def unnumber_nodename(nodename): + # stub out numbers + chunked = ["{}" if text.isdigit() else text.lower() + for text in re.split(numregex, nodename)] + return chunked + +def getnumbers_nodename(nodename): + return [int(x) for x in re.split(numregex, nodename) if x.isdigit()] + +def group_elements(elems): + """ Take the specefied elements and chunk them according to text similarity + """ + prev = None + currchunk = [] + chunked_elems = [currchunk] + for elem in elems: + elemtxt = unnumber_nodename(elem) + if not prev: + prev = elemtxt + currchunk.append(elem) + continue + if prev == elemtxt: + currchunk.append(elem) + else: + currchunk = [elem] + chunked_elems.append(currchunk) + prev = elemtxt + return chunked_elems + +def abbreviate_chunk(chunk, validset): + if len(chunk) < 3: + return sorted(chunk, key=humanify_nodename) + #chunk = sorted(chunk, key=humanify_nodename) + vset = set(validset) + cset = set(chunk) + mins = None + maxs = None + for name in chunk: + currns = getnumbers_nodename(name) + if mins is None: + mins = list(currns) + maxs = list(currns) + continue + for n in range(len(currns)): + if currns[n] < mins[n]: + mins[n] = currns[n] + if currns[n] > maxs[n]: + maxs[n] = currns[n] + tmplt = ''.join(unnumber_nodename(chunk[0])) + bgnr = tmplt.format(*mins) + endr = tmplt.format(*maxs) + nr = '{}:{}'.format(bgnr, endr) + prospect = NodeRange(nr).nodes + ranges = [] + discontinuities = (prospect - vset).union(prospect - cset) + currstart = None + prevnode = None + chunksize = 0 + prospect = sorted(prospect, key=humanify_nodename) + currstart = prospect[0] + while prospect: + currnode = prospect.pop(0) + if currnode in discontinuities: + if chunksize == 0: + continue + elif chunksize == 1: + ranges.append(prevnode) + elif chunksize == 2: + ranges.append(','.join([currstart, prevnode])) + else: + ranges.append(':'.join([currstart, prevnode])) + chunksize = 0 + currstart = None + continue + elif not currstart: + currstart = currnode + chunksize += 1 + prevnode = currnode + if chunksize == 1: + ranges.append(prevnode) + elif chunksize == 2: + ranges.append(','.join([currstart, prevnode])) + elif chunksize != 0: + ranges.append(':'.join([currstart, prevnode])) + return ranges + class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -71,7 +157,8 @@ class ReverseNodeRange(object): @property def noderange(self): subsetgroups = [] - for group in self.cfm.get_groups(sizesort=True): + allgroups = self.cfm.get_groups(sizesort=True) + for group in allgroups: if lastnoderange: for nr in lastnoderange: if lastnoderange[nr] - self.nodes: @@ -88,7 +175,28 @@ class ReverseNodeRange(object): self.nodes -= nl if not self.nodes: break - return ','.join(sorted(subsetgroups) + sorted(self.nodes)) + # then, analyze sequentially identifying matching alpha subsections + # then try out noderange from beginning to end + # we need to know discontinuities, which are either: + # nodes that appear in the noderange that are not in the nodes + # nodes that do not exist at all (we need a noderange modification + # that returns non existing nodes) + ranges = [] + try: + subsetgroups.sort(key=humanify_nodename) + groupchunks = group_elements(subsetgroups) + for gc in groupchunks: + ranges.extend(abbreviate_chunk(gc, allgroups)) + except Exception: + subsetgroups.sort() + try: + nodes = sorted(self.nodes, key=humanify_nodename) + nodechunks = group_elements(nodes) + for nc in nodechunks: + ranges.extend(abbreviate_chunk(nc, self.cfm.list_nodes())) + except Exception: + ranges = sorted(self.nodes) + return ','.join(ranges) From fe27cdea4a8df9bd10de5a6dd340a3716413a9ce Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 9 Oct 2023 17:18:44 -0400 Subject: [PATCH 017/319] Abbreviate harder, using brackets Add a round that collapses as is convenient to bracketed range. --- confluent_server/confluent/noderange.py | 109 +++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 5021d592..e70fa6fe 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -64,6 +64,105 @@ def unnumber_nodename(nodename): def getnumbers_nodename(nodename): return [int(x) for x in re.split(numregex, nodename) if x.isdigit()] + +class Bracketer(object): + __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens'] + + def __init__(self, nodename): + self.sequences = [] + realnodename = nodename + if ':' in nodename: + realnodename = nodename.split(':', 1)[0] + self.count = len(getnumbers_nodename(realnodename)) + self.nametmpl = unnumber_nodename(realnodename) + for n in range(self.count): + self.sequences.append(None) + self.diffn = None + self.tokens = [] + self.extend(nodename) + + def extend(self, nodeorseq): + # can only differentiate a single number + endname = None + endnums = None + enddifn = self.diffn + if ':' in nodeorseq: + nodename, endname = nodeorseq.split(':', 1) + else: + nodename = nodeorseq + nums = getnumbers_nodename(nodename) + if endname: + diffcount = 0 + endnums = getnumbers_nodename(endname) + ecount = len(endnums) + if ecount != self.count: + raise Exception("mismatched names passed") + for n in range(ecount): + if endnums[n] != nums[n]: + enddifn = n + diffcount += 1 + if diffcount > 1: + if self.sequences: + self.flush_current() + self.tokens.append(nodeorseq) # TODO: could just abbreviate this with multiple []... + return + for n in range(self.count): + if endnums and endnums[n] != nums[n]: + outval = '{}:{}'.format(nums[n], endnums[n]) + else: + outval = '{}'.format(nums[n]) + if self.sequences[n] is None: + # We initialize to text pieces, 'currstart', and 'prev' number + self.sequences[n] = [[outval], nums[n], nums[n]] + elif self.sequences[n][2] == nums[n]: + continue # new nodename has no new number, keep going + elif self.sequences[n][2] != nums[n]: + if self.diffn is not None and (n != self.diffn or enddifn != n): + self.flush_current() + self.sequences[n] = [[], nums[n], nums[n]] + self.diffn = n + self.sequences[n][0].append(outval) + self.sequences[n][2] = nums[n] + elif False: # previous attempt + # A discontinuity, need to close off previous chunk + currstart = self.sequences[n][1] + prevnum = self.sequences[n][2] + if currstart == prevnum: + self.sequences[n][0].append('{}'.format(currstart)) + elif prevnum == currstart + 1: + self.sequences[n][0].append('{},{}'.format(currstart, prevnum)) + else: + self.sequences[n][0].append('{}:{}'.format(currstart, prevnum)) + self.sequences[n][1] = nums[n] + self.sequences[n][2] = nums[n] + elif False: # self.sequences[n][2] == nums[n] - 1: # sequential, increment prev + self.sequences[n][2] = nums[n] + else: + raise Exception('Decreasing node in extend call, not supported') + + def flush_current(self): + txtfields = [] + for n in range(self.count): + txtfield = ','.join(self.sequences[n][0]) + #if self.sequences[n][1] == self.sequences[n][2]: + # txtfield.append('{}'.format(self.sequences[n][1])) + #else: + # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + if txtfield.isdigit(): + txtfields.append(txtfield) + else: + txtfields.append('[{}]'.format(txtfield)) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) + self.sequences = [] + for n in range(self.count): + self.sequences.append(None) + + @property + def range(self): + if self.sequences: + self.flush_current() + return ','.join(self.tokens) + def group_elements(elems): """ Take the specefied elements and chunk them according to text similarity """ @@ -123,7 +222,7 @@ def abbreviate_chunk(chunk, validset): elif chunksize == 1: ranges.append(prevnode) elif chunksize == 2: - ranges.append(','.join([currstart, prevnode])) + ranges.extend([currstart, prevnode]) else: ranges.append(':'.join([currstart, prevnode])) chunksize = 0 @@ -136,7 +235,7 @@ def abbreviate_chunk(chunk, validset): if chunksize == 1: ranges.append(prevnode) elif chunksize == 2: - ranges.append(','.join([currstart, prevnode])) + ranges.extend([currstart, prevnode]) elif chunksize != 0: ranges.append(':'.join([currstart, prevnode])) return ranges @@ -193,7 +292,11 @@ class ReverseNodeRange(object): nodes = sorted(self.nodes, key=humanify_nodename) nodechunks = group_elements(nodes) for nc in nodechunks: - ranges.extend(abbreviate_chunk(nc, self.cfm.list_nodes())) + currchunks = abbreviate_chunk(nc, self.cfm.list_nodes()) + bracketer = Bracketer(currchunks[0]) + for chnk in currchunks[1:]: + bracketer.extend(chnk) + ranges.append(bracketer.range) except Exception: ranges = sorted(self.nodes) return ','.join(ranges) From c254564f021264b7de200507882f8a57715a6926 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 10 Oct 2023 12:47:19 -0400 Subject: [PATCH 018/319] Fully give up on multi-iterator abbreviation There's too many cases that can go wrong. Note that with this lower ambition, it would be possible to significantly streamline the implementation. Notably, the 'find discontinuities' approach was selected to *try* to support multiple iterators, but since that didn't pan out, a more straightforward numerical strategy can be used from the onset. --- confluent_server/confluent/noderange.py | 78 ++++++++++++++++--------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index e70fa6fe..e2f88b74 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -100,11 +100,19 @@ class Bracketer(object): for n in range(ecount): if endnums[n] != nums[n]: enddifn = n + if self.diffn is None: + self.diffn = enddifn diffcount += 1 - if diffcount > 1: + if diffcount > 1 or enddifn != self.diffn: if self.sequences: self.flush_current() - self.tokens.append(nodeorseq) # TODO: could just abbreviate this with multiple []... + txtfields = [] + for idx in range(len(nums)): + if endnums[idx] == nums[idx]: + txtfields.append(nums[idx]) + else: + txtfields.append('[{}:{}]'.format(nums[idx], endnums[idx])) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) return for n in range(self.count): if endnums and endnums[n] != nums[n]: @@ -142,17 +150,18 @@ class Bracketer(object): def flush_current(self): txtfields = [] - for n in range(self.count): - txtfield = ','.join(self.sequences[n][0]) - #if self.sequences[n][1] == self.sequences[n][2]: - # txtfield.append('{}'.format(self.sequences[n][1])) - #else: - # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) - if txtfield.isdigit(): - txtfields.append(txtfield) - else: - txtfields.append('[{}]'.format(txtfield)) - self.tokens.append(''.join(self.nametmpl).format(*txtfields)) + if self.sequences and self.sequences[0] is not None: + for n in range(self.count): + txtfield = ','.join(self.sequences[n][0]) + #if self.sequences[n][1] == self.sequences[n][2]: + # txtfield.append('{}'.format(self.sequences[n][1])) + #else: + # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + if txtfield.isdigit(): + txtfields.append(txtfield) + else: + txtfields.append('[{}]'.format(txtfield)) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) self.sequences = [] for n in range(self.count): self.sequences.append(None) @@ -186,28 +195,41 @@ def group_elements(elems): def abbreviate_chunk(chunk, validset): if len(chunk) < 3: return sorted(chunk, key=humanify_nodename) - #chunk = sorted(chunk, key=humanify_nodename) vset = set(validset) cset = set(chunk) - mins = None - maxs = None + minmaxes = [None] + diffn = None + prevns = None for name in chunk: currns = getnumbers_nodename(name) - if mins is None: - mins = list(currns) - maxs = list(currns) + if minmaxes[-1] is None: + minmaxes[-1] = [list(currns), list(currns)] continue + if prevns is None: + prevns = currns for n in range(len(currns)): - if currns[n] < mins[n]: - mins[n] = currns[n] - if currns[n] > maxs[n]: - maxs[n] = currns[n] + if prevns[n] != currns[n]: + if diffn is None: + diffn = n + elif diffn != n: + minmaxes.append([list(currns), list(currns)]) + continue + if currns[n] < minmaxes[-1][0][n]: + minmaxes.append([list(currns), list(currns)]) + if currns[n] > minmaxes[-1][1][n]: + minmaxes[-1][1][n] = currns[n] + prevns = currns tmplt = ''.join(unnumber_nodename(chunk[0])) + ranges = [] + for x in minmaxes: + process_abbreviation(vset, cset, x[0], x[1], tmplt, ranges) + return ranges + +def process_abbreviation(vset, cset, mins, maxs, tmplt, ranges): bgnr = tmplt.format(*mins) endr = tmplt.format(*maxs) nr = '{}:{}'.format(bgnr, endr) prospect = NodeRange(nr).nodes - ranges = [] discontinuities = (prospect - vset).union(prospect - cset) currstart = None prevnode = None @@ -238,8 +260,6 @@ def abbreviate_chunk(chunk, validset): ranges.extend([currstart, prevnode]) elif chunksize != 0: ranges.append(':'.join([currstart, prevnode])) - return ranges - class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -285,7 +305,11 @@ class ReverseNodeRange(object): subsetgroups.sort(key=humanify_nodename) groupchunks = group_elements(subsetgroups) for gc in groupchunks: - ranges.extend(abbreviate_chunk(gc, allgroups)) + currchunks = abbreviate_chunk(gc, allgroups) + bracketer = Bracketer(currchunks[0]) + for chnk in currchunks[1:]: + bracketer.extend(chnk) + ranges.append(bracketer.range) except Exception: subsetgroups.sort() try: From e9a2f57ad8d262dae5da54bdddfc1906a0e652a4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 10 Oct 2023 16:56:32 -0400 Subject: [PATCH 019/319] Simplify the noderange abbreviation Since the multi-iterator ambition is out, ditch the expensive set wrangling step. Now the procedure is: -Suck nodes into groups, as possible -Separately for groups and nodes: -Sort the elements -Chunk the elements based on 'non-numberical' situation matching -analyze the iterators to apply [] to shorten the name -Multi-iterator will cause a discontinuity, and a new ',' delimited name gets constructed --- confluent_server/confluent/noderange.py | 146 ++++-------------------- 1 file changed, 23 insertions(+), 123 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index e2f88b74..9a7ed44b 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -80,70 +80,39 @@ class Bracketer(object): self.diffn = None self.tokens = [] self.extend(nodename) + if self.count == 0: + self.tokens = [nodename] def extend(self, nodeorseq): # can only differentiate a single number endname = None endnums = None - enddifn = self.diffn if ':' in nodeorseq: nodename, endname = nodeorseq.split(':', 1) else: nodename = nodeorseq nums = getnumbers_nodename(nodename) - if endname: - diffcount = 0 - endnums = getnumbers_nodename(endname) - ecount = len(endnums) - if ecount != self.count: - raise Exception("mismatched names passed") - for n in range(ecount): - if endnums[n] != nums[n]: - enddifn = n - if self.diffn is None: - self.diffn = enddifn - diffcount += 1 - if diffcount > 1 or enddifn != self.diffn: - if self.sequences: - self.flush_current() - txtfields = [] - for idx in range(len(nums)): - if endnums[idx] == nums[idx]: - txtfields.append(nums[idx]) - else: - txtfields.append('[{}:{}]'.format(nums[idx], endnums[idx])) - self.tokens.append(''.join(self.nametmpl).format(*txtfields)) - return for n in range(self.count): - if endnums and endnums[n] != nums[n]: - outval = '{}:{}'.format(nums[n], endnums[n]) - else: - outval = '{}'.format(nums[n]) if self.sequences[n] is None: # We initialize to text pieces, 'currstart', and 'prev' number - self.sequences[n] = [[outval], nums[n], nums[n]] + self.sequences[n] = [[], nums[n], nums[n]] elif self.sequences[n][2] == nums[n]: continue # new nodename has no new number, keep going elif self.sequences[n][2] != nums[n]: - if self.diffn is not None and (n != self.diffn or enddifn != n): + if self.diffn is not None and n != self.diffn: self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] - self.diffn = n - self.sequences[n][0].append(outval) - self.sequences[n][2] = nums[n] - elif False: # previous attempt - # A discontinuity, need to close off previous chunk - currstart = self.sequences[n][1] - prevnum = self.sequences[n][2] - if currstart == prevnum: - self.sequences[n][0].append('{}'.format(currstart)) - elif prevnum == currstart + 1: - self.sequences[n][0].append('{},{}'.format(currstart, prevnum)) + self.diffn = None else: - self.sequences[n][0].append('{}:{}'.format(currstart, prevnum)) - self.sequences[n][1] = nums[n] - self.sequences[n][2] = nums[n] - elif False: # self.sequences[n][2] == nums[n] - 1: # sequential, increment prev + self.diffn = n + if self.sequences[n][2] == (nums[n] - 1): + self.sequences[n][2] = nums[n] + elif self.sequences[n][2] < (nums[n] - 1): + if self.sequences[n][2] != self.sequences[n][1]: + self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + else: + self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + self.sequences[n][1] = nums[n] self.sequences[n][2] = nums[n] else: raise Exception('Decreasing node in extend call, not supported') @@ -152,11 +121,11 @@ class Bracketer(object): txtfields = [] if self.sequences and self.sequences[0] is not None: for n in range(self.count): + if self.sequences[n][1] == self.sequences[n][2]: + self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + else: + self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) txtfield = ','.join(self.sequences[n][0]) - #if self.sequences[n][1] == self.sequences[n][2]: - # txtfield.append('{}'.format(self.sequences[n][1])) - #else: - # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) if txtfield.isdigit(): txtfields.append(txtfield) else: @@ -172,6 +141,7 @@ class Bracketer(object): self.flush_current() return ','.join(self.tokens) + def group_elements(elems): """ Take the specefied elements and chunk them according to text similarity """ @@ -192,74 +162,6 @@ def group_elements(elems): prev = elemtxt return chunked_elems -def abbreviate_chunk(chunk, validset): - if len(chunk) < 3: - return sorted(chunk, key=humanify_nodename) - vset = set(validset) - cset = set(chunk) - minmaxes = [None] - diffn = None - prevns = None - for name in chunk: - currns = getnumbers_nodename(name) - if minmaxes[-1] is None: - minmaxes[-1] = [list(currns), list(currns)] - continue - if prevns is None: - prevns = currns - for n in range(len(currns)): - if prevns[n] != currns[n]: - if diffn is None: - diffn = n - elif diffn != n: - minmaxes.append([list(currns), list(currns)]) - continue - if currns[n] < minmaxes[-1][0][n]: - minmaxes.append([list(currns), list(currns)]) - if currns[n] > minmaxes[-1][1][n]: - minmaxes[-1][1][n] = currns[n] - prevns = currns - tmplt = ''.join(unnumber_nodename(chunk[0])) - ranges = [] - for x in minmaxes: - process_abbreviation(vset, cset, x[0], x[1], tmplt, ranges) - return ranges - -def process_abbreviation(vset, cset, mins, maxs, tmplt, ranges): - bgnr = tmplt.format(*mins) - endr = tmplt.format(*maxs) - nr = '{}:{}'.format(bgnr, endr) - prospect = NodeRange(nr).nodes - discontinuities = (prospect - vset).union(prospect - cset) - currstart = None - prevnode = None - chunksize = 0 - prospect = sorted(prospect, key=humanify_nodename) - currstart = prospect[0] - while prospect: - currnode = prospect.pop(0) - if currnode in discontinuities: - if chunksize == 0: - continue - elif chunksize == 1: - ranges.append(prevnode) - elif chunksize == 2: - ranges.extend([currstart, prevnode]) - else: - ranges.append(':'.join([currstart, prevnode])) - chunksize = 0 - currstart = None - continue - elif not currstart: - currstart = currnode - chunksize += 1 - prevnode = currnode - if chunksize == 1: - ranges.append(prevnode) - elif chunksize == 2: - ranges.extend([currstart, prevnode]) - elif chunksize != 0: - ranges.append(':'.join([currstart, prevnode])) class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -305,9 +207,8 @@ class ReverseNodeRange(object): subsetgroups.sort(key=humanify_nodename) groupchunks = group_elements(subsetgroups) for gc in groupchunks: - currchunks = abbreviate_chunk(gc, allgroups) - bracketer = Bracketer(currchunks[0]) - for chnk in currchunks[1:]: + bracketer = Bracketer(gc[0]) + for chnk in gc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: @@ -316,9 +217,8 @@ class ReverseNodeRange(object): nodes = sorted(self.nodes, key=humanify_nodename) nodechunks = group_elements(nodes) for nc in nodechunks: - currchunks = abbreviate_chunk(nc, self.cfm.list_nodes()) - bracketer = Bracketer(currchunks[0]) - for chnk in currchunks[1:]: + bracketer = Bracketer(nc[0]) + for chnk in nc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: From 2d906e188628aeb3255915a86864135b8d96de7d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 11 Oct 2023 10:15:24 -0400 Subject: [PATCH 020/319] Fix handling of pre-existing array --- misc/swraid | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/swraid b/misc/swraid index 3e234dc8..836f1fb1 100644 --- a/misc/swraid +++ b/misc/swraid @@ -1,6 +1,6 @@ DEVICES="/dev/sda /dev/sdb" RAIDLEVEL=1 -mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null && exit 0 +mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null || ( lvm vgchange -a n mdadm -S -s NUMDEVS=$(for dev in $DEVICES; do @@ -14,5 +14,6 @@ mdadm -C /dev/md/raid $DEVICES -n $NUMDEVS -e 1.0 -l $RAIDLEVEL # shut and restart array to prime things for anaconda mdadm -S -s mdadm --assemble --scan +) readlink /dev/md/raid|sed -e 's/.*\///' > /tmp/installdisk From 6e4d9d9eb485107f327eedf6477d7c5d7308f4ab Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 12 Oct 2023 14:46:09 -0400 Subject: [PATCH 021/319] Address potential slowdowns by misbehaving DNS For one, shorten the DNS timeout, if the DNS server is completely out, give up quickly. For another, if a host has a large number of net.X.hostnames, the sequential nature was intolerable. Have each network be evaluated in a greenthread concurrently to serve the DNS latency concurrently. --- confluent_server/confluent/netutil.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index 37e8d198..9e9fd597 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -25,6 +25,9 @@ import eventlet.support.greendns import os getaddrinfo = eventlet.support.greendns.getaddrinfo +eventlet.support.greendns.resolver.clear() +eventlet.support.greendns.resolver._resolver.lifetime = 1 + def msg_align(len): return (len + 3) & ~3 @@ -333,11 +336,13 @@ def get_full_net_config(configmanager, node, serverip=None): myaddrs = get_addresses_by_serverip(serverip) nm = NetManager(myaddrs, node, configmanager) defaultnic = {} + ppool = eventlet.greenpool.GreenPool(64) if None in attribs: - nm.process_attribs(None, attribs[None]) + ppool.spawn(nm.process_attribs, None, attribs[None]) del attribs[None] for netname in sorted(attribs): - nm.process_attribs(netname, attribs[netname]) + ppool.spawn(nm.process_attribs, netname, attribs[netname]) + ppool.waitall() retattrs = {} if None in nm.myattribs: retattrs['default'] = nm.myattribs[None] From 3a6932ea6dab02a69c03a2a8f30e3a4e7623e5e2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 12 Oct 2023 15:28:54 -0400 Subject: [PATCH 022/319] Start tracking padding during abbreviation This will take care of padding when padding is consistent across a range. However, we still have a problem with a progression like: 01 02 ... 98 099 100 Where numbers in the middle start getting padding unexpectedly without a leading digit. --- confluent_server/confluent/noderange.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 9a7ed44b..1c023707 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -62,14 +62,15 @@ def unnumber_nodename(nodename): return chunked def getnumbers_nodename(nodename): - return [int(x) for x in re.split(numregex, nodename) if x.isdigit()] + return [x for x in re.split(numregex, nodename) if x.isdigit()] class Bracketer(object): - __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens'] + __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens', 'numlens'] def __init__(self, nodename): self.sequences = [] + self.numlens = [] realnodename = nodename if ':' in nodename: realnodename = nodename.split(':', 1)[0] @@ -77,6 +78,7 @@ class Bracketer(object): self.nametmpl = unnumber_nodename(realnodename) for n in range(self.count): self.sequences.append(None) + self.numlens.append([0, 0]) self.diffn = None self.tokens = [] self.extend(nodename) @@ -84,6 +86,7 @@ class Bracketer(object): self.tokens = [nodename] def extend(self, nodeorseq): + # crap... failed to preserve 0 padding foro fixe width # can only differentiate a single number endname = None endnums = None @@ -91,29 +94,37 @@ class Bracketer(object): nodename, endname = nodeorseq.split(':', 1) else: nodename = nodeorseq - nums = getnumbers_nodename(nodename) + txtnums = getnumbers_nodename(nodename) + nums = [int(x) for x in txtnums] for n in range(self.count): if self.sequences[n] is None: # We initialize to text pieces, 'currstart', and 'prev' number self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] elif self.sequences[n][2] == nums[n]: continue # new nodename has no new number, keep going elif self.sequences[n][2] != nums[n]: if self.diffn is not None and n != self.diffn: self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] self.diffn = None else: self.diffn = n if self.sequences[n][2] == (nums[n] - 1): self.sequences[n][2] = nums[n] + self.numlens[n][1] = len(txtnums[n]) elif self.sequences[n][2] < (nums[n] - 1): if self.sequences[n][2] != self.sequences[n][1]: - self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2])) else: - self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) self.sequences[n][1] = nums[n] + self.numlens[n][0] = len(txtnums[n]) self.sequences[n][2] = nums[n] + self.numlens[n][1] = len(txtnums[n]) else: raise Exception('Decreasing node in extend call, not supported') From bfbb7c2843e4353c0342b0b7bb1a4bc40beda534 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 12 Oct 2023 16:09:40 -0400 Subject: [PATCH 023/319] Handle mid-range pad changing, and identical names with only pad difference This would be painful to operate, but if done at least reverse noderange will faithfully honor it now. --- confluent_server/confluent/noderange.py | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 1c023707..e76391e8 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -86,7 +86,6 @@ class Bracketer(object): self.tokens = [nodename] def extend(self, nodeorseq): - # crap... failed to preserve 0 padding foro fixe width # can only differentiate a single number endname = None endnums = None @@ -97,23 +96,26 @@ class Bracketer(object): txtnums = getnumbers_nodename(nodename) nums = [int(x) for x in txtnums] for n in range(self.count): + padto = len(txtnums[n]) + needpad = (padto != len('{}'.format(nums[n]))) if self.sequences[n] is None: # We initialize to text pieces, 'currstart', and 'prev' number self.sequences[n] = [[], nums[n], nums[n]] self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] - elif self.sequences[n][2] == nums[n]: + elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: continue # new nodename has no new number, keep going - elif self.sequences[n][2] != nums[n]: - if self.diffn is not None and n != self.diffn: + else: # if self.sequences[n][2] != nums[n] or : + if self.diffn is not None and (n != self.diffn or + (needpad and padto != self.numlens[n][1])): self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] - self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] + self.numlens[n] = [padto, padto] self.diffn = None else: self.diffn = n if self.sequences[n][2] == (nums[n] - 1): self.sequences[n][2] = nums[n] - self.numlens[n][1] = len(txtnums[n]) + self.numlens[n][1] = padto elif self.sequences[n][2] < (nums[n] - 1): if self.sequences[n][2] != self.sequences[n][1]: fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) @@ -122,20 +124,20 @@ class Bracketer(object): fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) self.sequences[n][1] = nums[n] - self.numlens[n][0] = len(txtnums[n]) + self.numlens[n][0] = padto self.sequences[n][2] = nums[n] - self.numlens[n][1] = len(txtnums[n]) - else: - raise Exception('Decreasing node in extend call, not supported') + self.numlens[n][1] = padto def flush_current(self): txtfields = [] if self.sequences and self.sequences[0] is not None: for n in range(self.count): if self.sequences[n][1] == self.sequences[n][2]: - self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) else: - self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2])) txtfield = ','.join(self.sequences[n][0]) if txtfield.isdigit(): txtfields.append(txtfield) From 0434f38ea12035b98e6997fa06c755bb55694bc9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 13 Oct 2023 15:25:08 -0400 Subject: [PATCH 024/319] Add iterm and kitty image support to stats This delivers improved graphics speed and quality for selected terminals. --- confluent_client/bin/stats | 45 ++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/confluent_client/bin/stats b/confluent_client/bin/stats index 7158e7a7..94af75db 100755 --- a/confluent_client/bin/stats +++ b/confluent_client/bin/stats @@ -16,13 +16,10 @@ # limitations under the License. import argparse +import base64 import csv -import fcntl import io import numpy as np - -import os -import subprocess import sys try: @@ -35,7 +32,31 @@ except ImportError: pass -def plot(gui, output, plotdata, bins): +def iterm_draw(data): + databuf = data.getbuffer() + datalen = len(databuf) + data = base64.b64encode(databuf).decode('utf8') + sys.stdout.write( + '\x1b]1337;File=inline=1;size={}:'.format(datalen)) + sys.stdout.write(data) + sys.stdout.write('\a') + sys.stdout.write('\n') + sys.stdout.flush() + + +def kitty_draw(data): + data = base64.b64encode(data.getbuffer()) + while data: + chunk, data = data[:4096], data[4096:] + m = 1 if data else 0 + sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m)) + sys.stdout.write(chunk.decode('utf8')) + sys.stdout.write('\x1b\\') + sys.stdout.flush() + sys.stdout.write('\n') + + +def plot(gui, output, plotdata, bins, fmt): import matplotlib as mpl if gui and mpl.get_backend() == 'agg': sys.stderr.write('Error: No GUI backend available and -g specified!\n') @@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins): tdata = io.BytesIO() plt.savefig(tdata) if not gui and not output: - writer = DumbWriter() - writer.draw(tdata) + if fmt == 'sixel': + writer = DumbWriter() + writer.draw(tdata) + elif fmt == 'kitty': + kitty_draw(tdata) + elif fmt == 'iterm': + iterm_draw(tdata) return n, bins def textplot(plotdata, bins): @@ -81,7 +107,8 @@ histogram = False aparser = argparse.ArgumentParser(description='Quick access to common statistics') aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)') aparser.add_argument('-d', default=None, help='Value used to separate columns') -aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format') +aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format') +aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)') aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing') aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window') aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format') @@ -138,7 +165,7 @@ while data: data = list(csv.reader([data], delimiter=delimiter))[0] n = None if args.g or args.o or args.x: - n, bins = plot(args.g, args.o, plotdata, bins=args.b) + n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f) if args.t: n, bins = textplot(plotdata, bins=args.b) print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata))) From 06d18cec63e2da6ddf3b0fe3f3bdc4bf0d0412aa Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 16 Oct 2023 08:29:45 -0400 Subject: [PATCH 025/319] Fix abbreviation when pad decreases This is a bizarre way to work, but it should be valid. --- confluent_server/confluent/noderange.py | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index e76391e8..dcf0a1cb 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -106,6 +106,7 @@ class Bracketer(object): continue # new nodename has no new number, keep going else: # if self.sequences[n][2] != nums[n] or : if self.diffn is not None and (n != self.diffn or + (padto < self.numlens[n][1]) or (needpad and padto != self.numlens[n][1])): self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] From b91a19418453b902c4b066e30fd89194c069d6d2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 17 Oct 2023 16:29:30 -0400 Subject: [PATCH 026/319] Improve selfselfice performance with yaml The yaml python default behavior is 'pure python' and is tortuously slow. As a test, yaml dump of a 17,000 element list took 70 seconds in default configuration. Opting into the C functions, that time comes down to 10 seconds, a nice and easy improvement for generic yaml. For dumping a simple dumb list (e.g. the nodelist for ssh), a special case yaml-looking result is done, which hits 0.4 seconds on that same test. So this special case is added to nodelist, which can be very long and very in demand at the same time. --- confluent_server/confluent/selfservice.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index 04030491..cd4180c7 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -19,6 +19,12 @@ import json import os import time import yaml +try: + from yaml import CSafeDumper as SafeDumper + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from yaml import SafeDumper import confluent.discovery.protocols.ssdp as ssdp import eventlet webclient = eventlet.import_patched('pyghmi.util.webclient') @@ -31,7 +37,20 @@ currtzvintage = None def yamldump(input): - return yaml.safe_dump(input, default_flow_style=False) + return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False) + +def yamlload(input): + return yaml.load(input, Loader=SafeLoader) + +def listdump(input): + # special case yaml for flat dumb list + # this is about 25x faster than doing full yaml dump even with CSafeDumper + # with a 17,000 element list + retval = '' + for entry in input: + retval += '- ' + entry + '\n' + return retval + def get_extra_names(nodename, cfg, myip=None): names = set([]) @@ -402,10 +421,13 @@ def handle_request(env, start_response): yield node + '\n' else: start_response('200 OK', (('Content-Type', retype),)) - yield dumper(list(util.natural_sort(nodes))) + if retype == 'application/yaml': + yield listdump(list(util.natural_sort(nodes))) + else: + yield dumper(list(util.natural_sort(nodes))) elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody: try: - reqbody = yaml.safe_load(reqbody) + reqbody = yamlload(reqbody) except Exception: reqbody = None cfgmod = reqbody.get('configmod', 'unspecified') @@ -419,7 +441,7 @@ def handle_request(env, start_response): start_response('200 Ok', ()) yield 'complete' elif env['PATH_INFO'] == '/self/updatestatus' and reqbody: - update = yaml.safe_load(reqbody) + update = yamlload(reqbody) statusstr = update.get('state', None) statusdetail = update.get('state_detail', None) didstateupdate = False @@ -522,7 +544,7 @@ def handle_request(env, start_response): '/var/lib/confluent/public/os/{0}/scripts/{1}') if slist: start_response('200 OK', (('Content-Type', 'application/yaml'),)) - yield yaml.safe_dump(util.natural_sort(slist), default_flow_style=False) + yield yamldump(util.natural_sort(slist)) else: start_response('200 OK', ()) yield '' From 8b150a904765f79fbe99d848a952b8513723f42e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 09:25:57 -0400 Subject: [PATCH 027/319] Fix for post group failures A node failure after group failure would erase the group from range. Further, correct an issue where an empty nodeset would trigger a bad behavior. --- confluent_server/confluent/noderange.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index dcf0a1cb..df4552b8 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -221,22 +221,27 @@ class ReverseNodeRange(object): subsetgroups.sort(key=humanify_nodename) groupchunks = group_elements(subsetgroups) for gc in groupchunks: + if not gc: + continue bracketer = Bracketer(gc[0]) for chnk in gc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: subsetgroups.sort() + ranges.extend(subsetgroups) try: nodes = sorted(self.nodes, key=humanify_nodename) nodechunks = group_elements(nodes) for nc in nodechunks: + if not nc: + continue bracketer = Bracketer(nc[0]) for chnk in nc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: - ranges = sorted(self.nodes) + ranges.extend(sorted(self.nodes)) return ','.join(ranges) From 063bfc17a57cad22e5fbc3f42d29458a5d271790 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 10:40:57 -0400 Subject: [PATCH 028/319] Start using container for final build process Makes supporting the base platform easier by largely ignoring the base platform. --- confluent_osdeploy/buildrpm-aarch64 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/buildrpm-aarch64 b/confluent_osdeploy/buildrpm-aarch64 index 867c0102..7f95852b 100644 --- a/confluent_osdeploy/buildrpm-aarch64 +++ b/confluent_osdeploy/buildrpm-aarch64 @@ -29,4 +29,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/ mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/ rm -rf el9bin rm -rf el8bin -rpmbuild -ba confluent_osdeploy-aarch64.spec +podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec + From 913a26aec93b3c0d55dea1963e3c5a8fe46dae14 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 10:42:39 -0400 Subject: [PATCH 029/319] Change to consistent CWD for osdeploy arm build --- confluent_osdeploy/buildrpm-aarch64 | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_osdeploy/buildrpm-aarch64 b/confluent_osdeploy/buildrpm-aarch64 index 7f95852b..83ffc519 100644 --- a/confluent_osdeploy/buildrpm-aarch64 +++ b/confluent_osdeploy/buildrpm-aarch64 @@ -1,3 +1,4 @@ +cd $(dirname $0) VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then From 9c9d71882c76bbf4bf989b8c8dacced5d7878794 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 15:51:40 -0400 Subject: [PATCH 030/319] Disable keepalive Unfortunately, apache can get a bit odd over how it reports a non-viable open socket for keepalive, which can happen in certain windows. Disable the keepalive feature and take some performance penalty in browsers for the sake of more consistent return behavior and fewer idle greenthreads doing nothing. --- confluent_server/confluent/httpapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index 5a145a0c..f36f2c73 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -1084,7 +1084,7 @@ def serve(bind_host, bind_port): pass # we gave it our best shot there try: eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False, - debug=False, socket_timeout=60) + debug=False, socket_timeout=60, keepalive=False) except TypeError: # Older eventlet in place, skip arguments it does not understand eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False) From ac68f1f22c90a7852459996bd87e07804900d90e Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Fri, 20 Oct 2023 11:38:11 +0200 Subject: [PATCH 031/319] new eaton pdu power readings --- .../plugins/hardwaremanagement/eatonpdu.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py index da60b182..9b5ade5b 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py @@ -156,6 +156,7 @@ def get_sensor_data(element, node, configmanager): gc.logout() _sensors_by_node[node] = [sdata, time.time() + 1] sn = _sensors_by_node.get(node, None) +# print(sn) for outlet in sn[0]: for sensename in sn[0][outlet]: myname = '{0} {1}'.format(outlet, sensename) @@ -277,22 +278,35 @@ class PDUClient(object): if outdata[0] == outlet: return 'on' if outdata[3] else 'off' return + def get_outlet_sensors(self): + rsp = self.do_request('cgi_pdu_outlets') + data = sanitize_json(rsp[0]) + data = json.loads(data) + data = data['data'][0] + + + return data def get_sensor_data(self): rsp = self.do_request1('cgi_overview') - + data = sanitize_json(rsp[0]) data = json.loads(data) + data1 = data['data'][4][0][8] - data = data['data'][0] - sdata = {} - for outdata in data: + data = self.get_outlet_sensors() + sdata = {} + + for outdata in data: + outsense = {} - outletname = outdata[3] + outletname = outdata[0][1] + + outsense['Power'] = { - 'value': outdata[5], - 'units': 'kW', + 'value': outdata[4], + 'units': 'W', 'type': 'Power', } sdata[outletname] = outsense From 62ab361ef80eba4ee3d0930f1bb0b40769047b71 Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Mon, 23 Oct 2023 08:16:27 +0200 Subject: [PATCH 032/319] nicer output for inventory --- .../plugins/hardwaremanagement/geist.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/geist.py b/confluent_server/confluent/plugins/hardwaremanagement/geist.py index a628eb34..06af8675 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/geist.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/geist.py @@ -115,7 +115,8 @@ class GeistClient(object): if len(dbt) != 1: raise Exception('Multiple PDUs not supported per pdu') pdutype = list(dbt)[0] - outlet = dbt[pdutype]['outlet'][ str(int(outlet.join(c for c in outlet if c.isdigit())) - 1) ] + outlet = dbt[pdutype]['outlet'][ str(int(outlet)- 1) ] +# print(outlet) state = outlet['state'].split('2')[-1] return state @@ -128,7 +129,7 @@ class GeistClient(object): self.logout() raise Exception('Multiple PDUs per endpoint not supported') pdu = dbt[list(dbt)[0]]['keyname'] - outlet = int(outlet.join(c for c in outlet if c.isdigit())) - 1 + outlet = int(outlet) - 1 rsp = self.wc.grab_json_response( '/api/dev/{0}/outlet/{1}'.format(pdu, outlet), @@ -237,17 +238,29 @@ def read_inventory(element, node, configmanager): inventory['present'] = True inventory['name'] = 'PDU' for elem in basedata.items(): - if elem[0] !='component' and elem[0] !='locale' and elem[0] !='state' and elem[0] !='contact': # and elem[0] !='name': - _inventory[elem[0]] = elem[1] + + if elem[0] !='component' and elem[0] !='locale' and elem[0] !='state' and elem[0] !='contact' and elem[0] !='appVersion' and elem[0] !='build' and elem[0] !='version' and elem[0] !='apiVersion': + temp = elem[0] + if elem[0] == "serialNumber": + temp = "Serial" + elif elem[0] == "partNumber": + temp = "P/N" + elif elem[0] == "modelNumber": + temp= "Lenovo P/N and Serial" + _inventory[temp] = elem[1] elif elem[0] =='component': tempname = '' for component in basedata['component'].items(): for item in component: if type(item) == str: + tempname = item else: for entry in item.items(): - _inventory[tempname + ' ' + entry[0]] = entry[1] + temp = entry[0] + if temp == 'sn': + temp = "Serial" + _inventory[tempname + ' ' + temp] = entry[1] inventory['information']= _inventory From 49a504972f2b17bce0ab396758b4dd4ac798e1af Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 25 Oct 2023 14:21:55 -0400 Subject: [PATCH 033/319] Fix syntax error in confignet --- confluent_osdeploy/common/profile/scripts/confignet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index f2a2edff..7e641205 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -435,7 +435,7 @@ if __name__ == '__main__': curridx = addr[-1] if curridx in doneidxs: continue - for tries in (1, 2 3): + for tries in (1, 2, 3): try: status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') break @@ -446,7 +446,7 @@ if __name__ == '__main__': continue nc = json.loads(nc) if not dc: - for tries in (1, 2 3): + for tries in (1, 2, 3): try: status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') break From 0857716f64a294b44de4ba883dd552af33800ff5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 26 Oct 2023 08:58:37 -0400 Subject: [PATCH 034/319] Add support for normalized sensors This opens the door for normalized common sensors for clients that care about the semantics but cannot keep track of inconsistent sensor names from implementation to implementation. --- confluent_server/confluent/core.py | 14 ++++++++++++++ .../plugins/hardwaremanagement/ipmi.py | 19 +++++++++++++++++++ .../plugins/hardwaremanagement/redfish.py | 19 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index f70bc6ae..6ab6bd59 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -481,6 +481,20 @@ def _init_core(): 'pluginattrs': ['hardwaremanagement.method'], 'default': 'ipmi', }), + 'normalized': { + 'inlet_temp': PluginRoute({ + 'pluginattrs': ['hardwaremanagement.method'], + 'default': 'ipmi', + }), + 'average_cpu_temp': PluginRoute({ + 'pluginattrs': ['hardwaremanagement.method'], + 'default': 'ipmi', + }), + 'total_power': PluginRoute({ + 'pluginattrs': ['hardwaremanagement.method'], + 'default': 'ipmi', + }), + }, 'energy': PluginCollection({ 'pluginattrs': ['hardwaremanagement.method'], 'default': 'ipmi', diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index 938b69ae..06a8c444 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -861,6 +861,23 @@ class IpmiHandler(object): resourcename = sensor['name'] self.ipmicmd.sensormap[simplify_name(resourcename)] = resourcename + def read_normalized(self, sensorname): + readings = None + if sensorname == 'average_cpu_temp': + cputemp = self.ipmicmd.get_average_processor_temperature() + readings = [cputemp] + elif sensorname == 'inlet_temp': + inltemp = self.ipmicmd.get_inlet_temperature() + readings = [inltemp] + elif sensorname == 'total_power': + sensor = EmptySensor('Total Power') + sensor.states = [] + sensor.units = 'W' + sensor.value = self.ipmicmd.get_system_power_watts() + readings = [sensor] + if readings: + self.output.put(msg.SensorReadings(readings, name=self.node)) + def read_sensors(self, sensorname): if sensorname == 'all': sensors = self.ipmicmd.get_sensor_descriptions() @@ -1157,6 +1174,8 @@ class IpmiHandler(object): if len(self.element) < 3: return self.sensorcategory = self.element[2] + if self.sensorcategory == 'normalized': + return self.read_normalized(self.element[-1]) # list sensors per category if len(self.element) == 3 and self.element[-2] == 'hardware': if self.sensorcategory == 'leds': diff --git a/confluent_server/confluent/plugins/hardwaremanagement/redfish.py b/confluent_server/confluent/plugins/hardwaremanagement/redfish.py index 20315134..f53cc393 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/redfish.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/redfish.py @@ -712,6 +712,23 @@ class IpmiHandler(object): resourcename = sensor['name'] self.sensormap[simplify_name(resourcename)] = resourcename + def read_normalized(self, sensorname): + readings = None + if sensorname == 'average_cpu_temp': + cputemp = self.ipmicmd.get_average_processor_temperature() + readings = [cputemp] + elif sensorname == 'inlet_temp': + inltemp = self.ipmicmd.get_inlet_temperature() + readings = [inltemp] + elif sensorname == 'total_power': + sensor = EmptySensor('Total Power') + sensor.states = [] + sensor.units = 'W' + sensor.value = self.ipmicmd.get_system_power_watts() + readings = [sensor] + if readings: + self.output.put(msg.SensorReadings(readings, name=self.node)) + def read_sensors(self, sensorname): if sensorname == 'all': sensors = self.ipmicmd.get_sensor_descriptions() @@ -1012,6 +1029,8 @@ class IpmiHandler(object): if len(self.element) < 3: return self.sensorcategory = self.element[2] + if self.sensorcategory == 'normalized': + return self.read_normalized(self.element[-1]) # list sensors per category if len(self.element) == 3 and self.element[-2] == 'hardware': if self.sensorcategory == 'leds': From d0826106780c82c3b2c60441b3d3e6c85f256983 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Oct 2023 13:34:52 -0400 Subject: [PATCH 035/319] Add more deep checking of node networking Whether due to the management node or node IP addresses, check if deployment can reasonably proceed using IPv4 or IPv6, and give a warning with some suggestions to check. Also, add nodeinventory -s as an example resolution for missing uuid. --- confluent_server/bin/confluent_selfcheck | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/confluent_server/bin/confluent_selfcheck b/confluent_server/bin/confluent_selfcheck index 1b504e95..cc1409cf 100755 --- a/confluent_server/bin/confluent_selfcheck +++ b/confluent_server/bin/confluent_selfcheck @@ -15,6 +15,7 @@ import confluent.sshutil as sshutil import confluent.certutil as certutil import confluent.client as client import confluent.config.configmanager as configmanager +import confluent.netutil as netutil import eventlet.green.subprocess as subprocess import tempfile import shutil @@ -244,7 +245,7 @@ if __name__ == '__main__': allok = False uuidok = True # not really, but suppress the spurious error dnsdomain = rsp.get('dns.domain', {}).get('value', '') - if ',' in dnsdomain or ' ' in dnsdomain: + if dnsdomain and (',' in dnsdomain or ' ' in dnsdomain): allok = False emprint(f'{args.node} has a dns.domain that appears to be a search instead of singular domain') uuidok = True # not really, but suppress the spurious error @@ -269,9 +270,28 @@ if __name__ == '__main__': switch_value = rsp[key].get('value',None) if switch_value and switch_value not in valid_nodes: emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).') + print(f"Checking network configuration for {args.node}") + cfg = configmanager.ConfigManager(None) + bootablev4nics = [] + bootablev6nics = [] + for nic in glob.glob("/sys/class/net/*/ifindex"): + idx = int(open(nic, "r").read()) + nicname = nic.split('/')[-2] + ncfg = netutil.get_nic_config(cfg, args.node, ifidx=idx) + if ncfg['ipv4_address'] or ncfg['ipv4_method'] == 'dhcp': + bootablev4nics.append(nicname) + if ncfg['ipv6_address']: + bootablev6nics.append(nicname) + if bootablev4nics: + print("{} appears to have network configuration suitable for IPv4 deployment via: {}".format(args.node, ",".join(bootablev4nics))) + elif bootablev6nics: + print('{} appears to have networking configuration suitable for IPv6 deployment via: {}'.format(args.node, ",".join(bootablev6nics))) + else: + emprint(f"{args.node} may not have any viable IP network configuration (check name resolution (DNS or hosts file) " + "and/or net.*ipv4_address, and verify that the deployment serer addresses and subnet mask/prefix length are accurate)") if not uuidok and not macok: allok = False - emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work') + emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work (Example resolution: nodeinventory {args.node} -s)') if allok: print(f'No issues detected with attributes of {args.node}') fprint("Checking name resolution: ") From a1ac234b73173c87a679d1f718cf4b70dd5115da Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 27 Oct 2023 15:31:14 -0400 Subject: [PATCH 036/319] Enhance error message for authentication issue during syncfiles --- confluent_server/confluent/sshutil.py | 16 +++++++++++++--- confluent_server/confluent/syncfiles.py | 2 ++ confluent_server/confluent/util.py | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/sshutil.py b/confluent_server/confluent/sshutil.py index 0a52fe81..d097ade1 100644 --- a/confluent_server/confluent/sshutil.py +++ b/confluent_server/confluent/sshutil.py @@ -129,11 +129,21 @@ def prep_ssh_key(keyname): ap.write('#!/bin/sh\necho $CONFLUENT_SSH_PASSPHRASE\nrm {0}\n'.format(askpass)) os.chmod(askpass, 0o700) os.environ['CONFLUENT_SSH_PASSPHRASE'] = get_passphrase() + olddisplay = os.environ.get('DISPLAY', None) + oldaskpass = os.environ.get('SSH_ASKPASS', None) os.environ['DISPLAY'] = 'NONE' os.environ['SSH_ASKPASS'] = askpass - with open(os.devnull, 'wb') as devnull: - subprocess.check_output(['ssh-add', keyname], stdin=devnull, stderr=devnull) - del os.environ['CONFLUENT_SSH_PASSPHRASE'] + try: + with open(os.devnull, 'wb') as devnull: + subprocess.check_output(['ssh-add', keyname], stdin=devnull, stderr=devnull) + finally: + del os.environ['CONFLUENT_SSH_PASSPHRASE'] + del os.environ['DISPLAY'] + del os.environ['SSH_ASKPASS'] + if olddisplay: + os.environ['DISPLAY'] = olddisplay + if oldaskpass: + os.environ['SSH_ASKPASS'] = oldaskpass ready_keys[keyname] = 1 finally: adding_key = False diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 556d9bcf..6c11d072 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -212,6 +212,8 @@ def sync_list_to_node(sl, node, suffixes, peerip=None): unreadablefiles.append(filename.replace(targdir, '')) if unreadablefiles: raise Exception("Syncing failed due to unreadable files: " + ','.join(unreadablefiles)) + elif b'Permission denied, please try again.' in e.stderr: + raise Exception('Syncing failed due to authentication error, is the confluent automation key not set up (osdeploy initialize -a) or is there some process replacing authorized_keys on the host?') else: raise finally: diff --git a/confluent_server/confluent/util.py b/confluent_server/confluent/util.py index 1509a827..8cf9bbc9 100644 --- a/confluent_server/confluent/util.py +++ b/confluent_server/confluent/util.py @@ -42,7 +42,7 @@ def run(cmd): stdout, stderr = process.communicate() retcode = process.poll() if retcode: - raise subprocess.CalledProcessError(retcode, process.args, output=stdout) + raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) return stdout, stderr From 814f4208529720989842aba900304941c9f09aa9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 31 Oct 2023 11:47:28 -0400 Subject: [PATCH 037/319] Update genesis to deal with CS9 sshd --- .../genesis/initramfs/opt/confluent/bin/rungenesis | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis b/confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis index b7035fe0..ebf0a380 100644 --- a/confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis +++ b/confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis @@ -174,6 +174,8 @@ dnsdomain=${dnsdomain#dnsdomain: } echo search $dnsdomain >> /etc/resolv.conf echo -n "Initializing ssh..." ssh-keygen -A +mkdir -p /usr/share/empty.sshd +rm /etc/ssh/ssh_host_dsa_key* for pubkey in /etc/ssh/ssh_host*key.pub; do certfile=${pubkey/.pub/-cert.pub} privfile=${pubkey%.pub} From 8a4ef0b1fe237fae9c579194d553f3fdebddfcf6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Nov 2023 10:42:33 -0400 Subject: [PATCH 038/319] Make link type detection more specific If the ip command shows altnames, do not let the altnames interfere with locking on to linktype. Further, use show dev instead of grep to be more specific. --- .../usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 2 +- .../usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index b2881e0b..65abf8f6 100644 --- a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -189,7 +189,7 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC EOC echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection -linktype=$(ip link |grep -A2 ${ifname}|tail -n 1|awk '{print $1}') +linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}') if [ "$linktype" = link/infiniband ]; then linktype="infiniband" else diff --git a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index 4fca92cf..a9eba388 100644 --- a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -154,7 +154,7 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC EOC echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection -linktype=$(ip link |grep -A2 ${ifname}|tail -n 1|awk '{print $1}') +linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}') if [ "$linktype" = link/infiniband ]; then linktype="infiniband" else From 8f927d94e9b9f29d62c3228e43cd2533d8ae467e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 1 Nov 2023 11:17:04 -0400 Subject: [PATCH 039/319] Switch to bond from team Teaming is deprecated and EL went back to bond, follow that guidance. --- confluent_osdeploy/common/profile/scripts/confignet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 7e641205..4e9fe9b3 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -344,7 +344,7 @@ class NetworkManager(object): bondcfg[stg] = deats[stg] if member in self.uuidbyname: subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]]) - subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team-slave', 'master', team, 'con-name', member, 'connection.interface-name', member]) + subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond-slave', 'master', team, 'con-name', member, 'connection.interface-name', member]) if bondcfg: args = [] for parm in bondcfg: @@ -378,7 +378,7 @@ class NetworkManager(object): for arg in cmdargs: cargs.append(arg) cargs.append(cmdargs[arg]) - subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team', 'con-name', cname, 'connection.interface-name', cname, 'team.runner', stgs['team_mode']] + cargs) + subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs) for iface in cfg['interfaces']: self.add_team_member(cname, iface) subprocess.check_call(['nmcli', 'c', 'u', cname]) From e90f2829abade00ee0cd84bb780d7fac912ed383 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 8 Nov 2023 09:37:44 -0500 Subject: [PATCH 040/319] Filter bind mounts from imgutil capture If bind mounts are in use, it will foul the capture. Notably, one example is if you install the firefox snap in ubuntu, snapd creates a bind mount. This will ignore bind mounts, and rely upon the system to put it straight. --- imgutil/imgutil | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index de3a9025..959c4a17 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -61,13 +61,27 @@ FALLOC_FL_PUNCH_HOLE = 2 numregex = re.compile('([0-9]+)') def get_partition_info(): + with open('/proc/self/mountinfo') as procinfo: + mountinfo = procinfo.read() + capmounts = set([]) + for entry in mountinfo.split('\n'): + if not entry: + continue + firstinf, lastinf = entry.split(' - ') + root, mount = firstinf.split()[3:5] + filesystem = lastinf.split()[0] + if root != '/': + continue + if filesystem not in ('ext3', 'ext4', 'xfs', 'btrfs', 'vfat'): + continue + capmounts.add(mount) with open('/proc/mounts') as procmounts: mountinfo = procmounts.read() for entry in mountinfo.split('\n'): if not entry: continue dev, mount, fs, flags = entry.split()[:4] - if fs not in ('ext3', 'ext4', 'xfs', 'btrfs', 'vfat'): + if mount not in capmounts: continue fsinfo = os.statvfs(mount) partinfo = { From 2cd75ef4252f2ceb5b70ef38e9be19f60f3602db Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 8 Nov 2023 10:22:51 -0500 Subject: [PATCH 041/319] Fix diskless autocons message --- .../ubuntu20.04-diskless/initramfs/conf/conf.d/confluent | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent b/confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent index 64a3713d..79787074 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent +++ b/confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent @@ -1,4 +1,5 @@ if ! grep console= /proc/cmdline > /dev/null; then + mkdir -p /custom-installation /opt/confluent/bin/autocons > /custom-installation/autocons.info cons=$(cat /custom-installation/autocons.info) if [ ! -z "$cons" ]; then From e03f010eac0f1afaf372453ed8bb1d2caaefb6a0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 9 Nov 2023 09:03:59 -0500 Subject: [PATCH 042/319] Fix Ubuntu confignet without IPv6 If confignet did not have ipv6 to work with, it would fail to work at all. Also handle when the configuration has a blank DNS server in it. --- confluent_osdeploy/common/profile/scripts/confignet | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 4e9fe9b3..eaaf2621 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -151,13 +151,14 @@ class NetplanManager(object): needcfgapply = False for devname in devnames: needcfgwrite = False - if stgs['ipv6_method'] == 'static': - curraddr = stgs['ipv6_address'] + # ipv6_method missing at uconn... + if stgs.get('ipv6_method', None) == 'static': + curraddr = stgs'ipv6_address'] currips = self.getcfgarrpath([devname, 'addresses']) if curraddr not in currips: needcfgwrite = True currips.append(curraddr) - if stgs['ipv4_method'] == 'static': + if stgs.get('ipv4_method', None) == 'static': curraddr = stgs['ipv4_address'] currips = self.getcfgarrpath([devname, 'addresses']) if curraddr not in currips: @@ -180,7 +181,7 @@ class NetplanManager(object): if dnsips: currdnsips = self.getcfgarrpath([devname, 'nameservers', 'addresses']) for dnsip in dnsips: - if dnsip not in currdnsips: + if dnsip and dnsip not in currdnsips: needcfgwrite = True currdnsips.append(dnsip) if dnsdomain: From 6e092934e7ec4222a175e5bc473cb3a7ae8d4db8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 9 Nov 2023 17:15:17 -0500 Subject: [PATCH 043/319] Fix for ubuntu clone to nvme --- .../ubuntu20.04-diskless/profiles/default/scripts/image2disk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py index 5d15e3d4..1d19ebad 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py @@ -206,6 +206,8 @@ def fixup(rootdir, vols): partnum = re.search('(\d+)$', targdev).group(1) targblock = re.search('(.*)\d+$', targdev).group(1) if targblock: + if targblock.endswith('p') and 'nvme' in targblock: + targblock = targblock[:-1] shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip() shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\') subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum]) From ec023831a5d322afb86ca73938a7a79c1fa54ddb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 9 Nov 2023 17:28:38 -0500 Subject: [PATCH 044/319] Fix syntax error in confignet --- confluent_osdeploy/common/profile/scripts/confignet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index eaaf2621..cb2569ce 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -153,7 +153,7 @@ class NetplanManager(object): needcfgwrite = False # ipv6_method missing at uconn... if stgs.get('ipv6_method', None) == 'static': - curraddr = stgs'ipv6_address'] + curraddr = stgs['ipv6_address'] currips = self.getcfgarrpath([devname, 'addresses']) if curraddr not in currips: needcfgwrite = True From f475d589559627f0222022b003dd03496028ea88 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 13 Nov 2023 15:43:11 -0500 Subject: [PATCH 045/319] Various permission fixes for osdeploy initialize Fix a few scenarios where certain ordering of initialize creates unworkable permissions. --- confluent_server/bin/osdeploy | 28 ++++++++++++++++--- confluent_server/confluent/certutil.py | 38 ++++++++++++++------------ confluent_server/confluent/sshutil.py | 13 +++++---- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/confluent_server/bin/osdeploy b/confluent_server/bin/osdeploy index ed39e78c..ef6859e3 100644 --- a/confluent_server/bin/osdeploy +++ b/confluent_server/bin/osdeploy @@ -373,9 +373,14 @@ def initialize(cmdset): for rsp in c.read('/uuid'): uuid = rsp.get('uuid', {}).get('value', None) if uuid: - with open('confluent_uuid', 'w') as uuidout: - uuidout.write(uuid) - uuidout.write('\n') + oum = os.umask(0o11) + try: + with open('confluent_uuid', 'w') as uuidout: + uuidout.write(uuid) + uuidout.write('\n') + os.chmod('confluent_uuid', 0o644) + finally: + os.umask(oum) totar.append('confluent_uuid') topack.append('confluent_uuid') if os.path.exists('ssh'): @@ -403,7 +408,17 @@ def initialize(cmdset): if res: sys.stderr.write('Error occurred while packing site initramfs') sys.exit(1) - os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio') + oum = os.umask(0o22) + try: + os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio') + os.chown('/var/lib/confluent/public/site/initramfs.cpio', 0o644) + finally: + os.umask(oum) + oum = os.umask(0o22) + try: + os.chown('/var/lib/confluent/public/site/initramfs.cpio', 0o644) + finally: + os.umask(oum) if cmdset.g: updateboot('genesis-x86_64') if totar: @@ -411,6 +426,11 @@ def initialize(cmdset): tarcmd = ['tar', '-czf', tmptarname] + totar subprocess.check_call(tarcmd) os.rename(tmptarname, '/var/lib/confluent/public/site/initramfs.tgz') + oum = os.umask(0o22) + try: + os.chown('/var/lib/confluent/public/site/initramfs.tgz', 0o644) + finally: + os.umask(0o22) os.chdir(opath) print('Site initramfs content packed successfully') diff --git a/confluent_server/confluent/certutil.py b/confluent_server/confluent/certutil.py index dffaf85e..2e788bad 100644 --- a/confluent_server/confluent/certutil.py +++ b/confluent_server/confluent/certutil.py @@ -95,27 +95,29 @@ def assure_tls_ca(): os.makedirs(os.path.dirname(fname)) except OSError as e: if e.errno != 17: + os.seteuid(ouid) raise + try: + shutil.copy2('/etc/confluent/tls/cacert.pem', fname) + hv, _ = util.run( + ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout']) + if not isinstance(hv, str): + hv = hv.decode('utf8') + hv = hv.strip() + hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv) + certname = '{0}.pem'.format(collective.get_myname()) + for currname in os.listdir('/var/lib/confluent/public/site/tls/'): + currname = os.path.join('/var/lib/confluent/public/site/tls/', currname) + if currname.endswith('.0'): + try: + realname = os.readlink(currname) + if realname == certname: + os.unlink(currname) + except OSError: + pass + os.symlink(certname, hashname) finally: os.seteuid(ouid) - shutil.copy2('/etc/confluent/tls/cacert.pem', fname) - hv, _ = util.run( - ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout']) - if not isinstance(hv, str): - hv = hv.decode('utf8') - hv = hv.strip() - hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv) - certname = '{0}.pem'.format(collective.get_myname()) - for currname in os.listdir('/var/lib/confluent/public/site/tls/'): - currname = os.path.join('/var/lib/confluent/public/site/tls/', currname) - if currname.endswith('.0'): - try: - realname = os.readlink(currname) - if realname == certname: - os.unlink(currname) - except OSError: - pass - os.symlink(certname, hashname) def substitute_cfg(setting, key, val, newval, cfgfile, line): if key.strip() == setting: diff --git a/confluent_server/confluent/sshutil.py b/confluent_server/confluent/sshutil.py index d097ade1..16e4db7e 100644 --- a/confluent_server/confluent/sshutil.py +++ b/confluent_server/confluent/sshutil.py @@ -98,14 +98,15 @@ def initialize_ca(): preexec_fn=normalize_uid) ouid = normalize_uid() try: - os.makedirs('/var/lib/confluent/public/site/ssh/', mode=0o755) - except OSError as e: - if e.errno != 17: - raise + try: + os.makedirs('/var/lib/confluent/public/site/ssh/', mode=0o755) + except OSError as e: + if e.errno != 17: + raise + cafilename = '/var/lib/confluent/public/site/ssh/{0}.ca'.format(myname) + shutil.copy('/etc/confluent/ssh/ca.pub', cafilename) finally: os.seteuid(ouid) - cafilename = '/var/lib/confluent/public/site/ssh/{0}.ca'.format(myname) - shutil.copy('/etc/confluent/ssh/ca.pub', cafilename) # newent = '@cert-authority * ' + capub.read() From cd07e0e212a8c526074a24ce0487e100e7dc1221 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 14 Nov 2023 15:14:54 -0500 Subject: [PATCH 046/319] Add missing disclaimer from tmt license --- confluent_vtbufferd/NOTICE | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/confluent_vtbufferd/NOTICE b/confluent_vtbufferd/NOTICE index 95b86a82..da174e81 100644 --- a/confluent_vtbufferd/NOTICE +++ b/confluent_vtbufferd/NOTICE @@ -22,3 +22,16 @@ modification, are permitted provided that the following conditions are met: * Neither the name of the copyright holder nor the names of contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS, +* COPYRIGHT HOLDERS, OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + From c9452e65e8f35916adab7cb7257ca02e537beda5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 15 Nov 2023 11:30:20 -0500 Subject: [PATCH 047/319] Fix some osdeploy ordering issues osdeploy initialization dependencies have been improved and marked if absolutely dependent. --- confluent_server/bin/osdeploy | 46 +++++++++++++++++---------- confluent_server/confluent/sshutil.py | 8 +++++ confluent_server/confluent/util.py | 4 +-- 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/confluent_server/bin/osdeploy b/confluent_server/bin/osdeploy index ef6859e3..fff220be 100644 --- a/confluent_server/bin/osdeploy +++ b/confluent_server/bin/osdeploy @@ -72,6 +72,12 @@ def main(args): return rebase(cmdset.profile) ap.print_help() +def symlinkp(src, trg): + try: + os.symlink(src, trg) + except Exception as e: + if e.errno != 17: + raise def initialize_genesis(): if not os.path.exists('/opt/confluent/genesis/x86_64/boot/kernel'): @@ -89,30 +95,33 @@ def initialize_genesis(): return retval[1] retcode = 0 try: + util.mkdirp('/var/lib/confluent', 0o755) if hasconfluentuser: + os.chown('/var/lib/confluent', hasconfluentuser.pw_uid, -1) os.setgid(hasconfluentuser.pw_gid) os.setuid(hasconfluentuser.pw_uid) os.umask(0o22) - os.makedirs('/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot', 0o755) - os.makedirs('/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs', 0o755) - os.symlink('/opt/confluent/genesis/x86_64/boot/efi/boot/BOOTX64.EFI', + util.mkdirp('/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot', 0o755) + util.mkdirp('/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs', 0o755) + symlinkp('/opt/confluent/genesis/x86_64/boot/efi/boot/BOOTX64.EFI', '/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot/BOOTX64.EFI') - os.symlink('/opt/confluent/genesis/x86_64/boot/efi/boot/grubx64.efi', + symlinkp('/opt/confluent/genesis/x86_64/boot/efi/boot/grubx64.efi', '/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot/grubx64.efi') - os.symlink('/opt/confluent/genesis/x86_64/boot/initramfs/distribution', + symlinkp('/opt/confluent/genesis/x86_64/boot/initramfs/distribution', '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/distribution') - os.symlink('/var/lib/confluent/public/site/initramfs.cpio', + symlinkp('/var/lib/confluent/public/site/initramfs.cpio', '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/site.cpio') - os.symlink('/opt/confluent/lib/osdeploy/genesis/initramfs/addons.cpio', + symlinkp('/opt/confluent/lib/osdeploy/genesis/initramfs/addons.cpio', '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/addons.cpio') - os.symlink('/opt/confluent/genesis/x86_64/boot/kernel', + symlinkp('/opt/confluent/genesis/x86_64/boot/kernel', '/var/lib/confluent/public/os/genesis-x86_64/boot/kernel') - shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/', - '/var/lib/confluent/public/os/genesis-x86_64/ansible/') - shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/', - '/var/lib/confluent/public/os/genesis-x86_64/scripts/') - shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml', - '/var/lib/confluent/public/os/genesis-x86_64/profile.yaml') + if not os.path.exists('/var/lib/confluent/public/os/genesis-x86_64/ansible/'): + shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/', + '/var/lib/confluent/public/os/genesis-x86_64/ansible/') + shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/', + '/var/lib/confluent/public/os/genesis-x86_64/scripts/') + shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml', + '/var/lib/confluent/public/os/genesis-x86_64/profile.yaml') except Exception as e: sys.stderr.write(str(e) + '\n') retcode = 1 @@ -411,12 +420,12 @@ def initialize(cmdset): oum = os.umask(0o22) try: os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio') - os.chown('/var/lib/confluent/public/site/initramfs.cpio', 0o644) + os.chmod('/var/lib/confluent/public/site/initramfs.cpio', 0o644) finally: os.umask(oum) oum = os.umask(0o22) try: - os.chown('/var/lib/confluent/public/site/initramfs.cpio', 0o644) + os.chmod('/var/lib/confluent/public/site/initramfs.cpio', 0o644) finally: os.umask(oum) if cmdset.g: @@ -428,7 +437,7 @@ def initialize(cmdset): os.rename(tmptarname, '/var/lib/confluent/public/site/initramfs.tgz') oum = os.umask(0o22) try: - os.chown('/var/lib/confluent/public/site/initramfs.tgz', 0o644) + os.chmod('/var/lib/confluent/public/site/initramfs.tgz', 0o644) finally: os.umask(0o22) os.chdir(opath) @@ -441,6 +450,9 @@ def initialize(cmdset): def updateboot(profilename): + if not os.path.exists('/var/lib/confluent/public/site/initramfs.cpio'): + emprint('Must generate site content first (TLS (-t) and/or SSH (-s))') + return 1 c = client.Command() for rsp in c.update('/deployment/profiles/{0}'.format(profilename), {'updateboot': 1}): diff --git a/confluent_server/confluent/sshutil.py b/confluent_server/confluent/sshutil.py index 16e4db7e..cf17f37a 100644 --- a/confluent_server/confluent/sshutil.py +++ b/confluent_server/confluent/sshutil.py @@ -186,6 +186,14 @@ def initialize_root_key(generate, automation=False): if os.path.exists('/etc/confluent/ssh/automation'): alreadyexist = True else: + ouid = normalize_uid() + try: + os.makedirs('/etc/confluent/ssh', mode=0o700) + except OSError as e: + if e.errno != 17: + raise + finally: + os.seteuid(ouid) subprocess.check_call( ['ssh-keygen', '-t', 'ed25519', '-f','/etc/confluent/ssh/automation', '-N', get_passphrase(), diff --git a/confluent_server/confluent/util.py b/confluent_server/confluent/util.py index 8cf9bbc9..96d2291b 100644 --- a/confluent_server/confluent/util.py +++ b/confluent_server/confluent/util.py @@ -29,9 +29,9 @@ import struct import eventlet.green.subprocess as subprocess -def mkdirp(path): +def mkdirp(path, mode=0o777): try: - os.makedirs(path) + os.makedirs(path, mode) except OSError as e: if e.errno != 17: raise From 9757cd1ae32e343f6eddcd47067f2440a1c070d4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 16 Nov 2023 10:17:55 -0500 Subject: [PATCH 048/319] Check the profile *before* rebooting systems This provides a much better experience when a typo or other mistake has a profile that is not actionable. --- confluent_client/bin/nodedeploy | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/confluent_client/bin/nodedeploy b/confluent_client/bin/nodedeploy index 2417f2c5..52e3a7d9 100755 --- a/confluent_client/bin/nodedeploy +++ b/confluent_client/bin/nodedeploy @@ -90,17 +90,6 @@ def main(args): if 'error' in rsp: sys.stderr.write(rsp['error'] + '\n') sys.exit(1) - if not args.clear and args.network and not args.prepareonly: - rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network', - bootmode='uefi', - persistent=False, - errnodes=errnodes) - if errnodes: - sys.stderr.write( - 'Unable to set boot device for following nodes: {0}\n'.format( - ','.join(errnodes))) - return 1 - rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot') if args.clear: cleararm(args.noderange, c) clearpending(args.noderange, c) @@ -120,7 +109,7 @@ def main(args): for profname in profnames: sys.stderr.write(' ' + profname + '\n') else: - sys.stderr.write('No deployment profiles available, try osdeploy fiimport or imgutil capture\n') + sys.stderr.write('No deployment profiles available, try osdeploy import or imgutil capture\n') sys.exit(1) armonce(args.noderange, c) setpending(args.noderange, args.profile, c) @@ -166,6 +155,17 @@ def main(args): else: print('{0}: {1}{2}'.format(node, profile, armed)) sys.exit(0) + if not args.clear and args.network and not args.prepareonly: + rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network', + bootmode='uefi', + persistent=False, + errnodes=errnodes) + if errnodes: + sys.stderr.write( + 'Unable to set boot device for following nodes: {0}\n'.format( + ','.join(errnodes))) + return 1 + rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot') if args.network and not args.prepareonly: return rc return 0 From 68ce3d039d7b26125222d2bd51e5d911b0ea14fe Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 27 Nov 2023 08:34:34 -0500 Subject: [PATCH 049/319] Filter out nvme 'c' devnames, that are used to refer to paths to nvme Some versions start manifesting nvme devnames with 'c', which are to be used to interact with multipath to have raw devices backing a traditional nvme device. --- .../el7-diskless/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk | 2 ++ .../el8-diskless/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk | 2 ++ .../el9-diskless/profiles/default/scripts/getinstalldisk | 2 ++ .../rhvh4/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk | 2 ++ .../suse15/profiles/server/scripts/getinstalldisk | 2 ++ .../profiles/default/scripts/getinstalldisk | 2 ++ .../ubuntu20.04/profiles/default/scripts/getinstalldisk | 2 ++ .../ubuntu22.04/profiles/default/scripts/getinstalldisk | 2 ++ 11 files changed, 22 insertions(+) diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk +++ b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None From 0b28d64c83f439412fcbf4c02b6aae84657b52c3 Mon Sep 17 00:00:00 2001 From: Christian Goll Date: Mon, 27 Nov 2023 15:00:27 +0100 Subject: [PATCH 050/319] python3-dbm is required for SUSE the python module `anydbm` is part of this python package Signed-off-by: Christian Goll --- confluent_server/confluent_server.spec.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index c7e2aa3a..51046a8f 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -22,7 +22,7 @@ Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3- %if "%{dist}" == ".el9" Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute %else -Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodome >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dnspython, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-PyYAML openssl iproute +Requires: python3-dbm,python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodome >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dnspython, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-PyYAML openssl iproute %endif %endif %endif From 3730ba049f97b4d007b538b058055fd93e0aa8a4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Nov 2023 13:11:25 -0500 Subject: [PATCH 051/319] Fix potential doubling up of IPv6 brackets There were scenarios where IPv6 URL brackets may double up. --- .../usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 4 ++-- .../usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 4 ++-- .../lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index 65abf8f6..cdcc12fd 100644 --- a/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -155,7 +155,7 @@ fi ready=0 while [ $ready = "0" ]; do get_remote_apikey - if [[ $confluent_mgr == *:* ]]; then + if [[ $confluent_mgr == *:* ]] && [[ $confluent_mgr != "["* ]]; then confluent_mgr="[$confluent_mgr]" fi tmperr=$(mktemp) @@ -324,7 +324,7 @@ fi echo '[proxy]' >> /run/NetworkManager/system-connections/$ifname.nmconnection chmod 600 /run/NetworkManager/system-connections/*.nmconnection confluent_websrv=$confluent_mgr -if [[ $confluent_websrv == *:* ]]; then +if [[ $confluent_websrv == *:* ]] && [[ $confluent_websrv != "["* ]]; then confluent_websrv="[$confluent_websrv]" fi echo -n "Initializing ssh..." diff --git a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index a9eba388..a4f10ee2 100644 --- a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -120,7 +120,7 @@ fi ready=0 while [ $ready = "0" ]; do get_remote_apikey - if [[ $confluent_mgr == *:* ]]; then + if [[ $confluent_mgr == *:* ]] && [[ $confluent_mgr != "["* ]]; then confluent_mgr="[$confluent_mgr]" fi tmperr=$(mktemp) @@ -281,7 +281,7 @@ fi echo '[proxy]' >> /run/NetworkManager/system-connections/$ifname.nmconnection chmod 600 /run/NetworkManager/system-connections/*.nmconnection confluent_websrv=$confluent_mgr -if [[ $confluent_websrv == *:* ]]; then +if [[ $confluent_websrv == *:* ]] && [[ $confluent_websrv != "["* ]]; then confluent_websrv="[$confluent_websrv]" fi echo -n "Initializing ssh..." diff --git a/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index 146c4797..5586978c 100644 --- a/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -116,7 +116,7 @@ fi ready=0 while [ $ready = "0" ]; do get_remote_apikey - if [[ $confluent_mgr == *:* ]]; then + if [[ $confluent_mgr == *:* ]] && [[ $confluent_mgr != "["* ]]; then confluent_mgr="[$confluent_mgr]" fi tmperr=$(mktemp) From 55e60d52fd0c693b831df0983938d85f4e0b6a33 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Nov 2023 13:33:21 -0500 Subject: [PATCH 052/319] Avoid potential multiple brackets in imageboot.sh --- .../el8-diskless/profiles/default/scripts/imageboot.sh | 2 +- .../el9-diskless/profiles/default/scripts/imageboot.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh index ee2a8125..fe53bf38 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh @@ -1,6 +1,6 @@ . /lib/dracut-lib.sh confluent_whost=$confluent_mgr -if [[ "$confluent_whost" == *:* ]]; then +if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then confluent_whost="[$confluent_mgr]" fi mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh index ee2a8125..fe53bf38 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh @@ -1,6 +1,6 @@ . /lib/dracut-lib.sh confluent_whost=$confluent_mgr -if [[ "$confluent_whost" == *:* ]]; then +if [[ "$confluent_whost" == *:* ]] && [[ "$confluent_whost" != "["* ]]; then confluent_whost="[$confluent_mgr]" fi mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay From 63b737dc52551b43e2f000d53b5b6d940cb3264d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Nov 2023 14:09:59 -0500 Subject: [PATCH 053/319] Correct bonding in confignet for NetworkManager --- confluent_osdeploy/common/profile/scripts/confignet | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index cb2569ce..f19e620c 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -379,6 +379,8 @@ class NetworkManager(object): for arg in cmdargs: cargs.append(arg) cargs.append(cmdargs[arg]) + if stgs['team_mode'] == 'lacp': + stgs['team_mode'] = '802.3ad' subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs) for iface in cfg['interfaces']: self.add_team_member(cname, iface) From 6763c863879368371b6c134f549e52c6dafff998 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 28 Nov 2023 14:35:01 -0500 Subject: [PATCH 054/319] Add DNS to NetworkManager Similar to netplan, apply DNS to every interface. --- .../common/profile/scripts/confignet | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index f19e620c..44eb32ed 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -295,7 +295,8 @@ class WickedManager(object): class NetworkManager(object): - def __init__(self, devtypes): + def __init__(self, devtypes, deploycfg): + self.deploycfg = deploycfg self.connections = {} self.uuidbyname = {} self.uuidbydev = {} @@ -367,6 +368,20 @@ class NetworkManager(object): cmdargs['ipv4.gateway'] = stgs['ipv4_gateway'] if stgs.get('ipv6_gateway', None): cmdargs['ipv6.gateway'] = stgs['ipv6_gateway'] + dnsips = self.deploycfg.get('nameservers', []) + if not dnsips: + dnsips = [] + dns4 = [] + dns6 = [] + for dnsip in dnsips: + if '.' in dnsip: + dns4.append(dnsip) + elif ':' in dnsip: + dns6.append(dnsip) + if dns4: + cmdargs['ipv4.dns'] = dns4.join(',') + if dns6: + cmdargs['ipv6.dns'] = dns6.join(',') if len(cfg['interfaces']) > 1: # team time.. should be.. if not cfg['settings'].get('team_mode', None): sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces']))) @@ -487,7 +502,7 @@ if __name__ == '__main__': if os.path.exists('/usr/sbin/netplan'): nm = NetplanManager(dc) if os.path.exists('/usr/bin/nmcli'): - nm = NetworkManager(devtypes) + nm = NetworkManager(devtypes, dc) elif os.path.exists('/usr/sbin/wicked'): nm = WickedManager() for netn in netname_to_interfaces: From 3aa91b61e5befc32e0ebe851b986c84d34634053 Mon Sep 17 00:00:00 2001 From: Christian Goll Date: Fri, 1 Dec 2023 10:57:31 +0100 Subject: [PATCH 055/319] disable online repos for openSUSE leap online repositories may not be accesible for the cluster nodes but were added from the content.xml. Editing this files with initprofile.sh is impossible as they are executed in parallel, so all repos starting with https?://download.opensuse.org are removed during post Signed-off-by: Christian Goll --- confluent_osdeploy/suse15/profiles/hpc/autoyast.leap | 6 ++++++ confluent_osdeploy/suse15/profiles/hpc/initprofile.sh | 3 +++ .../profiles/hpc/scripts/post.d/10-remove-online-repos.sh | 3 +++ confluent_osdeploy/suse15/profiles/server/autoyast.leap | 6 ++++++ confluent_osdeploy/suse15/profiles/server/initprofile.sh | 3 +++ .../server/scripts/post.d/10-remove-online-repos.sh | 3 +++ 6 files changed, 24 insertions(+) create mode 100644 confluent_osdeploy/suse15/profiles/hpc/scripts/post.d/10-remove-online-repos.sh create mode 100644 confluent_osdeploy/suse15/profiles/server/scripts/post.d/10-remove-online-repos.sh diff --git a/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap b/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap index 7f9d08f7..e92ec9fd 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap +++ b/confluent_osdeploy/suse15/profiles/hpc/autoyast.leap @@ -10,6 +10,12 @@ dynamic behavior and replace with static configuration. UTC %%TIMEZONE%% + + false + + + false + false diff --git a/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh b/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh index 9c6c295e..62a2663e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh +++ b/confluent_osdeploy/suse15/profiles/hpc/initprofile.sh @@ -1,4 +1,7 @@ #!/bin/sh +# WARNING +# be careful when editing files here as this script is called +# in parallel to other copy operations, so changes to files can be lost discnum=$(basename $1) if [ "$discnum" != 1 ]; then exit 0; fi if [ -e $2/boot/kernel ]; then exit 0; fi diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/post.d/10-remove-online-repos.sh b/confluent_osdeploy/suse15/profiles/hpc/scripts/post.d/10-remove-online-repos.sh new file mode 100644 index 00000000..9ae8224e --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/post.d/10-remove-online-repos.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash +# remove online repos +grep -lE "baseurl=https?://download.opensuse.org" /etc/zypp/repos.d/*repo | xargs rm -- diff --git a/confluent_osdeploy/suse15/profiles/server/autoyast.leap b/confluent_osdeploy/suse15/profiles/server/autoyast.leap index 7f9d08f7..e92ec9fd 100644 --- a/confluent_osdeploy/suse15/profiles/server/autoyast.leap +++ b/confluent_osdeploy/suse15/profiles/server/autoyast.leap @@ -10,6 +10,12 @@ dynamic behavior and replace with static configuration. UTC %%TIMEZONE%% + + false + + + false + false diff --git a/confluent_osdeploy/suse15/profiles/server/initprofile.sh b/confluent_osdeploy/suse15/profiles/server/initprofile.sh index 9c6c295e..62a2663e 100644 --- a/confluent_osdeploy/suse15/profiles/server/initprofile.sh +++ b/confluent_osdeploy/suse15/profiles/server/initprofile.sh @@ -1,4 +1,7 @@ #!/bin/sh +# WARNING +# be careful when editing files here as this script is called +# in parallel to other copy operations, so changes to files can be lost discnum=$(basename $1) if [ "$discnum" != 1 ]; then exit 0; fi if [ -e $2/boot/kernel ]; then exit 0; fi diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/post.d/10-remove-online-repos.sh b/confluent_osdeploy/suse15/profiles/server/scripts/post.d/10-remove-online-repos.sh new file mode 100644 index 00000000..9ae8224e --- /dev/null +++ b/confluent_osdeploy/suse15/profiles/server/scripts/post.d/10-remove-online-repos.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash +# remove online repos +grep -lE "baseurl=https?://download.opensuse.org" /etc/zypp/repos.d/*repo | xargs rm -- From 7b89054b35e63728755c51035ef86338ded391a1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 1 Dec 2023 15:55:17 -0500 Subject: [PATCH 056/319] Fix a few noderange abbreviations Also, add some test cases on abbreviation to help sanity check things in the future. --- confluent_server/confluent/noderange.py | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index df4552b8..cf99dd72 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -96,6 +96,7 @@ class Bracketer(object): txtnums = getnumbers_nodename(nodename) nums = [int(x) for x in txtnums] for n in range(self.count): + # First pass to see if we have exactly one different number padto = len(txtnums[n]) needpad = (padto != len('{}'.format(nums[n]))) if self.sequences[n] is None: @@ -105,7 +106,24 @@ class Bracketer(object): elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: continue # new nodename has no new number, keep going else: # if self.sequences[n][2] != nums[n] or : - if self.diffn is not None and (n != self.diffn or + if self.diffn is not None and (n != self.diffn or + (padto < self.numlens[n][1]) or + (needpad and padto != self.numlens[n][1])): + self.flush_current() + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [padto, padto] + self.diffn = n + for n in range(self.count): + padto = len(txtnums[n]) + needpad = (padto != len('{}'.format(nums[n]))) + if self.sequences[n] is None: + # We initialize to text pieces, 'currstart', and 'prev' number + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] + elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: + continue # new nodename has no new number, keep going + else: # if self.sequences[n][2] != nums[n] or : + if self.diffn is not None and (n != self.diffn or (padto < self.numlens[n][1]) or (needpad and padto != self.numlens[n][1])): self.flush_current() @@ -449,3 +467,29 @@ class NodeRange(object): if self.cfm is None: return set([element]) raise Exception(element + ' not a recognized node, group, or alias') + +if __name__ == '__main__': + cases = [ + (['r3u4', 'r5u6'], 'r3u4,r5u6'), # should not erroneously gather + (['r3u4s1', 'r5u6s3'], 'r3u4s1,r5u6s3'), # should not erroneously gather + (['r3u4s1', 'r3u4s2', 'r5u4s3'], 'r3u4s[1:2],r5u4s3'), # should not erroneously gather + (['r3u4', 'r3u5', 'r3u6', 'r3u9', 'r4u1'], 'r3u[4:6,9],r4u1'), + (['n01', 'n2', 'n03'], 'n01,n2,n03'), + (['n7', 'n8', 'n09', 'n10', 'n11', 'n12', 'n13', 'n14', 'n15', 'n16', + 'n17', 'n18', 'n19', 'n20'], 'n[7:8],n[09:20]') + ] + for case in cases: + gc = case[0] + bracketer = Bracketer(gc[0]) + for chnk in gc[1:]: + bracketer.extend(chnk) + br = bracketer.range + resnodes = NodeRange(br).nodes + if set(resnodes) != set(gc): + print('FAILED: ' + repr(sorted(gc))) + print('RESULT: ' + repr(sorted(resnodes))) + print('EXPECTED: ' + repr(case[1])) + print('ACTUAL: ' + br) + + + From 7aef012a42f6859df8e8bbece7f208dc568935d8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 5 Dec 2023 14:39:36 -0500 Subject: [PATCH 057/319] Correct string join syntax in confignet --- confluent_osdeploy/common/profile/scripts/confignet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 44eb32ed..8cda6c83 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -379,9 +379,9 @@ class NetworkManager(object): elif ':' in dnsip: dns6.append(dnsip) if dns4: - cmdargs['ipv4.dns'] = dns4.join(',') + cmdargs['ipv4.dns'] = ','.join(dns4) if dns6: - cmdargs['ipv6.dns'] = dns6.join(',') + cmdargs['ipv6.dns'] = ','.join(dns6) if len(cfg['interfaces']) > 1: # team time.. should be.. if not cfg['settings'].get('team_mode', None): sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces']))) From 93269a05ebb66c4b97ab484402ce55808fda2101 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 Dec 2023 17:06:09 -0500 Subject: [PATCH 058/319] Fix cloning with ipv6 and EL9 --- .../profiles/default/scripts/firstboot.sh | 11 +++++++++-- .../el9-diskless/profiles/default/scripts/post.sh | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh index 2bab4136..ed11d9e7 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh @@ -9,9 +9,16 @@ HOME=$(getent passwd $(whoami)|cut -d: -f 6) export HOME nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') confluent_apikey=$(cat /etc/confluent/confluent.apikey) -confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') +confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}') +if [ -z "$confluent_mgr" ] || [ "$confluent_mgr" == "null" ] || ! ping -c 1 $confluent_mgr >& /dev/null; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') +fi +confluent_websrv=$confluent_mgr +if [[ "$confluent_mgr" == *:* ]]; then + confluent_websrv="[$confluent_mgr]" +fi confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') -export nodename confluent_mgr confluent_profile +export nodename confluent_mgr confluent_profile confluent_websrv . /etc/confluent/functions ( exec >> /var/log/confluent/confluent-firstboot.log diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh index 3a52d128..3b20a946 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh @@ -5,9 +5,16 @@ nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') confluent_apikey=$(cat /etc/confluent/confluent.apikey) -confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') -export nodename confluent_mgr confluent_profile +confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}') +if [ -z "$confluent_mgr" ] || [ "$confluent_mgr" == "null" ] || ! ping -c 1 $confluent_mgr >& /dev/null; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') +fi +confluent_websrv=$confluent_mgr +if [[ "$confluent_mgr" == *:* ]]; then + confluent_websrv="[$confluent_mgr]" +fi +export nodename confluent_mgr confluent_profile confluent_websrv . /etc/confluent/functions mkdir -p /var/log/confluent chmod 700 /var/log/confluent From 85629dea64d202c02a5f99191b4535b7743a1e03 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 7 Dec 2023 14:44:42 -0500 Subject: [PATCH 059/319] Prevent unitiailized collective info When doing proxyconsole, don't land in a useless retach loop when managerinfo is None. --- confluent_server/confluent/consoleserver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index 37274792..fb607a27 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -622,6 +622,8 @@ def connect_node(node, configmanager, username=None, direct=True, width=80, myname = collective.get_myname() if myc and myc != collective.get_myname() and direct: minfo = configmodule.get_collective_member(myc) + if not minfo: + raise Exception('Unable to get collective member for {}'.format(node)) return ProxyConsole(node, minfo, myname, configmanager, username, width, height) consk = (node, configmanager.tenant) From b0e23121a84c9b8d980e5bb0cd4f3edc19f2b325 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 3 Jan 2024 15:03:49 -0500 Subject: [PATCH 060/319] Add stub resize handler For uninitialized console handlers, provide a stub to do nothing on resize. This avoids such a request crashing a shared websocket session. --- confluent_server/confluent/consoleserver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index fb607a27..ebfd8c97 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -175,6 +175,9 @@ class ConsoleHandler(object): self.connectstate = 'connecting' eventlet.spawn(self._connect) + def resize(self, width, height): + return None + def _get_retry_time(self): clustsize = len(self.cfgmgr._cfgstore['nodes']) self._retrytime = self._retrytime * 2 + 1 From 39c00323b391d1ef0e83aad60fd5f543b68702bd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 3 Jan 2024 15:58:24 -0500 Subject: [PATCH 061/319] Fix error where layout would bail if a partial error were encountered --- confluent_server/confluent/plugins/info/layout.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/confluent_server/confluent/plugins/info/layout.py b/confluent_server/confluent/plugins/info/layout.py index 8397af7f..76b07ac7 100644 --- a/confluent_server/confluent/plugins/info/layout.py +++ b/confluent_server/confluent/plugins/info/layout.py @@ -93,6 +93,9 @@ def retrieve(nodes, element, configmanager, inputdata): '/noderange/{0}/description'.format(needheight), 'retrieve', configmanager, inputdata=None): + if not hasattr(rsp, 'kvpairs'): + results['errors'].append((rsp.node, rsp.error)) + continue kvp = rsp.kvpairs for node in kvp: allnodedata[node]['height'] = kvp[node]['height'] From 4d639081645339ca9dcee1c5922eeb489172e9c1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 4 Jan 2024 11:17:02 -0500 Subject: [PATCH 062/319] Have a fallback height of 1 for any missing height --- confluent_server/confluent/plugins/info/layout.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/confluent_server/confluent/plugins/info/layout.py b/confluent_server/confluent/plugins/info/layout.py index 76b07ac7..ca7f120c 100644 --- a/confluent_server/confluent/plugins/info/layout.py +++ b/confluent_server/confluent/plugins/info/layout.py @@ -99,5 +99,8 @@ def retrieve(nodes, element, configmanager, inputdata): kvp = rsp.kvpairs for node in kvp: allnodedata[node]['height'] = kvp[node]['height'] + for node in allnodedata: + if 'height' not in allnodedata[node]: + allnodedata[node]['height'] = 1 yield msg.Generic(results) From 70f91d59b293b97b51d76025e0b939adf0d7186f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Jan 2024 13:32:54 -0500 Subject: [PATCH 063/319] Update license material in gathering genesis --- genesis/buildgenesis.sh | 4 ++++ genesis/getlicenses.py | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 81c46790..680702ef 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -44,6 +44,10 @@ for lic in $(cat /tmp/tmpliclist); do cp $lic licenses/$dlo/$fname lo=$dlo/$fname echo %license /opt/confluent/genesis/%{arch}/licenses/$lo >> confluent-genesis-out.spec + if [ "$fname" == README ] && [ "$dlo" == "zlib" ]; then + cp $lic licenses/nss/$fname + echo %license /opt/confluent/genesis/%{arch}/licenses/nss/$fname >> confluent-genesis-out.spec + fi done mkdir -p licenses/ipmitool cp /usr/share/doc/ipmitool/COPYING licenses/ipmitool diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index 5a9b2ac2..92899850 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -29,12 +29,17 @@ with open(sys.argv[1]) as rpmlist: rpmlist = rpmlist.read().split('\n') licenses = set([]) licensesbyrpm = {} +lfirmlicenses = [ + 'WHENCE', + 'chelsio_firmware', + 'hfi1_firmware', + 'ice_enhaced', + 'rtlwifi_firmware.txt', +] for rpm in rpmlist: if not rpm: continue srpm = rpmtosrpm[rpm] - if srpm.startswith('linux-firmware'): - continue for relrpm in srpmtorpm[srpm]: if relrpm.startswith('libss-'): continue @@ -44,6 +49,12 @@ for rpm in rpmlist: continue if lic == '(contains no files)': continue + if srpm.startswith('linux-firmware'): + for desired in lfirmlicenses: + if desired in lic: + break + else: + continue licensesbyrpm[rpm] = lic licenses.add(lic) for lic in sorted(licenses): @@ -63,6 +74,8 @@ manualrpms = [ ] manuallicenses = [ '/usr/share/licenses/lz4/LICENSE.BSD', + '/usr/share/licenses/nss/LICENSE.APACHE', # http://www.apache.org/licenses/LICENSE-2.0 + '/usr/share/licenses/openssh/COPYING.blowfish, # from header of blowfish file in bsd-compat # cp /usr/share/doc/lz4-libs/LICENSE /usr/share/licenses/lz4/LICENSE.BSD #'lz4-1.8.3]# cp LICENSE /usr/share/licenses/lz4/LICENSE' # net-snmp has a bundled openssl, but the build does not avail itself of that copy From cfccb046bc4a39157993701774979f9ea953c660 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Jan 2024 13:36:55 -0500 Subject: [PATCH 064/319] Correct syntax error in draft attempt --- genesis/getlicenses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index 92899850..037699bc 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -75,7 +75,7 @@ manualrpms = [ manuallicenses = [ '/usr/share/licenses/lz4/LICENSE.BSD', '/usr/share/licenses/nss/LICENSE.APACHE', # http://www.apache.org/licenses/LICENSE-2.0 - '/usr/share/licenses/openssh/COPYING.blowfish, # from header of blowfish file in bsd-compat + '/usr/share/licenses/openssh/COPYING.blowfish', # from header of blowfish file in bsd-compat # cp /usr/share/doc/lz4-libs/LICENSE /usr/share/licenses/lz4/LICENSE.BSD #'lz4-1.8.3]# cp LICENSE /usr/share/licenses/lz4/LICENSE' # net-snmp has a bundled openssl, but the build does not avail itself of that copy From 0dfe66f1b28ebf3b3b59a935dbe882a5449ce91d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Jan 2024 16:08:49 -0500 Subject: [PATCH 065/319] Fix overzealous reaping of '-lib' folders --- genesis/buildgenesis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genesis/buildgenesis.sh b/genesis/buildgenesis.sh index 680702ef..8e0de608 100644 --- a/genesis/buildgenesis.sh +++ b/genesis/buildgenesis.sh @@ -29,7 +29,7 @@ for lic in $(cat /tmp/tmpliclist); do fname=$(basename $lo) dlo=$(dirname $lo) if [[ "$dlo" == *"-lib"* ]]; then - dlo=${dlo/-*} + dlo=${dlo/-lib*} elif [[ "$dlo" == "device-mapper-"* ]]; then dlo=${dlo/-*}-mapper elif [[ "$dlo" == "bind-"* ]]; then From 382feea68d619572ab4650c0dd5bd8aff69904b4 Mon Sep 17 00:00:00 2001 From: henglikuang1 Date: Thu, 11 Jan 2024 11:44:25 +0800 Subject: [PATCH 066/319] Add default time zone as UTC --- confluent_server/confluent/selfservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index cd4180c7..3d7feebb 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -30,7 +30,7 @@ import eventlet webclient = eventlet.import_patched('pyghmi.util.webclient') -currtz = None +currtz = 'UTC' keymap = 'us' currlocale = 'en_US.UTF-8' currtzvintage = None From ea88ccb0add2b661dcd7daa5fe6393f0e5cdeb40 Mon Sep 17 00:00:00 2001 From: henglikuang1 Date: Thu, 11 Jan 2024 14:31:45 +0800 Subject: [PATCH 067/319] Fix efivars handling of unexpected unmount --- confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh | 1 + confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh index 7b970285..16a624c3 100755 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh @@ -56,6 +56,7 @@ cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin mount -o bind /dev /target/dev mount -o bind /proc /target/proc mount -o bind /sys /target/sys +mount -o bind /sys/firmware/efi/efivars /target/sys/firmware/efi/efivars if [ 1 = $updategrub ]; then chroot /target update-grub fi diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 773bf8ad..d9730889 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -60,6 +60,7 @@ cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin mount -o bind /dev /target/dev mount -o bind /proc /target/proc mount -o bind /sys /target/sys +mount -o bind /sys/firmware/efi/efivars /target/sys/firmware/efi/efivars if [ 1 = $updategrub ]; then chroot /target update-grub fi From 56b644ead9927b25f86d869aa6cf1b63233ebed6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 11 Jan 2024 16:30:45 -0500 Subject: [PATCH 068/319] The 3rd party monotonic is no longer needed --- confluent_server/confluent_server.spec.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 51046a8f..bf81c969 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -17,7 +17,7 @@ Requires: confluent_vtbufferd Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python-websocket-client python2-msgpack python-libarchive-c python-yaml python-monotonic %else %if "%{dist}" == ".el8" -Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute +Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute %else %if "%{dist}" == ".el9" Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute From 07f91d792a5fc354750691094718892de0249eb9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jan 2024 10:52:49 -0500 Subject: [PATCH 069/319] Fix omission of info dir in plugins --- confluent_server/setup.py.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index e6bd08b2..871497e3 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -19,6 +19,7 @@ setup( 'confluent/plugins/hardwaremanagement/', 'confluent/plugins/deployment/', 'confluent/plugins/console/', + 'confluent/plugins/info/', 'confluent/plugins/shell/', 'confluent/collective/', 'confluent/plugins/configuration/'], From 5fdd6973f12279fbfae31823cd2c02a4e4130b46 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 16 Jan 2024 11:11:53 -0500 Subject: [PATCH 070/319] Update with more license content --- genesis/getlicenses.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index 037699bc..5306eb68 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -80,6 +80,12 @@ manuallicenses = [ #'lz4-1.8.3]# cp LICENSE /usr/share/licenses/lz4/LICENSE' # net-snmp has a bundled openssl, but the build does not avail itself of that copy '/usr/share/licenses/perl-libs/LICENSE', # ./dist/ExtUtils-CBuilder/LICENSE from perl srpm + '/usr/share/licenses/pam/COPYING.bison', # pam_conv_y + '/usr/share/licenses/pcre/LICENSE.BSD2', # stack-less just in time compiler, Zoltan Herzeg + '/usr/share/licenses/sqlite/LICENSE.md', # https://raw.githubusercontent.com/sqlite/sqlite/master/LICENSE.md + '/usr/share/licenses/pcre2/LICENSE.BSD2', + '/usr/share/licenses/perl/COPYING.regexec', # regexec.c + '/usr/share/doc/platform-python/README.rst', '/usr/share/licenses/lz4/LICENSE', '/usr/share/licenses/lm_sensors/COPYING', '/usr/share/doc/libunistring/README', @@ -93,7 +99,9 @@ manuallicenses = [ '/usr/share/doc/libnl3/COPYING', '/usr/share/licenses/xfsprogs/GPL-2.0', '/usr/share/licenses/xfsprogs/LGPL-2.1', - '/usr/share/licenses/tmux/NOTICE', + '/usr/share/licenses/tmux/NOTICE', # built by extracttmuxlicenses.py + '/usr/share/licenses/tmux/COPYING', # extracted from source + '/usr/share/licenses/tmux/README', # extracted from source '/usr/share/licenses/kernel-extra/exceptions/Linux-syscall-note', '/usr/share/licenses/kernel-extra/other/Apache-2.0', '/usr/share/licenses/kernel-extra/other/CC-BY-SA-4.0', From 9203ac32e979fa29681cd4bbea97e154c70adb24 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 17 Jan 2024 17:01:49 -0500 Subject: [PATCH 071/319] Start work on browserfs concept This will allow WebUI reactivity even with large files for import. --- confluent_server/confluent/core.py | 13 ++++- confluent_server/confluent/mountmanager.py | 58 ++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 confluent_server/confluent/mountmanager.py diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index 6ab6bd59..f21d36b5 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -44,6 +44,7 @@ import confluent.discovery.core as disco import confluent.interface.console as console import confluent.exceptions as exc import confluent.messages as msg +import confluent.mountmanager as mountmanager import confluent.networking.macmap as macmap import confluent.noderange as noderange import confluent.osimage as osimage @@ -159,7 +160,7 @@ def _merge_dict(original, custom): rootcollections = ['deployment/', 'discovery/', 'events/', 'networking/', - 'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' , + 'noderange/', 'nodes/', 'nodegroups/', 'storage/', 'usergroups/' , 'users/', 'uuid', 'version'] @@ -169,6 +170,13 @@ class PluginRoute(object): +def handle_storage(configmanager, inputdata, pathcomponents, operation): + if len(pathcomponents) == 1: + yield msg.ChildCollection('remote/') + return + if pathcomponents[1] == 'remote': + for rsp in mountmanager.handle_request(configmanager, inputdata, pathcomponents[2:], operation): + yield rsp def handle_deployment(configmanager, inputdata, pathcomponents, operation): if len(pathcomponents) == 1: @@ -1245,6 +1253,9 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True): elif pathcomponents[0] == 'deployment': return handle_deployment(configmanager, inputdata, pathcomponents, operation) + elif pathcomponents[0] == 'storage': + return handle_storage(configmanager, inputdata, pathcomponents, + operation) elif pathcomponents[0] == 'nodegroups': return handle_nodegroup_request(configmanager, inputdata, pathcomponents, diff --git a/confluent_server/confluent/mountmanager.py b/confluent_server/confluent/mountmanager.py new file mode 100644 index 00000000..c73b87a2 --- /dev/null +++ b/confluent_server/confluent/mountmanager.py @@ -0,0 +1,58 @@ + +import confluent.messages as msg +import confluent.exceptions as exc +import struct +import eventlet.green.socket as socket +mountsbyuser = {} + +def handle_request(configmanager, inputdata, pathcomponents, operation): + curruser = configmanager.current_user + if len(pathcomponents) == 0: + mounts = mountsbyuser.get(curruser, []) + if operation == 'retrieve': + for mount in mounts: + yield msg.ChildCollection(mount['index']) + elif operation == 'create': + if 'name' not in inputdata: + raise exc.InvalidArgumentException('Required parameter "name" is missing') + usedidx = set([]) + for mount in mounts: + usedidx.add(mount['index']) + curridx = 1 + while curridx in usedidx: + curridx += 1 + currmount = requestmount(curruser, inputdata['name']) + currmount['index'] = curridx + if curruser not in mountsbyuser: + mountsbyuser[curruser] = [] + mountsbyuser[curruser].append(currmount) + yield msg.KeyValueData({ + 'path': currmount['path'], + 'authtoken': currmount['authtoken'] + }) + +def requestmount(subdir, filename): + a = socket.socket(socket.AF_UNIX) + a.connect('/var/run/confluent/browserfs/control') + subname = subdir.encode() + a.send(struct.pack('!II', 1, len(subname))) + a.send(subname) + fname = filename.encode() + a.send(struct.pack('!I', len(fname))) + a.send(fname) + rsp = a.recv(4) + retcode = struct.unpack('!I', rsp)[0] + if retcode != 0: + raise Exception("Bad return code") + rsp = a.recv(4) + nlen = struct.unpack('!I', rsp)[0] + idstr = a.recv(nlen).decode('utf8') + rsp = a.recv(4) + nlen = struct.unpack('!I', rsp)[0] + authtok = a.recv(nlen).decode('utf8') + thismount = { + 'id': idstr, + 'path': '{}/{}/{}'.format(idstr, subdir, filename), + 'authtoken': authtok + } + return thismount From bcc631f88d821c474010ca3ad2f12936cd832801 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jan 2024 10:39:17 -0500 Subject: [PATCH 072/319] Set static hostname in diskless boot --- .../el8-diskless/profiles/default/scripts/onboot.sh | 1 + .../el9-diskless/profiles/default/scripts/onboot.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh index 3c99ad12..b2c0d1b3 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh @@ -16,6 +16,7 @@ if [ -z "$confluent_mgr" ]; then fi confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}') +hostnamectl set-hostname $nodename export nodename confluent_mgr confluent_profile . /etc/confluent/functions mkdir -p /var/log/confluent diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh index 3c99ad12..b2c0d1b3 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/onboot.sh @@ -16,6 +16,7 @@ if [ -z "$confluent_mgr" ]; then fi confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}') +hostnamectl set-hostname $nodename export nodename confluent_mgr confluent_profile . /etc/confluent/functions mkdir -p /var/log/confluent From b90718982efb2bd1895a4c2b1fa2c587da217de7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jan 2024 17:22:47 -0500 Subject: [PATCH 073/319] Implement a number of OS deployment management enhancements. Add capability to fingerprint media without doing a full import (/deployment/fingerprinting/) Add fetching the profile info as json under the /deployment/ api. Prepare to support custom distribution name on import --- confluent_server/confluent/core.py | 36 +++++++++++++++++++--- confluent_server/confluent/mountmanager.py | 1 + confluent_server/confluent/osimage.py | 36 ++++++++++++++-------- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index f21d36b5..a8a4412b 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -70,6 +70,7 @@ import os import eventlet.green.socket as socket import struct import sys +import yaml pluginmap = {} dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos') @@ -177,6 +178,7 @@ def handle_storage(configmanager, inputdata, pathcomponents, operation): if pathcomponents[1] == 'remote': for rsp in mountmanager.handle_request(configmanager, inputdata, pathcomponents[2:], operation): yield rsp + def handle_deployment(configmanager, inputdata, pathcomponents, operation): if len(pathcomponents) == 1: @@ -199,8 +201,19 @@ def handle_deployment(configmanager, inputdata, pathcomponents, for prof in osimage.list_profiles(): yield msg.ChildCollection(prof + '/') return - if len(pathcomponents) == 3: - profname = pathcomponents[-1] + if len(pathcomponents) >= 3: + profname = pathcomponents[2] + if len(pathcomponents) == 4: + if operation == 'retrieve': + if len(pathcomponents) == 4 and pathcomponents[-1] == 'info': + with open('/var/lib/confluent/public/os/{}/profile.yaml'.format(profname)) as profyaml: + profinfo = yaml.safe_load(profyaml) + profinfo['name'] = profname + yield msg.KeyValueData(profinfo) + return + elif len(pathcomponents) == 3: + if operation == 'retrieve': + yield msg.ChildCollection('info') if operation == 'update': if 'updateboot' in inputdata: osimage.update_boot(profname) @@ -216,6 +229,17 @@ def handle_deployment(configmanager, inputdata, pathcomponents, for cust in customized: yield msg.KeyValueData({'customized': cust}) return + if pathcomponents[1] == 'fingerprint': + if operation == 'create': + importer = osimage.MediaImporter(inputdata['filename'], configmanager, checkonly=True) + medinfo = { + 'targetpath': importer.targpath, + 'name': importer.osname, + 'oscategory': importer.oscategory, + 'errors': importer.errors, + } + yield msg.KeyValueData(medinfo) + return if pathcomponents[1] == 'importing': if len(pathcomponents) == 2 or not pathcomponents[-1]: if operation == 'retrieve': @@ -223,8 +247,12 @@ def handle_deployment(configmanager, inputdata, pathcomponents, yield imp return elif operation == 'create': - importer = osimage.MediaImporter(inputdata['filename'], - configmanager) + if inputdata.get('custname', None): + importer = osimage.MediaImporter(inputdata['filename'], + configmanager, inputdata['custname']) + else: + importer = osimage.MediaImporter(inputdata['filename'], + configmanager) yield msg.KeyValueData({'target': importer.targpath, 'name': importer.importkey}) return diff --git a/confluent_server/confluent/mountmanager.py b/confluent_server/confluent/mountmanager.py index c73b87a2..c6c10bc1 100644 --- a/confluent_server/confluent/mountmanager.py +++ b/confluent_server/confluent/mountmanager.py @@ -28,6 +28,7 @@ def handle_request(configmanager, inputdata, pathcomponents, operation): mountsbyuser[curruser].append(currmount) yield msg.KeyValueData({ 'path': currmount['path'], + 'fullpath': '/var/run/confluent/browserfs/mount/{}'.format(currmount['path']), 'authtoken': currmount['authtoken'] }) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 8884e0e9..969e05b4 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -747,9 +747,9 @@ def rebase_profile(dirname): # customization detected, skip # else # update required, manifest update - - - + + + def get_hashes(dirname): hashmap = {} for dname, _, fnames in os.walk(dirname): @@ -776,7 +776,7 @@ def generate_stock_profiles(defprofile, distpath, targpath, osname, continue oumask = os.umask(0o22) shutil.copytree(srcname, dirname) - hmap = get_hashes(dirname) + hmap = get_hashes(dirname) profdata = None try: os.makedirs('{0}/boot/initramfs'.format(dirname), 0o755) @@ -824,11 +824,12 @@ def generate_stock_profiles(defprofile, distpath, targpath, osname, class MediaImporter(object): - def __init__(self, media, cfm=None): + def __init__(self, media, cfm=None, customname=None, checkonly=False): self.worker = None if not os.path.exists('/var/lib/confluent/public'): raise Exception('`osdeploy initialize` must be executed before importing any media') self.profiles = [] + self.errors = [] medfile = None self.medfile = None if cfm and media in cfm.clientfiles: @@ -848,25 +849,34 @@ class MediaImporter(object): self.phase = 'copying' if not identity: raise Exception('Unrecognized OS Media') - if 'subname' in identity: + if customname: + importkey = customname + elif 'subname' in identity: importkey = '{0}-{1}'.format(identity['name'], identity['subname']) else: importkey = identity['name'] - if importkey in importing: + if importkey in importing and not checkonly: raise Exception('Media import already in progress for this media') self.importkey = importkey - importing[importkey] = self - self.importkey = importkey self.osname = identity['name'] self.oscategory = identity.get('category', None) - targpath = identity['name'] + if customname: + targpath = customname + else: + targpath = identity['name'] self.distpath = '/var/lib/confluent/distributions/' + targpath - if identity.get('subname', None): + if identity.get('subname', None): # subname is to indicate disk number in a media set targpath += '/' + identity['subname'] self.targpath = '/var/lib/confluent/distributions/' + targpath if os.path.exists(self.targpath): - del importing[importkey] - raise Exception('{0} already exists'.format(self.targpath)) + errstr = '{0} already exists'.format(self.targpath) + if checkonly: + self.errors = [errstr] + else: + raise Exception(errstr) + if checkonly: + return + importing[importkey] = self self.filename = os.path.abspath(media) self.error = '' self.importer = eventlet.spawn(self.importmedia) From e5736ecb51837bd9144114e6dc304cd1bbabae2c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jan 2024 14:18:25 -0500 Subject: [PATCH 074/319] Update license assets in genesis --- genesis/getlicenses.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index 5306eb68..d5e9cd12 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -94,6 +94,8 @@ manuallicenses = [ '/usr/share/doc/zstd/README.md', '/usr/share/doc/hwdata/LICENSE', '/usr/share/doc/ipmitool/COPYING', + '/usr/share/licenses/linux-firmware/LICENSE.hfi1_firmware', # these two need to be extracted from srcrpm + '/usr/share/licenses/linux-firmware/LICENSE.ice_enhanced', # '/usr/share/doc/libaio/COPYING', '/usr/share/doc/net-snmp/COPYING', '/usr/share/doc/libnl3/COPYING', @@ -121,6 +123,8 @@ manuallicenses = [ '/usr/share/licenses/kmod/tools/COPYING', # GPL not LGPL, must extract from kmod srpm '/usr/share/licenses/krb5-libs/NOTICE', # copy it verbatim from LICENSE, exact same file '/usr/share/doc/less/README', + '/usr/share/almalinux-release/EULA', + '/usr/share/doc/almalinux-release/GPL', '/usr/share/licenses/libcap-ng-utils/COPYING', '/usr/share/licenses/libdb/copyright', # from libdb, db-5.3.28, lang/sql/odbc/debian/copyright '/usr/share/licenses/libgcrypt/LICENSES.ppc-aes-gcm', # libgcrypt license to carry forward From fa7cd2940e58a513f2ddb288a123fbcfa41fb7f7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jan 2024 14:39:32 -0500 Subject: [PATCH 075/319] More license updates for genesis --- genesis/getlicenses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index d5e9cd12..edb47b90 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -76,6 +76,8 @@ manuallicenses = [ '/usr/share/licenses/lz4/LICENSE.BSD', '/usr/share/licenses/nss/LICENSE.APACHE', # http://www.apache.org/licenses/LICENSE-2.0 '/usr/share/licenses/openssh/COPYING.blowfish', # from header of blowfish file in bsd-compat + '/usr/share/licenses/bc/COPYING.GPLv2', + '/usr/share/licenses/bind-license/LICENSE', # MPLv2 from the source code # cp /usr/share/doc/lz4-libs/LICENSE /usr/share/licenses/lz4/LICENSE.BSD #'lz4-1.8.3]# cp LICENSE /usr/share/licenses/lz4/LICENSE' # net-snmp has a bundled openssl, but the build does not avail itself of that copy From e5051408e53c087c3c5a3cbaddff4a13ff2b249f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jan 2024 15:02:08 -0500 Subject: [PATCH 076/319] More license handling --- genesis/getlicenses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index edb47b90..5514a5cb 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -78,6 +78,7 @@ manuallicenses = [ '/usr/share/licenses/openssh/COPYING.blowfish', # from header of blowfish file in bsd-compat '/usr/share/licenses/bc/COPYING.GPLv2', '/usr/share/licenses/bind-license/LICENSE', # MPLv2 from the source code + '/usr/share/licenses/procps-ng/COPYING.LIBv2.1', # fetched internet # cp /usr/share/doc/lz4-libs/LICENSE /usr/share/licenses/lz4/LICENSE.BSD #'lz4-1.8.3]# cp LICENSE /usr/share/licenses/lz4/LICENSE' # net-snmp has a bundled openssl, but the build does not avail itself of that copy From a3d386dc39a119d375c80ffcb2e02e146d51f28d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jan 2024 14:47:23 -0500 Subject: [PATCH 077/319] Add NOTICE gathering for some genesis packages --- genesis/getlicenses.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index 5514a5cb..3f79c319 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -87,6 +87,9 @@ manuallicenses = [ '/usr/share/licenses/pcre/LICENSE.BSD2', # stack-less just in time compiler, Zoltan Herzeg '/usr/share/licenses/sqlite/LICENSE.md', # https://raw.githubusercontent.com/sqlite/sqlite/master/LICENSE.md '/usr/share/licenses/pcre2/LICENSE.BSD2', + '/usr/share/licenses/dhcp/NOTICE', + '/usr/share/licenses/bash/NOTICE', + '/usr/share/licenses/libsepol/NOTICE', '/usr/share/licenses/perl/COPYING.regexec', # regexec.c '/usr/share/doc/platform-python/README.rst', '/usr/share/licenses/lz4/LICENSE', From 41675e528f5c74debacf69ec8947112c2839c7cd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jan 2024 15:31:27 -0500 Subject: [PATCH 078/319] Amend dhcp license path --- genesis/getlicenses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index 3f79c319..f4df2fca 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -87,7 +87,7 @@ manuallicenses = [ '/usr/share/licenses/pcre/LICENSE.BSD2', # stack-less just in time compiler, Zoltan Herzeg '/usr/share/licenses/sqlite/LICENSE.md', # https://raw.githubusercontent.com/sqlite/sqlite/master/LICENSE.md '/usr/share/licenses/pcre2/LICENSE.BSD2', - '/usr/share/licenses/dhcp/NOTICE', + '/usr/share/licenses/dhcp-common/NOTICE', '/usr/share/licenses/bash/NOTICE', '/usr/share/licenses/libsepol/NOTICE', '/usr/share/licenses/perl/COPYING.regexec', # regexec.c From fa3e1202c47a450bad4c0a054daaef4266303050 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jan 2024 09:24:41 -0500 Subject: [PATCH 079/319] Relax systemd device policy to allow /dev/fuse access --- confluent_server/systemd/confluent.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/systemd/confluent.service b/confluent_server/systemd/confluent.service index da0fee7b..598c23c9 100644 --- a/confluent_server/systemd/confluent.service +++ b/confluent_server/systemd/confluent.service @@ -16,7 +16,7 @@ Restart=on-failure AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID CAP_CHOWN CAP_NET_RAW User=confluent Group=confluent -DevicePolicy=closed +#DevicePolicy=closed # fuse filesystem requires us to interact with /dev/fuse ProtectControlGroups=true ProtectSystem=true From 87454c1ab1dfdbee68c37ca9de8437c2d48c0cf1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jan 2024 09:31:59 -0500 Subject: [PATCH 080/319] Start browserfs if not yet running --- confluent_server/confluent/mountmanager.py | 20 ++++++++++++++++++++ genesis/getlicenses.py | 1 + 2 files changed, 21 insertions(+) diff --git a/confluent_server/confluent/mountmanager.py b/confluent_server/confluent/mountmanager.py index c6c10bc1..36f654d2 100644 --- a/confluent_server/confluent/mountmanager.py +++ b/confluent_server/confluent/mountmanager.py @@ -1,9 +1,27 @@ +import eventlet import confluent.messages as msg import confluent.exceptions as exc import struct import eventlet.green.socket as socket +import eventlet.green.subprocess as subprocess +import os mountsbyuser = {} +_browserfsd = None + +def assure_browserfs(): + global _browserfsd + if _browserfsd is None: + os.makedirs('/var/run/confluent/browserfs/mount', exist_ok=True) + _browserfsd = subprocess.Popen( + ['/opt/confluent/bin/browserfs', + '-c', '/var/run/confluent/browserfs/control', + '-s', '127.0.0.1:4006', + # browserfs supports unix domain websocket, however apache reverse proxy is dicey that way in some versions + '-w', '/var/run/confluent/browserfs/mount']) + while not os.path.exists('/var/run/confluent/browserfs/control'): + eventlet.sleep(0.5) + def handle_request(configmanager, inputdata, pathcomponents, operation): curruser = configmanager.current_user @@ -33,6 +51,7 @@ def handle_request(configmanager, inputdata, pathcomponents, operation): }) def requestmount(subdir, filename): + assure_browserfs() a = socket.socket(socket.AF_UNIX) a.connect('/var/run/confluent/browserfs/control') subname = subdir.encode() @@ -57,3 +76,4 @@ def requestmount(subdir, filename): 'authtoken': authtok } return thismount + diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index f4df2fca..a0118c48 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -88,6 +88,7 @@ manuallicenses = [ '/usr/share/licenses/sqlite/LICENSE.md', # https://raw.githubusercontent.com/sqlite/sqlite/master/LICENSE.md '/usr/share/licenses/pcre2/LICENSE.BSD2', '/usr/share/licenses/dhcp-common/NOTICE', + '/usr/share/licenses/xz/COPYING.GPLv3', # manually extracted from xz source '/usr/share/licenses/bash/NOTICE', '/usr/share/licenses/libsepol/NOTICE', '/usr/share/licenses/perl/COPYING.regexec', # regexec.c From 16ad4e776feab656a5ce5066658882dcf410af63 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Fri, 26 Jan 2024 12:39:25 -0500 Subject: [PATCH 081/319] opening web ui using default ip --- confluent_server/confluent/httpapi.py | 47 ++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index f36f2c73..4688ddd5 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -618,6 +618,31 @@ def resourcehandler(env, start_response): yield '500 - ' + str(e) return +def targ_ip_family(targip, first_pass=True): + # check ipv4 + try: + socket.inet_aton(targip) + return 'is_ipv4' + except socket.error: + pass + # check ipv6 + try: + check_ip = targip + if '%' in targip: + check_ip = targip.split('%')[0] + socket.inet_pton(socket.AF_INET6, check_ip) + return 'is_ipv6' + except socket.error: + # at this point we now know its not both ipv6 or ipv4 so we check if its hostname + if first_pass: + try: + ip_address = socket.gethostbyname(targip) + return targ_ip_family(ip_address, False) + except socket.gaierror: + return 'Cant figure that guy' + else: + return 'Cant figure it out' + def resourcehandler_backend(env, start_response): """Function to handle new wsgi requests @@ -728,7 +753,13 @@ def resourcehandler_backend(env, start_response): elif (env['PATH_INFO'].endswith('/forward/web') and env['PATH_INFO'].startswith('/nodes/')): prefix, _, _ = env['PATH_INFO'].partition('/forward/web') - _, _, nodename = prefix.rpartition('/') + #_, _, nodename = prefix.rpartition('/') + default = False + if 'default' in env['PATH_INFO']: + default = True + _,_,nodename,_ = prefix.split('/') + else: + _, _, nodename = prefix.rpartition('/') hm = cfgmgr.get_node_attributes(nodename, 'hardwaremanagement.manager') targip = hm.get(nodename, {}).get( 'hardwaremanagement.manager', {}).get('value', None) @@ -737,6 +768,20 @@ def resourcehandler_backend(env, start_response): yield 'No hardwaremanagement.manager defined for node' return targip = targip.split('/', 1)[0] + if default: + # understand targip + ip_family = targ_ip_family(targip) + if ip_family == 'is_ipv4': + url = 'https://{0}'.format(targip) + elif ip_family == 'is_ipv6': + url = 'https://[{0}]'.format(targip) + else: + start_response('404 Not Found', headers) + yield 'Cant figure out the hardwaremanagenent.manager attribute ip' + return + start_response('302', [('Location', url)]) + yield 'Our princess is in another castle!' + return funport = forwarder.get_port(targip, env['HTTP_X_FORWARDED_FOR'], authorized['sessionid']) host = env['HTTP_X_FORWARDED_HOST'] From fcb3d917db33d08d512b156b3d5e5ac5b986d9a2 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Fri, 26 Jan 2024 17:14:04 -0500 Subject: [PATCH 082/319] use socket.getaddrinfo --- confluent_server/confluent/httpapi.py | 51 +++++++++------------------ 1 file changed, 17 insertions(+), 34 deletions(-) diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index 4688ddd5..e30df36d 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -618,32 +618,6 @@ def resourcehandler(env, start_response): yield '500 - ' + str(e) return -def targ_ip_family(targip, first_pass=True): - # check ipv4 - try: - socket.inet_aton(targip) - return 'is_ipv4' - except socket.error: - pass - # check ipv6 - try: - check_ip = targip - if '%' in targip: - check_ip = targip.split('%')[0] - socket.inet_pton(socket.AF_INET6, check_ip) - return 'is_ipv6' - except socket.error: - # at this point we now know its not both ipv6 or ipv4 so we check if its hostname - if first_pass: - try: - ip_address = socket.gethostbyname(targip) - return targ_ip_family(ip_address, False) - except socket.gaierror: - return 'Cant figure that guy' - else: - return 'Cant figure it out' - - def resourcehandler_backend(env, start_response): """Function to handle new wsgi requests """ @@ -769,15 +743,24 @@ def resourcehandler_backend(env, start_response): return targip = targip.split('/', 1)[0] if default: - # understand targip - ip_family = targ_ip_family(targip) - if ip_family == 'is_ipv4': - url = 'https://{0}'.format(targip) - elif ip_family == 'is_ipv6': - url = 'https://[{0}]'.format(targip) - else: + try: + ip_info = socket.getaddrinfo(targip, 0, 0, socket.SOCK_STREAM) + except socket.gaierror: start_response('404 Not Found', headers) - yield 'Cant figure out the hardwaremanagenent.manager attribute ip' + yield 'hardwaremanagement.manager definition could not be resolved' + return + # this is just to future proof just in case the indexes of the address family change in future + for i in range(len(ip_info)): + if ip_info[i][0] == socket.AF_INET: + url = 'https://{0}/'.format(ip_info[i][-1][0]) + start_response('302', [('Location', url)]) + yield 'Our princess is in another castle!' + return + elif ip_info[i][0] == socket.AF_INET6: + url = 'https://[{0}]/'.format(ip_info[i][-1][0]) + if url.startswith('https://[fe80'): + start_response('405 Method Not Allowed', headers) + yield 'link local ipv6 address cannot be used in browser' return start_response('302', [('Location', url)]) yield 'Our princess is in another castle!' From d0373977b35464d15374f6abd7d8d80b03fd4365 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 09:08:28 -0500 Subject: [PATCH 083/319] Fix FFDC preflight checks The code was comparing two string constants, instead of a variable to a constant. Correct the problem to enable the preflight checks to work as intended. --- confluent_server/confluent/firmwaremanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/firmwaremanager.py b/confluent_server/confluent/firmwaremanager.py index a7713943..eb5d4c86 100644 --- a/confluent_server/confluent/firmwaremanager.py +++ b/confluent_server/confluent/firmwaremanager.py @@ -53,7 +53,7 @@ def execupdate(handler, filename, updateobj, type, owner, node, datfile): return if type == 'ffdc' and os.path.isdir(filename): filename += '/' + node - if 'type' == 'ffdc': + if type == 'ffdc': errstr = False if os.path.exists(filename): errstr = '{0} already exists on {1}, cannot overwrite'.format( From 72cace5a50f55836db19035d3da51068e3c5bac9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 11:11:27 -0500 Subject: [PATCH 084/319] More thoroughly wire up custom name Have custom name go through to actual import and influence profile names --- confluent_server/confluent/osimage.py | 31 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 969e05b4..2289a048 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -601,7 +601,7 @@ def fingerprint(archive): return imginfo, None, None -def import_image(filename, callback, backend=False, mfd=None): +def import_image(filename, callback, backend=False, mfd=None, custtargpath=None, custdistpath=None, custname=''): if mfd: archive = os.fdopen(int(mfd), 'rb') else: @@ -610,11 +610,16 @@ def import_image(filename, callback, backend=False, mfd=None): if not identity: return -1 identity, imginfo, funname = identity - targpath = identity['name'] - distpath = '/var/lib/confluent/distributions/' + targpath - if identity.get('subname', None): - targpath += '/' + identity['subname'] - targpath = '/var/lib/confluent/distributions/' + targpath + distpath = custdistpath + if not distpath: + targpath = identity['name'] + distpath = '/var/lib/confluent/distributions/' + targpath + if not custtargpath: + if identity.get('subname', None): + targpath += '/' + identity['subname'] + targpath = '/var/lib/confluent/distributions/' + targpath + else: + targpath = custtargpath try: os.makedirs(targpath, 0o755) except Exception as e: @@ -765,12 +770,15 @@ def get_hashes(dirname): def generate_stock_profiles(defprofile, distpath, targpath, osname, - profilelist): + profilelist, customname): osd, osversion, arch = osname.split('-') bootupdates = [] for prof in os.listdir('{0}/profiles'.format(defprofile)): srcname = '{0}/profiles/{1}'.format(defprofile, prof) - profname = '{0}-{1}'.format(osname, prof) + if customname: + profname = '{0}-{1}'.format(customname, prof) + else: + profname = '{0}-{1}'.format(osname, prof) dirname = '/var/lib/confluent/public/os/{0}'.format(profname) if os.path.exists(dirname): continue @@ -849,6 +857,7 @@ class MediaImporter(object): self.phase = 'copying' if not identity: raise Exception('Unrecognized OS Media') + self.customname = customname if customname else '' if customname: importkey = customname elif 'subname' in identity: @@ -894,7 +903,7 @@ class MediaImporter(object): os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno()) with open(os.devnull, 'w') as devnull: self.worker = subprocess.Popen( - [sys.executable, __file__, self.filename, '-b'], + [sys.executable, __file__, self.filename, '-b', self.targpath, self.distpath, self.customname], stdin=devnull, stdout=subprocess.PIPE, close_fds=False) wkr = self.worker currline = b'' @@ -934,7 +943,7 @@ class MediaImporter(object): self.oscategory) try: generate_stock_profiles(defprofile, self.distpath, self.targpath, - self.osname, self.profiles) + self.osname, self.profiles, self.customname) except Exception as e: self.phase = 'error' self.error = str(e) @@ -961,7 +970,7 @@ if __name__ == '__main__': os.umask(0o022) if len(sys.argv) > 2: mfd = os.environ.get('CONFLUENT_MEDIAFD', None) - sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd)) + sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd, custtargpath=sys.argv[3], custdistpath=sys.argv[4], custname=sys.argv[5])) else: sys.exit(import_image(sys.argv[1], callback=printit)) From 170e585e57c5d77689576317e9efed3d53ced43e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 13:53:59 -0500 Subject: [PATCH 085/319] Add preliminary ubuntu 24.04 support --- confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl | 5 +++-- confluent_osdeploy/confluent_osdeploy.spec.tmpl | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl index f1b7c804..add86e6c 100644 --- a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl @@ -26,7 +26,8 @@ mkdir -p opt/confluent/bin mkdir -p stateless-bin cp -a el8bin/* . ln -s el8 el9 -for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do +ln -s ubuntu22.04 ubuntu24.04 +for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do mkdir ${os}out cd ${os}out if [ -d ../${os}bin ]; then @@ -76,7 +77,7 @@ cp -a esxi7 esxi8 %install mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ #cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ -for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do +for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/ cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/ if [ -d ${os}disklessout ]; then diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index d939a0c3..07506bbf 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -28,7 +28,8 @@ This contains support utilities for enabling deployment of x86_64 architecture s #cp start_root urlmount ../stateless-bin/ #cd .. ln -s el8 el9 -for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 coreos el9; do +ln -s ubuntu22.04 ubuntu24.04 +for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do mkdir ${os}out cd ${os}out if [ -d ../${os}bin ]; then @@ -42,7 +43,7 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 coreo mv ../addons.cpio . cd .. done -for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04; do +for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04 ubuntu24.04; do mkdir ${os}disklessout cd ${os}disklessout if [ -d ../${os}bin ]; then From 7618fa8b634ca65d2ca0107c27e1624e41007fd8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 14:21:25 -0500 Subject: [PATCH 086/319] Add diskless links --- confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl | 1 + confluent_osdeploy/confluent_osdeploy.spec.tmpl | 1 + 2 files changed, 2 insertions(+) diff --git a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl index add86e6c..db2df9f0 100644 --- a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl @@ -27,6 +27,7 @@ mkdir -p stateless-bin cp -a el8bin/* . ln -s el8 el9 ln -s ubuntu22.04 ubuntu24.04 +ln -s ubuntu22.04-diskless ubuntu24.04-diskless for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do mkdir ${os}out cd ${os}out diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index 07506bbf..b8cdca12 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -29,6 +29,7 @@ This contains support utilities for enabling deployment of x86_64 architecture s #cd .. ln -s el8 el9 ln -s ubuntu22.04 ubuntu24.04 +ln -s ubuntu22.04-diskless ubuntu24.04-diskless for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do mkdir ${os}out cd ${os}out From 9ad9912ef1cad623c4e66e9175311b8dfb22449c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 14:28:11 -0500 Subject: [PATCH 087/319] Change to pre-made links for ubuntu24.04 --- confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl | 2 -- confluent_osdeploy/confluent_osdeploy.spec.tmpl | 2 -- confluent_osdeploy/ubuntu24.04 | 1 + confluent_osdeploy/ubuntu24.04-diskless | 1 + 4 files changed, 2 insertions(+), 4 deletions(-) create mode 120000 confluent_osdeploy/ubuntu24.04 create mode 120000 confluent_osdeploy/ubuntu24.04-diskless diff --git a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl index db2df9f0..fb6f6ddc 100644 --- a/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl @@ -26,8 +26,6 @@ mkdir -p opt/confluent/bin mkdir -p stateless-bin cp -a el8bin/* . ln -s el8 el9 -ln -s ubuntu22.04 ubuntu24.04 -ln -s ubuntu22.04-diskless ubuntu24.04-diskless for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do mkdir ${os}out cd ${os}out diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index b8cdca12..46648790 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -28,8 +28,6 @@ This contains support utilities for enabling deployment of x86_64 architecture s #cp start_root urlmount ../stateless-bin/ #cd .. ln -s el8 el9 -ln -s ubuntu22.04 ubuntu24.04 -ln -s ubuntu22.04-diskless ubuntu24.04-diskless for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 ubuntu24.04 coreos el9; do mkdir ${os}out cd ${os}out diff --git a/confluent_osdeploy/ubuntu24.04 b/confluent_osdeploy/ubuntu24.04 new file mode 120000 index 00000000..13759564 --- /dev/null +++ b/confluent_osdeploy/ubuntu24.04 @@ -0,0 +1 @@ +ubuntu22.04 \ No newline at end of file diff --git a/confluent_osdeploy/ubuntu24.04-diskless b/confluent_osdeploy/ubuntu24.04-diskless new file mode 120000 index 00000000..00822b05 --- /dev/null +++ b/confluent_osdeploy/ubuntu24.04-diskless @@ -0,0 +1 @@ +ubuntu20.04-diskless \ No newline at end of file From fc82021f2bd80e918a7cb89d79b7c8cbbda36794 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 15:04:04 -0500 Subject: [PATCH 088/319] Add missing ubuntu24.04 to packaging --- confluent_osdeploy/confluent_osdeploy.spec.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/confluent_osdeploy.spec.tmpl b/confluent_osdeploy/confluent_osdeploy.spec.tmpl index 46648790..5faab31f 100644 --- a/confluent_osdeploy/confluent_osdeploy.spec.tmpl +++ b/confluent_osdeploy/confluent_osdeploy.spec.tmpl @@ -78,7 +78,7 @@ cp -a esxi7 esxi8 %install mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/ -for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do +for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 ubuntu24.04 esxi6 esxi7 esxi8 coreos; do mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs From 7377c44e0fadc79b91cfe0daee164d6cbd7a2759 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 1 Feb 2024 08:50:44 -0500 Subject: [PATCH 089/319] Fix problem where one multicast/broadcast attempt could tank other interfaces Carrying over change from ssdp, ignore failures on transmit, particularly if firewall --- confluent_server/confluent/discovery/protocols/slp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/slp.py b/confluent_server/confluent/discovery/protocols/slp.py index e42c1577..ac332def 100644 --- a/confluent_server/confluent/discovery/protocols/slp.py +++ b/confluent_server/confluent/discovery/protocols/slp.py @@ -246,11 +246,11 @@ def _find_srvtype(net, net4, srvtype, addresses, xid): try: net4.sendto(data, ('239.255.255.253', 427)) except socket.error as se: - # On occasion, multicasting may be disabled - # tolerate this scenario and move on - if se.errno != 101: - raise - net4.sendto(data, (bcast, 427)) + pass + try: + net4.sendto(data, (bcast, 427)) + except socket.error as se: + pass def _grab_rsps(socks, rsps, interval, xidmap, deferrals): From a17695ad0653e7ee7425a371179b6f6d223ce783 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 1 Feb 2024 16:38:04 -0500 Subject: [PATCH 090/319] Extend confluent PXE support For relay agent options, preserve and echo back the option, needed for certain environments. Also, it turns out that for whatever reason on some platforms, iPXE's proxyDHCP logic can't seem to get a reply. In this scenario, provide the filename in the DHCP offer without waiting for proxyDHCP. This change may be worth evaluating more broadly, but may carry risk of not working right with unmanaged DHCP servers. --- .../confluent/discovery/protocols/pxe.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index a9a07963..6dd34efa 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -771,6 +771,14 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): node, profile, len(bootfile) - 127)}) return repview[108:108 + len(bootfile)] = bootfile + elif info['architecture'] == 'uefi-aarch64' and packet.get(77, None) == b'iPXE': + if not profile: + profile = get_deployment_profile(node, cfg) + if not profile: + log.log({'info': 'No pending profile for {0}, skipping proxyDHCP eply'.format(node)}) + return + bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myipn, profile).encode('utf8') + repview[108:108 + len(bootfile)] = bootfile myip = myipn myipn = socket.inet_aton(myipn) orepview[12:16] = myipn @@ -812,6 +820,13 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): repview[replen - 1:replen + 1] = b'\x03\x04' repview[replen + 1:replen + 5] = gateway replen += 6 + if 82 in packet: + reloptionslen = len(packet[82]) + reloptionshdr = struct.pack('BB', 82, reloptionslen) + repview[replen - 1:replen + 1] = reloptionshdr + repview[replen + 1:replen + reloptionslen + 1] = packet[82] + replen += 2 + reloptionslen + repview[replen - 1:replen] = b'\xff' # end of options, should always be last byte repview = memoryview(reply) pktlen = struct.pack('!H', replen + 28) # ip+udp = 28 From 59a31d38a25f1c44bad3dcfc2daad2a0520c5501 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 2 Feb 2024 08:51:12 -0500 Subject: [PATCH 091/319] Make reseat concurrent Spawn reseat activity concurrently between chassis. This should reduce time to nodes per chassis rather than total nodes. --- .../plugins/hardwaremanagement/enclosure.py | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py b/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py index 933a852b..4658e2a0 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py @@ -15,10 +15,32 @@ import confluent.core as core import confluent.messages as msg import pyghmi.exceptions as pygexc import confluent.exceptions as exc +import eventlet +import eventlet.queue as queue +import eventlet.greenpool as greenpool + + +def reseat_bays(encmgr, bays, configmanager, rspq): + try: + for encbay in bays: + node = bays[encbay] + try: + for rsp in core.handle_path( + '/nodes/{0}/_enclosure/reseat_bay'.format(encmgr), + 'update', configmanager, + inputdata={'reseat': int(encbay)}): + rspq.put(rsp) + except pygexc.UnsupportedFunctionality as uf: + rspq.put(msg.ConfluentNodeError(node, str(uf))) + except exc.TargetEndpointUnreachable as uf: + rspq.put(msg.ConfluentNodeError(node, str(uf))) + finally: + rspq.put(None) def update(nodes, element, configmanager, inputdata): emebs = configmanager.get_node_attributes( nodes, (u'enclosure.manager', u'enclosure.bay')) + baysbyencmgr = {} for node in nodes: try: em = emebs[node]['enclosure.manager']['value'] @@ -30,13 +52,20 @@ def update(nodes, element, configmanager, inputdata): em = node if not eb: eb = -1 - try: - for rsp in core.handle_path( - '/nodes/{0}/_enclosure/reseat_bay'.format(em), - 'update', configmanager, - inputdata={'reseat': int(eb)}): - yield rsp - except pygexc.UnsupportedFunctionality as uf: - yield msg.ConfluentNodeError(node, str(uf)) - except exc.TargetEndpointUnreachable as uf: - yield msg.ConfluentNodeError(node, str(uf)) + if em not in baysbyencmgr: + baysbyencmgr[em] = {} + baysbyencmgr[em][eb] = node + rspq = queue.Queue() + gp = greenpool.GreenPool(64) + for encmgr in baysbyencmgr: + gp.spawn_n(reseat_bays, encmgr, baysbyencmgr[encmgr], configmanager, rspq) + while gp.running(): + nrsp = rspq.get() + if nrsp is not None: + yield nrsp + while not rspq.empty(): + nrsp = rspq.get() + if nrsp is not None: + yield nrsp + + From 3a0172ccccd9a14ef63794a3bd646187b13cff3f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 2 Feb 2024 10:35:47 -0500 Subject: [PATCH 092/319] Make indirect PDU operations concurrent Similar to the enclosure reseat work, have indirect PDU operations be made concurrent across PDUs, though still serial within a PDU. --- .../plugins/hardwaremanagement/enclosure.py | 1 - .../plugins/hardwaremanagement/pdu.py | 68 ++++++++++++++++--- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py b/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py index 4658e2a0..a59422c0 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/enclosure.py @@ -15,7 +15,6 @@ import confluent.core as core import confluent.messages as msg import pyghmi.exceptions as pygexc import confluent.exceptions as exc -import eventlet import eventlet.queue as queue import eventlet.greenpool as greenpool diff --git a/confluent_server/confluent/plugins/hardwaremanagement/pdu.py b/confluent_server/confluent/plugins/hardwaremanagement/pdu.py index b19c9b22..3db21636 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/pdu.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/pdu.py @@ -15,10 +15,16 @@ import confluent.core as core import confluent.messages as msg import pyghmi.exceptions as pygexc import confluent.exceptions as exc +import eventlet.greenpool as greenpool +import eventlet.queue as queue + +class TaskDone: + pass def retrieve(nodes, element, configmanager, inputdata): emebs = configmanager.get_node_attributes( nodes, (u'power.*pdu', u'power.*outlet')) + relpdus = {} if element == ['power', 'inlets']: outletnames = set([]) for node in nodes: @@ -39,13 +45,36 @@ def retrieve(nodes, element, configmanager, inputdata): for pgroup in outlets[node]: pdu = outlets[node][pgroup]['pdu'] outlet = outlets[node][pgroup]['outlet'] - try: - for rsp in core.handle_path( - '/nodes/{0}/power/outlets/{1}'.format(pdu, outlet), - 'retrieve', configmanager): - yield msg.KeyValueData({pgroup: rsp.kvpairs['state']['value']}, node) - except exc.TargetEndpointBadCredentials: - yield msg.ConfluentTargetInvalidCredentials(pdu) + if pdu not in relpdus: + relpdus[pdu] = {} + relpdus[pdu][outlet] = (node, pgroup) + rspq = queue.Queue() + gp = greenpool.GreenPool(64) + for pdu in relpdus: + gp.spawn(readpdu, pdu, relpdus[pdu], configmanager, rspq) + while gp.running(): + nrsp = rspq.get() + if not isinstance(nrsp, TaskDone): + yield nrsp + while not rspq.empty(): + nrsp = rspq.get() + if not isinstance(nrsp, TaskDone): + yield nrsp + +def readpdu(pdu, outletmap, configmanager, rspq): + try: + for outlet in outletmap: + node, pgroup = outletmap[outlet] + try: + for rsp in core.handle_path( + '/nodes/{0}/power/outlets/{1}'.format(pdu, outlet), + 'retrieve', configmanager): + rspq.put(msg.KeyValueData({pgroup: rsp.kvpairs['state']['value']}, node)) + except exc.TargetEndpointBadCredentials: + rspq.put(msg.ConfluentTargetInvalidCredentials(pdu)) + finally: # ensure thhat at least one thing triggers the get + rspq.put(TaskDone()) + def get_outlets(nodes, emebs, inletname): outlets = {} @@ -72,11 +101,34 @@ def update(nodes, element, configmanager, inputdata): emebs = configmanager.get_node_attributes( nodes, (u'power.*pdu', u'power.*outlet')) inletname = element[-1] + relpdus = {} + rspq = queue.Queue() + gp = greenpool.GreenPool(64) outlets = get_outlets(nodes, emebs, inletname) for node in outlets: for pgroup in outlets[node]: pdu = outlets[node][pgroup]['pdu'] outlet = outlets[node][pgroup]['outlet'] + if pdu not in relpdus: + relpdus[pdu] = {} + relpdus[pdu][outlet] = (node, pgroup) + for pdu in relpdus: + gp.spawn(updatepdu, pdu, relpdus[pdu], configmanager, inputdata, rspq) + while gp.running(): + nrsp = rspq.get() + if not isinstance(nrsp, TaskDone): + yield nrsp + while not rspq.empty(): + nrsp = rspq.get() + if not isinstance(nrsp, TaskDone): + yield nrsp + +def updatepdu(pdu, outletmap, configmanager, inputdata, rspq): + try: + for outlet in outletmap: + node, pgroup = outletmap[outlet] for rsp in core.handle_path('/nodes/{0}/power/outlets/{1}'.format(pdu, outlet), 'update', configmanager, inputdata={'state': inputdata.powerstate(node)}): - yield msg.KeyValueData({pgroup: rsp.kvpairs['state']['value']}, node) + rspq.put(msg.KeyValueData({pgroup: rsp.kvpairs['state']['value']}, node)) + finally: + rspq.put(TaskDone()) From d07e6f86c0ede225e0d314f5cd7e1cdf90566b11 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 13 Feb 2024 15:58:08 -0500 Subject: [PATCH 093/319] Provide more useful error messages on mistakes within [] --- confluent_server/confluent/noderange.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index cf99dd72..4a5cb808 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -402,12 +402,16 @@ class NodeRange(object): def _expandstring(self, element, filternodes=None): prefix = '' if element[0][0] in ('/', '~'): + if self.purenumeric: + raise Exception('Regular expression not supported within "[]"') element = ''.join(element) nameexpression = element[1:] if self.cfm is None: raise Exception('Verification configmanager required') return set(self.cfm.filter_nodenames(nameexpression, filternodes)) elif '=' in element[0] or '!~' in element[0]: + if self.purenumeric: + raise Exception('The "=" character is invalid within "[]"') element = ''.join(element) if self.cfm is None: raise Exception('Verification configmanager required') From 21f691cbd8fd984582b969d28a83d7be4f119ab5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 13 Feb 2024 16:00:50 -0500 Subject: [PATCH 094/319] Correct the equality message in better messagesw --- confluent_server/confluent/noderange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 4a5cb808..7657292c 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -411,7 +411,7 @@ class NodeRange(object): return set(self.cfm.filter_nodenames(nameexpression, filternodes)) elif '=' in element[0] or '!~' in element[0]: if self.purenumeric: - raise Exception('The "=" character is invalid within "[]"') + raise Exception('Equality/Inequality operators (=, !=, =~, !~) are invalid within "[]"') element = ''.join(element) if self.cfm is None: raise Exception('Verification configmanager required') From 72e26caf360e84d81ac057ad4998c41c119e8fa7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Feb 2024 15:05:56 -0500 Subject: [PATCH 095/319] Change to unix domain for vtbuffer communication The semaphore arbitrated single channel sharing was proving to be too slow. Make the communication lockless by having dedicated sockets per request. --- confluent_server/confluent/consoleserver.py | 56 +++--- confluent_vtbufferd/vtbufferd.c | 179 +++++++++++++++----- 2 files changed, 161 insertions(+), 74 deletions(-) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index ebfd8c97..19509eb5 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -62,39 +62,38 @@ def chunk_output(output, n): yield output[i:i + n] def get_buffer_output(nodename): - out = _bufferdaemon.stdin - instream = _bufferdaemon.stdout + out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) + out.connect("\x00confluent-vtbuffer") if not isinstance(nodename, bytes): nodename = nodename.encode('utf8') outdata = bytearray() - with _bufferlock: - out.write(struct.pack('I', len(nodename))) - out.write(nodename) - out.flush() - select.select((instream,), (), (), 30) - while not outdata or outdata[-1]: - try: - chunk = os.read(instream.fileno(), 128) - except IOError: - chunk = None - if chunk: - outdata.extend(chunk) - else: - select.select((instream,), (), (), 0) - return bytes(outdata[:-1]) + out.send(struct.pack('I', len(nodename))) + out.send(nodename) + select.select((out,), (), (), 30) + while not outdata or outdata[-1]: + try: + chunk = os.read(out.fileno(), 128) + except IOError: + chunk = None + if chunk: + outdata.extend(chunk) + else: + select.select((out,), (), (), 0) + return bytes(outdata[:-1]) def send_output(nodename, output): if not isinstance(nodename, bytes): nodename = nodename.encode('utf8') - with _bufferlock: - _bufferdaemon.stdin.write(struct.pack('I', len(nodename) | (1 << 29))) - _bufferdaemon.stdin.write(nodename) - _bufferdaemon.stdin.flush() - for chunk in chunk_output(output, 8192): - _bufferdaemon.stdin.write(struct.pack('I', len(chunk) | (2 << 29))) - _bufferdaemon.stdin.write(chunk) - _bufferdaemon.stdin.flush() + out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) + out.connect("\x00confluent-vtbuffer") + out.send(struct.pack('I', len(nodename) | (1 << 29))) + out.send(nodename) + for chunk in chunk_output(output, 8192): + out.send(struct.pack('I', len(chunk) | (2 << 29))) + out.send(chunk) def _utf8_normalize(data, decoder): # first we give the stateful decoder a crack at the byte stream, @@ -607,11 +606,8 @@ def initialize(): _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen( - ['/opt/confluent/bin/vtbufferd'], bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL) - fcntl.fcntl(_bufferdaemon.stdout.fileno(), - fcntl.F_SETFL, fl | os.O_NONBLOCK) + ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) def start_console_sessions(): configmodule.hook_new_configmanagers(_start_tenant_sessions) diff --git a/confluent_vtbufferd/vtbufferd.c b/confluent_vtbufferd/vtbufferd.c index e89269b4..055a5263 100644 --- a/confluent_vtbufferd/vtbufferd.c +++ b/confluent_vtbufferd/vtbufferd.c @@ -1,8 +1,14 @@ +#include +#define _GNU_SOURCE #include #include #include #include #include +#include +#include +#include +#include #include "tmt.h" #define HASHSIZE 2053 #define MAXNAMELEN 256 @@ -10,13 +16,17 @@ struct terment { struct terment *next; char *name; + int fd; TMT *vt; }; #define SETNODE 1 #define WRITE 2 #define READBUFF 0 +#define CLOSECONN 3 +#define MAXEVTS 16 static struct terment *buffers[HASHSIZE]; +static char* nodenames[HASHSIZE]; unsigned long hash(char *str) /* djb2a */ @@ -37,10 +47,13 @@ TMT *get_termentbyname(char *name) { return NULL; } -TMT *set_termentbyname(char *name) { +TMT *set_termentbyname(char *name, int fd) { struct terment *ret; int idx; + if (nodenames[fd] == NULL) { + nodenames[fd] = strdup(name); + } idx = hash(name); for (ret = buffers[idx]; ret != NULL; ret = ret->next) if (strcmp(name, ret->name) == 0) @@ -48,12 +61,13 @@ TMT *set_termentbyname(char *name) { ret = (struct terment *)malloc(sizeof(*ret)); ret->next = buffers[idx]; ret->name = strdup(name); + ret->fd = fd; ret->vt = tmt_open(31, 100, NULL, NULL, L"→←↑↓■◆▒°±▒┘┐┌└┼⎺───⎽├┤┴┬│≤≥π≠£•"); buffers[idx] = ret; return ret->vt; } -void dump_vt(TMT* outvt) { +void dump_vt(TMT* outvt, int outfd) { const TMTSCREEN *out = tmt_screen(outvt); const TMTPOINT *curs = tmt_cursor(outvt); int line, idx, maxcol, maxrow; @@ -67,9 +81,10 @@ void dump_vt(TMT* outvt) { tmt_color_t fg = TMT_COLOR_DEFAULT; tmt_color_t bg = TMT_COLOR_DEFAULT; wchar_t sgrline[30]; + char strbuffer[128]; size_t srgidx = 0; char colorcode = 0; - wprintf(L"\033c"); + write(outfd, "\033c", 2); maxcol = 0; maxrow = 0; for (line = out->nline - 1; line >= 0; --line) { @@ -148,60 +163,136 @@ void dump_vt(TMT* outvt) { } if (sgrline[0] != 0) { sgrline[wcslen(sgrline) - 1] = 0; // Trim last ; - wprintf(L"\033[%lsm", sgrline); + + snprintf(strbuffer, sizeof(strbuffer), "\033[%lsm", sgrline); + write(outfd, strbuffer, strlen(strbuffer)); + write(outfd, "\033[]", 3); } - wprintf(L"%lc", out->lines[line]->chars[idx].c); + snprintf(strbuffer, sizeof(strbuffer), "%lc", out->lines[line]->chars[idx].c); + write(outfd, strbuffer, strlen(strbuffer)); } if (line < maxrow) - wprintf(L"\r\n"); + write(outfd, "\r\n", 2); } - fflush(stdout); - wprintf(L"\x1b[%ld;%ldH", curs->r + 1, curs->c + 1); - fflush(stdout); + //fflush(stdout); + snprintf(strbuffer, sizeof(strbuffer), "\x1b[%ld;%ldH", curs->r + 1, curs->c + 1); + write(outfd, strbuffer, strlen(strbuffer)); + //fflush(stdout); +} + +int handle_traffic(int fd) { + int cmd, length; + char currnode[MAXNAMELEN]; + char cmdbuf[MAXDATALEN]; + char *nodename; + TMT *currvt = NULL; + TMT *outvt = NULL; + length = read(fd, &cmd, 4); + if (length <= 0) { + return 0; + } + length = cmd & 536870911; + cmd = cmd >> 29; + if (cmd == SETNODE) { + cmd = read(fd, currnode, length); + currnode[length] = 0; + if (cmd < 0) + return 0; + currvt = set_termentbyname(currnode, fd); + } else if (cmd == WRITE) { + if (currvt == NULL) { + nodename = nodenames[fd]; + currvt = set_termentbyname(nodename, fd); + } + cmd = read(fd, cmdbuf, length); + cmdbuf[length] = 0; + if (cmd < 0) + return 0; + tmt_write(currvt, cmdbuf, length); + } else if (cmd == READBUFF) { + cmd = read(fd, cmdbuf, length); + cmdbuf[length] = 0; + if (cmd < 0) + return 0; + outvt = get_termentbyname(cmdbuf); + if (outvt != NULL) + dump_vt(outvt, fd); + length = write(fd, "\x00", 1); + if (length < 0) + return 0; + } else if (cmd == CLOSECONN) { + return 0; + } + return 1; } int main(int argc, char* argv[]) { - int cmd, length; setlocale(LC_ALL, ""); - char cmdbuf[MAXDATALEN]; - char currnode[MAXNAMELEN]; - TMT *currvt = NULL; - TMT *outvt = NULL; + struct sockaddr_un addr; + int numevts; + int status; + int poller; + int n; + socklen_t len; + int ctlsock, currsock; + socklen_t addrlen; + struct ucred ucr; + + struct epoll_event epvt, evts[MAXEVTS]; stdin = freopen(NULL, "rb", stdin); if (stdin == NULL) { exit(1); } + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path + 1, argv[1], sizeof(addr.sun_path) - 2); // abstract namespace socket + ctlsock = socket(AF_UNIX, SOCK_STREAM, 0); + status = bind(ctlsock, (const struct sockaddr*)&addr, sizeof(sa_family_t) + strlen(argv[1]) + 1); //sizeof(struct sockaddr_un)); + if (status < 0) { + perror("Unable to open unix socket - "); + exit(1); + } + listen(ctlsock, 128); + poller = epoll_create(1); + memset(&epvt, 0, sizeof(struct epoll_event)); + epvt.events = EPOLLIN; + epvt.data.fd = ctlsock; + if (epoll_ctl(poller, EPOLL_CTL_ADD, ctlsock, &epvt) < 0) { + perror("Unable to poll the socket"); + exit(1); + } + // create a unix domain socket for accepting, each connection is only allowed to either read or write, not both while (1) { - length = fread(&cmd, 4, 1, stdin); - if (length < 0) - continue; - length = cmd & 536870911; - cmd = cmd >> 29; - if (cmd == SETNODE) { - cmd = fread(currnode, 1, length, stdin); - currnode[length] = 0; - if (cmd < 0) - continue; - currvt = set_termentbyname(currnode); - } else if (cmd == WRITE) { - if (currvt == NULL) - currvt = set_termentbyname(""); - cmd = fread(cmdbuf, 1, length, stdin); - cmdbuf[length] = 0; - if (cmd < 0) - continue; - tmt_write(currvt, cmdbuf, length); - } else if (cmd == READBUFF) { - cmd = fread(cmdbuf, 1, length, stdin); - cmdbuf[length] = 0; - if (cmd < 0) - continue; - outvt = get_termentbyname(cmdbuf); - if (outvt != NULL) - dump_vt(outvt); - length = write(1, "\x00", 1); - if (length < 0) - continue; + numevts = epoll_wait(poller, evts, MAXEVTS, -1); + if (numevts < 0) { + perror("Failed wait"); + exit(1); + } + for (n = 0; n < numevts; ++n) { + if (evts[n].data.fd == ctlsock) { + currsock = accept(ctlsock, (struct sockaddr *) &addr, &addrlen); + len = sizeof(ucr); + getsockopt(currsock, SOL_SOCKET, SO_PEERCRED, &ucr, &len); + if (ucr.uid != getuid()) { // block access for other users + close(currsock); + continue; + } + memset(&epvt, 0, sizeof(struct epoll_event)); + epvt.events = EPOLLIN; + epvt.data.fd = currsock; + epoll_ctl(poller, EPOLL_CTL_ADD, currsock, &epvt); + } else { + if (!handle_traffic(evts[n].data.fd)) { + epoll_ctl(poller, EPOLL_CTL_DEL, evts[n].data.fd, NULL); + close(evts[n].data.fd); + if (nodenames[evts[n].data.fd] != NULL) { + free(nodenames[evts[n].data.fd]); + nodenames[evts[n].data.fd] = NULL; + } + } + } } } } + + From fa5b1c671ef54e55f9f04b57a894a06dd2f23123 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Feb 2024 15:07:12 -0500 Subject: [PATCH 096/319] Remove disused bufferlock We no longer use a lock on buffer communication, eliminate the stale variable. --- confluent_server/confluent/consoleserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index 19509eb5..aa05b9b7 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -49,7 +49,6 @@ _handled_consoles = {} _tracelog = None _bufferdaemon = None -_bufferlock = None try: range = xrange @@ -602,8 +601,6 @@ def _start_tenant_sessions(cfm): def initialize(): global _tracelog global _bufferdaemon - global _bufferlock - _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen( ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL, From 75db6da621632db72e40d1a208c812f327c0b6f1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 4 Mar 2024 08:06:01 -0500 Subject: [PATCH 097/319] Opportunisticlly use sshd_config.d when detected --- .../el8/profiles/default/scripts/setupssh.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/setupssh.sh b/confluent_osdeploy/el8/profiles/default/scripts/setupssh.sh index f06c4d61..bc74faf5 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/setupssh.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/setupssh.sh @@ -1,8 +1,12 @@ #!/bin/sh -grep HostCert /etc/ssh/sshd_config.anaconda >> /mnt/sysimage/etc/ssh/sshd_config -echo HostbasedAuthentication yes >> /mnt/sysimage/etc/ssh/sshd_config -echo HostbasedUsesNameFromPacketOnly yes >> /mnt/sysimage/etc/ssh/sshd_config -echo IgnoreRhosts no >> /mnt/sysimage/etc/ssh/sshd_config +targssh=/mnt/sysimage/etc/ssh/sshd_config +if [ -d /mnt/sysimage/etc/ssh/sshd_config.d/ ]; then + targssh=/mnt/sysimage/etc/ssh/sshd_config.d/90-confluent.conf +fi +grep HostCert /etc/ssh/sshd_config.anaconda >> $targssh +echo HostbasedAuthentication yes >> $targssh +echo HostbasedUsesNameFromPacketOnly yes >> $targssh +echo IgnoreRhosts no >> $targssh sshconf=/mnt/sysimage/etc/ssh/ssh_config if [ -d /mnt/sysimage/etc/ssh/ssh_config.d/ ]; then sshconf=/mnt/sysimage/etc/ssh/ssh_config.d/01-confluent.conf From 2f8dfac9bce1e127d742f7cade64c3a3522369e1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 Mar 2024 08:45:23 -0500 Subject: [PATCH 098/319] Dump stderr to client if ansible had an utterly disastrous condition --- confluent_server/confluent/runansible.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/confluent_server/confluent/runansible.py b/confluent_server/confluent/runansible.py index cbbecc58..8e5d1a3d 100644 --- a/confluent_server/confluent/runansible.py +++ b/confluent_server/confluent/runansible.py @@ -63,6 +63,9 @@ class PlayRunner(object): else: textout += result['state'] + '\n' textout += '\n' + if self.stderr: + textout += "ERRORS **********************************\n" + textout += self.stderr return textout def dump_json(self): From 5ae3f4c62aa8b9df37f4cff335d089a1a3717363 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 Mar 2024 09:27:53 -0500 Subject: [PATCH 099/319] Properly address runansible error relay --- confluent_server/confluent/runansible.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/runansible.py b/confluent_server/confluent/runansible.py index 8e5d1a3d..50696742 100644 --- a/confluent_server/confluent/runansible.py +++ b/confluent_server/confluent/runansible.py @@ -32,6 +32,7 @@ anspypath = None running_status = {} class PlayRunner(object): def __init__(self, playfiles, nodes): + self.stderr = '' self.playfiles = playfiles self.nodes = nodes self.worker = None @@ -96,7 +97,8 @@ class PlayRunner(object): [mypath, __file__, targnodes, playfilename], stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, self.stderr = worker.communicate() + stdout, stder = worker.communicate() + self.stderr += stder.decode('utf8') current = memoryview(stdout) while len(current): sz = struct.unpack('=q', current[:8])[0] From 3ffeef5cf306d4c6040b898a0e1b7ad34a4e8a22 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 6 Mar 2024 16:27:09 -0500 Subject: [PATCH 100/319] Fix stray blank line at end of nodelist Wrong indentation level for nodelist resulting in spurious line. --- confluent_client/bin/nodelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist index 462ed922..c1b9c436 100755 --- a/confluent_client/bin/nodelist +++ b/confluent_client/bin/nodelist @@ -68,7 +68,7 @@ def main(): else: elem=(res['item']['href'].replace('/', '')) list.append(elem) - print(options.delim.join(list)) + print(options.delim.join(list)) sys.exit(exitcode) From cdefb400f9b1eaf94142d350fdc8f7c1006fac41 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 11 Mar 2024 13:32:45 -0400 Subject: [PATCH 101/319] Expose fingerprinting and better error handling to osdeploy This allows custom name and pre-import checking. --- confluent_client/confluent_env.sh | 4 ++-- confluent_server/bin/osdeploy | 35 +++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/confluent_client/confluent_env.sh b/confluent_client/confluent_env.sh index 81a70198..925a873d 100644 --- a/confluent_client/confluent_env.sh +++ b/confluent_client/confluent_env.sh @@ -153,11 +153,11 @@ _confluent_osimage_completion() { _confluent_get_args if [ $NUMARGS == 2 ]; then - COMPREPLY=($(compgen -W "initialize import updateboot rebase" -- ${COMP_WORDS[COMP_CWORD]})) + COMPREPLY=($(compgen -W "initialize import importcheck updateboot rebase" -- ${COMP_WORDS[COMP_CWORD]})) return elif [ ${CMPARGS[1]} == 'initialize' ]; then COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]})) - elif [ ${CMPARGS[1]} == 'import' ]; then + elif [ ${CMPARGS[1]} == 'import' ] || [ ${CMPARGS[1]} == 'importcheck' ]; then compopt -o default COMPREPLY=() return diff --git a/confluent_server/bin/osdeploy b/confluent_server/bin/osdeploy index fff220be..47ebc4a8 100644 --- a/confluent_server/bin/osdeploy +++ b/confluent_server/bin/osdeploy @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 __author__ = 'jjohnson2,bfinley' @@ -49,8 +49,11 @@ def main(args): wiz.add_argument('-p', help='Copy in TFTP contents required for PXE support', action='store_true') wiz.add_argument('-i', help='Interactively prompt for behaviors', action='store_true') wiz.add_argument('-l', help='Set up local management node to allow login from managed nodes', action='store_true') + osip = sp.add_parser('importcheck', help='Check import of an OS image from an ISO image') + osip.add_argument('imagefile', help='File to use for source of importing') osip = sp.add_parser('import', help='Import an OS image from an ISO image') osip.add_argument('imagefile', help='File to use for source of importing') + osip.add_argument('-n', help='Specific a custom distribution name') upb = sp.add_parser( 'updateboot', help='Push profile.yaml of the named profile data into boot assets as appropriate') @@ -63,7 +66,9 @@ def main(args): if cmdset.command == 'list': return oslist() if cmdset.command == 'import': - return osimport(cmdset.imagefile) + return osimport(cmdset.imagefile, custname=cmdset.n) + if cmdset.command == 'importcheck': + return osimport(cmdset.imagefile, checkonly=True) if cmdset.command == 'initialize': return initialize(cmdset) if cmdset.command == 'updateboot': @@ -496,7 +501,7 @@ def oslist(): print("") -def osimport(imagefile): +def osimport(imagefile, checkonly=False, custname=None): c = client.Command() imagefile = os.path.abspath(imagefile) if c.unixdomain: @@ -507,11 +512,33 @@ def osimport(imagefile): pass importing = False shortname = None - for rsp in c.create('/deployment/importing/', {'filename': imagefile}): + apipath = '/deployment/importing/' + if checkonly: + apipath = '/deployment/fingerprint/' + apiargs = {'filename': imagefile} + if custname: + apiargs['custname'] = custname + for rsp in c.create(apipath, apiargs): if 'target' in rsp: importing = True shortname = rsp['name'] print('Importing from {0} to {1}'.format(imagefile, rsp['target'])) + elif 'targetpath' in rsp: + tpath = rsp.get('targetpath', None) + tname = rsp.get('name', None) + oscat = rsp.get('oscategory', None) + if tpath: + print('Detected target directory: ' + tpath) + if tname: + print('Detected distribution name: ' + tname) + if oscat: + print('Detected OS category: ' + oscat) + for err in rsp.get('errors', []): + sys.stderr.write('Error: ' + err + '\n') + + elif 'error' in rsp: + sys.stderr.write(rsp['error'] + '\n') + sys.exit(rsp.get('errorcode', 1)) else: print(repr(rsp)) try: From 49e614eb32fe1cf3f2887932cf5b5d4b71092220 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 11 Mar 2024 17:10:33 -0400 Subject: [PATCH 102/319] Have image2disk delay exit on error Debugging cloning is difficult when system immediately reboots on error. --- .../el8-diskless/profiles/default/scripts/image2disk.py | 8 +++++++- .../el9-diskless/profiles/default/scripts/image2disk.py | 8 +++++++- .../profiles/default/scripts/image2disk.py | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py index aaaca9d4..655aaedc 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py @@ -10,6 +10,7 @@ import stat import struct import sys import subprocess +import traceback bootuuid = None @@ -426,4 +427,9 @@ def install_to_disk(imgpath): if __name__ == '__main__': - install_to_disk(os.environ['mountsrc']) + try: + install_to_disk(os.environ['mountsrc']) + except Exception: + traceback.print_exc() + time.sleep(86400) + raise diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py index 7b312a93..48a15767 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py @@ -10,6 +10,7 @@ import stat import struct import sys import subprocess +import traceback bootuuid = None @@ -426,4 +427,9 @@ def install_to_disk(imgpath): if __name__ == '__main__': - install_to_disk(os.environ['mountsrc']) + try: + install_to_disk(os.environ['mountsrc']) + except Exception: + traceback.print_exc() + time.sleep(86400) + raise diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py index 1d19ebad..91afc5cb 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py @@ -10,6 +10,7 @@ import stat import struct import sys import subprocess +import traceback bootuuid = None @@ -424,5 +425,10 @@ def install_to_disk(imgpath): if __name__ == '__main__': - install_to_disk(os.environ['mountsrc']) + try: + install_to_disk(os.environ['mountsrc']) + except Exception: + traceback.print_exc() + time.sleep(86400) + raise From 0d720baf2539a188d5b80cf4721bdcf5bcab66e8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 Mar 2024 09:36:40 -0400 Subject: [PATCH 103/319] Fix lldp when peername is null Some neighbors result in a null name, handle that. --- confluent_server/confluent/networking/lldp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/networking/lldp.py b/confluent_server/confluent/networking/lldp.py index e1fd8d4e..e181d46f 100644 --- a/confluent_server/confluent/networking/lldp.py +++ b/confluent_server/confluent/networking/lldp.py @@ -381,9 +381,10 @@ def list_info(parms, requestedparameter): break else: candidate = info[requestedparameter] - candidate = candidate.strip() - if candidate != '': - results.add(_api_sanitize_string(candidate)) + if candidate: + candidate = candidate.strip() + if candidate != '': + results.add(_api_sanitize_string(candidate)) return [msg.ChildCollection(x + suffix) for x in util.natural_sort(results)] def _handle_neighbor_query(pathcomponents, configmanager): From 17af9c74b81927601e84845965de421db9deb022 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 Mar 2024 15:32:44 -0400 Subject: [PATCH 104/319] Fix nodeapply redoing a single node multiple times --- confluent_client/bin/nodeapply | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/bin/nodeapply b/confluent_client/bin/nodeapply index e39447bc..2e798742 100755 --- a/confluent_client/bin/nodeapply +++ b/confluent_client/bin/nodeapply @@ -102,9 +102,9 @@ def run(): cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0] if currprocs < concurrentprocs: currprocs += 1 - run_cmdv(node, cmdv, all, pipedesc) + run_cmdv(sshnode, cmdv, all, pipedesc) else: - pendingexecs.append((node, cmdv)) + pendingexecs.append((sshnode, cmdv)) if not all or exitcode: sys.exit(exitcode) rdy, _, _ = select.select(all, [], [], 10) From 58d9bc1816101ac814beaa32d4da237e35aea9bc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 Mar 2024 10:50:01 -0400 Subject: [PATCH 105/319] Updates to confluent_selfcheck Reap ssh-agent to avoid stale agents lying around. Remove nuisance warnings about virbr0 when present. Do a full runthrough as the confluent user to ssh to a node when user requests with '-a', marking known_hosts and automation key issues. --- confluent_server/bin/confluent_selfcheck | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/confluent_server/bin/confluent_selfcheck b/confluent_server/bin/confluent_selfcheck index cc1409cf..1539434f 100755 --- a/confluent_server/bin/confluent_selfcheck +++ b/confluent_server/bin/confluent_selfcheck @@ -22,6 +22,8 @@ import shutil import eventlet.green.socket as socket import eventlet import greenlet +import pwd +import signal def fprint(txt): sys.stdout.write(txt) @@ -109,6 +111,8 @@ def nics_missing_ipv6(): iname, state = comps[:2] if iname == b'lo': continue + if iname == b'virbr0': + continue addrs = comps[2:] hasv6 = False hasv4 = False @@ -157,6 +161,7 @@ def lookup_node(node): if __name__ == '__main__': ap = argparse.ArgumentParser(description='Run configuration checks for a system running confluent service') ap.add_argument('-n', '--node', help='A node name to run node specific checks against') + ap.add_argument('-a', '--automation', help='Do checks against a deployed node for automation and syncfiles function', action='store_true') args, extra = ap.parse_known_args(sys.argv) if len(extra) > 1: ap.print_help() @@ -217,6 +222,7 @@ if __name__ == '__main__': print('OK') except subprocess.CalledProcessError: emprint('Failed to load confluent automation key, syncfiles and profile ansible plays will not work (Example resolution: osdeploy initialize -a)') + os.kill(int(sshutil.agent_pid), signal.SIGTERM) fprint('Checking for blocked insecure boot: ') if insecure_boot_attempts(): emprint('Some nodes are attempting network boot using PXE or HTTP boot, but the node is not configured to allow this (Example resolution: nodegroupattrib everything deployment.useinsecureprotocols=firmware)') @@ -311,6 +317,34 @@ if __name__ == '__main__': emprint('Name resolution failed for node, it is normally a good idea for the node name to resolve to an IP') if result: print("OK") + if args.automation: + print(f'Checking confluent automation access to {args.node}...') + child = os.fork() + if child > 0: + pid, extcode = os.waitpid(child, 0) + else: + sshutil.ready_keys = {} + sshutil.agent_pid = None + cuser = pwd.getpwnam('confluent') + os.setgid(cuser.pw_gid) + os.setuid(cuser.pw_uid) + sshutil.prep_ssh_key('/etc/confluent/ssh/automation') + srun = subprocess.run( + ['ssh', '-Tn', '-o', 'BatchMode=yes', '-l', 'root', + '-o', 'StrictHostKeyChecking=yes', args.node, 'true'], + stdin=subprocess.DEVNULL, stderr=subprocess.PIPE) + os.kill(int(sshutil.agent_pid), signal.SIGTERM) + if srun.returncode == 0: + print(f'Confluent automation access to {args.node} seems OK') + else: + if b'Host key verification failed' in srun.stderr: + emprint('Confluent ssh unable to verify host key, check /etc/ssh/ssh_known_hosts. (Example resolution: osdeploy initialize -k)') + elif b'ermission denied' in srun.stderr: + emprint('Confluent user unable to ssh in, check /root/.ssh/authorized_keys on the target system versus /etc/confluent/ssh/automation.pub (Example resolution: osdeploy initialize -a)') + else: + emprint('Unknown error attempting confluent automation ssh:') + sys.stderr.buffer.write(srun.stderr) + os.kill(int(sshutil.agent_pid), signal.SIGTERM) else: print("Skipping node checks, no node specified (Example: confluent_selfcheck -n n1)") # possible checks: From 876b59c1f0a2998ad58888f33c4fb099da5f7319 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 Mar 2024 10:52:52 -0400 Subject: [PATCH 106/319] Remove redundant kill on the agent pid Extraneous kill on the agent pid is removed. --- confluent_server/bin/confluent_selfcheck | 1 - 1 file changed, 1 deletion(-) diff --git a/confluent_server/bin/confluent_selfcheck b/confluent_server/bin/confluent_selfcheck index 1539434f..f558cf46 100755 --- a/confluent_server/bin/confluent_selfcheck +++ b/confluent_server/bin/confluent_selfcheck @@ -344,7 +344,6 @@ if __name__ == '__main__': else: emprint('Unknown error attempting confluent automation ssh:') sys.stderr.buffer.write(srun.stderr) - os.kill(int(sshutil.agent_pid), signal.SIGTERM) else: print("Skipping node checks, no node specified (Example: confluent_selfcheck -n n1)") # possible checks: From 1d4505ff3ca1916e1a4eeed5a7b3d886477c9c25 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 14 Mar 2024 11:20:36 -0400 Subject: [PATCH 107/319] SSH test by IP, to reflect actual usage and catch issues One issue is modified ssh_known_hosts wildcard customization failing to cover IP address. --- confluent_server/bin/confluent_selfcheck | 33 ++++++++++++++---------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/confluent_server/bin/confluent_selfcheck b/confluent_server/bin/confluent_selfcheck index f558cf46..b9651d17 100755 --- a/confluent_server/bin/confluent_selfcheck +++ b/confluent_server/bin/confluent_selfcheck @@ -280,13 +280,17 @@ if __name__ == '__main__': cfg = configmanager.ConfigManager(None) bootablev4nics = [] bootablev6nics = [] + targsships = [] for nic in glob.glob("/sys/class/net/*/ifindex"): idx = int(open(nic, "r").read()) nicname = nic.split('/')[-2] ncfg = netutil.get_nic_config(cfg, args.node, ifidx=idx) + if ncfg['ipv4_address']: + targsships.append(ncfg['ipv4_address']) if ncfg['ipv4_address'] or ncfg['ipv4_method'] == 'dhcp': bootablev4nics.append(nicname) if ncfg['ipv6_address']: + targsships.append(ncfg['ipv6_address']) bootablev6nics.append(nicname) if bootablev4nics: print("{} appears to have network configuration suitable for IPv4 deployment via: {}".format(args.node, ",".join(bootablev4nics))) @@ -329,21 +333,22 @@ if __name__ == '__main__': os.setgid(cuser.pw_gid) os.setuid(cuser.pw_uid) sshutil.prep_ssh_key('/etc/confluent/ssh/automation') - srun = subprocess.run( - ['ssh', '-Tn', '-o', 'BatchMode=yes', '-l', 'root', - '-o', 'StrictHostKeyChecking=yes', args.node, 'true'], - stdin=subprocess.DEVNULL, stderr=subprocess.PIPE) - os.kill(int(sshutil.agent_pid), signal.SIGTERM) - if srun.returncode == 0: - print(f'Confluent automation access to {args.node} seems OK') - else: - if b'Host key verification failed' in srun.stderr: - emprint('Confluent ssh unable to verify host key, check /etc/ssh/ssh_known_hosts. (Example resolution: osdeploy initialize -k)') - elif b'ermission denied' in srun.stderr: - emprint('Confluent user unable to ssh in, check /root/.ssh/authorized_keys on the target system versus /etc/confluent/ssh/automation.pub (Example resolution: osdeploy initialize -a)') + for targ in targsships: + srun = subprocess.run( + ['ssh', '-Tn', '-o', 'BatchMode=yes', '-l', 'root', + '-o', 'StrictHostKeyChecking=yes', targ, 'true'], + stdin=subprocess.DEVNULL, stderr=subprocess.PIPE) + if srun.returncode == 0: + print(f'Confluent automation access to {targ} seems OK') else: - emprint('Unknown error attempting confluent automation ssh:') - sys.stderr.buffer.write(srun.stderr) + if b'Host key verification failed' in srun.stderr: + emprint(f'Confluent ssh unable to verify host key for {targ}, check /etc/ssh/ssh_known_hosts. (Example resolution: osdeploy initialize -k)') + elif b'ermission denied' in srun.stderr: + emprint(f'Confluent user unable to ssh in to {targ}, check /root/.ssh/authorized_keys on the target system versus /etc/confluent/ssh/automation.pub (Example resolution: osdeploy initialize -a)') + else: + emprint('Unknown error attempting confluent automation ssh:') + sys.stderr.buffer.write(srun.stderr) + os.kill(int(sshutil.agent_pid), signal.SIGTERM) else: print("Skipping node checks, no node specified (Example: confluent_selfcheck -n n1)") # possible checks: From 789376029dc56c9a6bceb18c1cf0476f37bf1df5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 15 Mar 2024 09:57:23 -0400 Subject: [PATCH 108/319] Numerous fixes to the EL9 cloning Fix various callbacks when using IPv6 based deployment. Do not attempt to restore erroneously cloned zram partitions. Convert LVM names to new LVM names consistent with source naming scheme. Push new kernel command line into /boot/loader and /etc/kernel/cmdline. --- .../profiles/default/scripts/firstboot.sh | 2 +- .../profiles/default/scripts/image2disk.py | 99 ++++++++++++++++--- .../profiles/default/scripts/post.sh | 6 +- 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh index ed11d9e7..fabb9385 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh @@ -41,7 +41,7 @@ if [ ! -f /etc/confluent/firstboot.ran ]; then run_remote_config firstboot.d fi -curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus +curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_websrv/confluent-api/self/updatestatus systemctl disable firstboot rm /etc/systemd/system/firstboot.service rm /etc/confluent/firstboot.ran diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py index 48a15767..83cffc6b 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py @@ -13,6 +13,13 @@ import subprocess import traceback bootuuid = None +vgname = 'localstorage' +oldvgname = None + +def convert_lv(oldlvname): + if oldvgname is None: + return None + return oldlvname.replace(oldvgname, vgname) def get_partname(devname, idx): if devname[-1] in '0123456789': @@ -54,6 +61,8 @@ def get_image_metadata(imgpath): header = img.read(16) if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1': for md in get_multipart_image_meta(img): + if md.get('device', '').startswith('/dev/zram'): + continue yield md else: raise Exception('Installation from single part image not supported') @@ -87,14 +96,14 @@ def fixup(rootdir, vols): if tab.startswith('#ORIGFSTAB#'): if entry[1] in devbymount: targetdev = devbymount[entry[1]] - if targetdev.startswith('/dev/localstorage/'): + if targetdev.startswith('/dev/{}/'.format(vgname)): entry[0] = targetdev else: uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8') uuid = uuid.strip() entry[0] = 'UUID={}'.format(uuid) elif entry[2] == 'swap': - entry[0] = '/dev/mapper/localstorage-swap' + entry[0] = '/dev/mapper/{}-swap'.format(vgname) entry[0] = entry[0].ljust(42) entry[1] = entry[1].ljust(16) entry[3] = entry[3].ljust(28) @@ -142,6 +151,46 @@ def fixup(rootdir, vols): grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub') if not os.path.exists(grubsyscfg): grubsyscfg = os.path.join(rootdir, 'etc/default/grub') + kcmdline = os.path.join(rootdir, 'etc/kernel/cmdline') + if os.path.exists(kcmdline): + with open(kcmdline) as kcmdlinein: + kcmdlinecontent = kcmdlinein.read() + newkcmdlineent = [] + for ent in kcmdlinecontent.split(): + if ent.startswith('resume='): + newkcmdlineent.append('resume={}'.format(newswapdev)) + elif ent.startswith('root='): + newkcmdlineent.append('root={}'.format(newrootdev)) + elif ent.startswith('rd.lvm.lv='): + ent = convert_lv(ent) + if ent: + newkcmdlineent.append(ent) + else: + newkcmdlineent.append(ent) + with open(kcmdline, 'w') as kcmdlineout: + kcmdlineout.write(' '.join(newkcmdlineent) + '\n') + for loadent in glob.glob(os.path.join(rootdir, 'boot/loader/entries/*.conf')): + with open(loadent) as loadentin: + currentry = loadentin.read().split('\n') + with open(loadent, 'w') as loadentout: + for cfgline in currentry: + cfgparts = cfgline.split() + if not cfgparts or cfgparts[0] != 'options': + loadentout.write(cfgline + '\n') + continue + newcfgparts = [cfgparts[0]] + for cfgpart in cfgparts[1:]: + if cfgpart.startswith('root='): + newcfgparts.append('root={}'.format(newrootdev)) + elif cfgpart.startswith('resume='): + newcfgparts.append('resume={}'.format(newswapdev)) + elif cfgpart.startswith('rd.lvm.lv='): + cfgpart = convert_lv(cfgpart) + if cfgpart: + newcfgparts.append(cfgpart) + else: + newcfgparts.append(cfgpart) + loadentout.write(' '.join(newcfgparts) + '\n') with open(grubsyscfg) as defgrubin: defgrub = defgrubin.read().split('\n') with open(grubsyscfg, 'w') as defgrubout: @@ -149,9 +198,16 @@ def fixup(rootdir, vols): gline = gline.split() newline = [] for ent in gline: - if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'): - continue - newline.append(ent) + if ent.startswith('resume='): + newline.append('resume={}'.format(newswapdev)) + elif ent.startswith('root='): + newline.append('root={}'.format(newrootdev)) + elif ent.startswith('rd.lvm.lv='): + ent = convert_lv(ent) + if ent: + newline.append(ent) + else: + newline.append(ent) defgrubout.write(' '.join(newline) + '\n') grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/') grubcfg = grubcfg.split('\n') @@ -228,8 +284,14 @@ def had_swap(): return True return False +newrootdev = None +newswapdev = None def install_to_disk(imgpath): global bootuuid + global newrootdev + global newswapdev + global vgname + global oldvgname lvmvols = {} deftotsize = 0 mintotsize = 0 @@ -261,6 +323,12 @@ def install_to_disk(imgpath): biggestfs = fs biggestsize = fs['initsize'] if fs['device'].startswith('/dev/mapper'): + oldvgname = fs['device'].rsplit('/', 1)[-1] + if '_' in oldvgname and '-' in oldvgname.split('_')[-1]: + oldvgname = oldvgname.rsplit('-', 1)[0] + osname = oldvgname.split('_')[0] + nodename = socket.gethostname().split('.')[0] + vgname = '{}_{}'.format(osname, nodename) lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs deflvmsize += fs['initsize'] minlvmsize += fs['minsize'] @@ -305,6 +373,8 @@ def install_to_disk(imgpath): end = sectors parted.run('mkpart primary {}s {}s'.format(curroffset, end)) vol['targetdisk'] = get_partname(instdisk, volidx) + if vol['mount'] == '/': + newrootdev = vol['targetdisk'] curroffset += size + 1 if not lvmvols: if swapsize: @@ -314,13 +384,14 @@ def install_to_disk(imgpath): if end > sectors: end = sectors parted.run('mkpart swap {}s {}s'.format(curroffset, end)) - subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)]) + newswapdev = get_partname(instdisk, volidx + 1) + subprocess.check_call(['mkswap', newswapdev]) else: parted.run('mkpart lvm {}s 100%'.format(curroffset)) lvmpart = get_partname(instdisk, volidx + 1) subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart]) - subprocess.check_call(['vgcreate', 'localstorage', lvmpart]) - vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8') + subprocess.check_call(['vgcreate', vgname, lvmpart]) + vginfo = subprocess.check_output(['vgdisplay', vgname, '--units', 'b']).decode('utf8') vginfo = vginfo.split('\n') pesize = 0 pes = 0 @@ -347,13 +418,17 @@ def install_to_disk(imgpath): extents += 1 if vol['mount'] == '/': lvname = 'root' + else: lvname = vol['mount'].replace('/', '_') - subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage']) - vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname) + subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, vgname]) + vol['targetdisk'] = '/dev/{}/{}'.format(vgname, lvname) + if vol['mount'] == '/': + newrootdev = vol['targetdisk'] if swapsize: - subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage']) - subprocess.check_call(['mkswap', '/dev/localstorage/swap']) + subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', vgname]) + subprocess.check_call(['mkswap', '/dev/{}/swap'.format(vgname)]) + newswapdev = '/dev/{}/swap'.format(vgname) os.makedirs('/run/imginst/targ') for vol in allvols: with open(vol['targetdisk'], 'wb') as partition: diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh b/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh index 3b20a946..7a7ac01e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/post.sh @@ -23,9 +23,9 @@ exec 2>> /var/log/confluent/confluent-post.log chmod 600 /var/log/confluent/confluent-post.log tail -f /var/log/confluent/confluent-post.log > /dev/console & logshowpid=$! -curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service +curl -f https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service mkdir -p /opt/confluent/bin -curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh +curl -f https://$confluent_websrv/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh chmod +x /opt/confluent/bin/firstboot.sh systemctl enable firstboot selinuxpolicy=$(grep ^SELINUXTYPE /etc/selinux/config |awk -F= '{print $2}') @@ -40,7 +40,7 @@ run_remote_parts post.d # Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/ run_remote_config post.d -curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus +curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_websrv/confluent-api/self/updatestatus kill $logshowpid From bd2288ccb79b60fc4b12fbdc6975453c976091cc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 15 Mar 2024 12:29:37 -0400 Subject: [PATCH 109/319] Ensure preservation of " if rename fails If ent would swallow a ", make sure to put it back. --- .../el9-diskless/profiles/default/scripts/image2disk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py index 83cffc6b..79d0008e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py @@ -206,6 +206,8 @@ def fixup(rootdir, vols): ent = convert_lv(ent) if ent: newline.append(ent) + elif '""' in ent: + newline.append('""') else: newline.append(ent) defgrubout.write(' '.join(newline) + '\n') From 60fe306890a08a8fb128145d83de3a342e9cf77b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 15 Mar 2024 13:03:46 -0400 Subject: [PATCH 110/319] Numerous fixes Normalize cloning by wipefs prior to image2disk Have imgutil filter out zram mounts. Fix syncfiles error handling. --- .../el9-diskless/profiles/default/scripts/installimage | 1 + confluent_server/confluent/syncfiles.py | 5 +++-- imgutil/imgutil | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage b/confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage index 2e791ce6..56597086 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage @@ -30,6 +30,7 @@ if [ ! -f /sysroot/tmp/installdisk ]; then done fi lvm vgchange -a n +/sysroot/usr/sbin/wipefs -a /dev/$(cat /sysroot/tmp/installdisk) udevadm control -e if [ -f /sysroot/etc/lvm/devices/system.devices ]; then rm /sysroot/etc/lvm/devices/system.devices diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 6c11d072..70e5bdaf 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -193,6 +193,7 @@ def sync_list_to_node(sl, node, suffixes, peerip=None): targip = node if peerip: targip = peerip + #BOOO, need stderr!!! output = util.run( ['rsync', '-rvLD', targdir + '/', 'root@[{}]:/'.format(targip)])[0] except Exception as e: @@ -212,7 +213,7 @@ def sync_list_to_node(sl, node, suffixes, peerip=None): unreadablefiles.append(filename.replace(targdir, '')) if unreadablefiles: raise Exception("Syncing failed due to unreadable files: " + ','.join(unreadablefiles)) - elif b'Permission denied, please try again.' in e.stderr: + elif hasattr(e, 'stderr') and e.stderr and b'Permission denied, please try again.' in e.stderr: raise Exception('Syncing failed due to authentication error, is the confluent automation key not set up (osdeploy initialize -a) or is there some process replacing authorized_keys on the host?') else: raise @@ -231,7 +232,7 @@ def stage_ent(currmap, ent, targdir, appendexist=False): everyfent = [] allfents = ent.split() for tmpent in allfents: - fents = glob.glob(tmpent) + fents = glob.glob(tmpent) # TODO: recursive globbing? if not fents: raise Exception('No matching files for "{}"'.format(tmpent)) everyfent.extend(fents) diff --git a/imgutil/imgutil b/imgutil/imgutil index 959c4a17..022279cc 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -83,6 +83,8 @@ def get_partition_info(): dev, mount, fs, flags = entry.split()[:4] if mount not in capmounts: continue + if '/dev/zram' in dev: + continue fsinfo = os.statvfs(mount) partinfo = { 'mount': mount, From b157e55f000c401d6c0d9f5a3874a75073d1d265 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 15 Mar 2024 15:50:58 -0400 Subject: [PATCH 111/319] Fallback to unverified noderange on candidate manager check in PXE When doing pxe and the noderange of the candidate managers fails, try again without validation in case the user omitted collective members from nodelist, but still used ',' to enumerate them. --- confluent_server/confluent/discovery/protocols/pxe.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 6dd34efa..4a39654f 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -587,7 +587,10 @@ def get_deployment_profile(node, cfg, cfd=None): return None candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) if candmgrs: - candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: # fallback to unverified noderange + candmgrs = noderange.NodeRange(candmgrs).nodes if collective.get_myname() not in candmgrs: return None return profile From a595d31e946b1d9fee8ae3420465314148bb66a0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 18 Mar 2024 08:56:06 -0400 Subject: [PATCH 112/319] Explicitly invoke bash for ubuntu post Ubuntu really tries to use non-bash, explicitly use bash when we need it. --- .../ubuntu20.04/initramfs/custom-installation/post.sh | 3 ++- .../ubuntu22.04/initramfs/custom-installation/post.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/post.sh b/confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/post.sh index 5bd43bc6..d9dc27b2 100755 --- a/confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/post.sh +++ b/confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/post.sh @@ -4,4 +4,5 @@ confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}') confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}') export deploycfg confluent_mgr confluent_profile curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/post.sh -. /tmp/post.sh +bash /tmp/post.sh +true diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh index 5bd43bc6..d9dc27b2 100755 --- a/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh +++ b/confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh @@ -4,4 +4,5 @@ confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}') confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}') export deploycfg confluent_mgr confluent_profile curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/post.sh -. /tmp/post.sh +bash /tmp/post.sh +true From 3dd09b95e47ae238f24b3b437f466cc2f71623cd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 18 Mar 2024 09:13:53 -0400 Subject: [PATCH 113/319] Fix Ubuntu 20 pre script to match 22 --- .../profiles/default/scripts/pre.sh | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/pre.sh index ddfe598b..5db222a7 100755 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/pre.sh @@ -1,5 +1,16 @@ #!/bin/bash deploycfg=/custom-installation/confluent/confluent.deploycfg +mkdir -p /var/log/confluent +mkdir -p /opt/confluent/bin +mkdir -p /etc/confluent +cp /custom-installation/confluent/confluent.info /custom-installation/confluent/confluent.apikey /etc/confluent/ +cat /custom-installation/tls/*.pem >> /etc/confluent/ca.pem +cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin +cp $deploycfg /etc/confluent/ +( +exec >> /var/log/confluent/confluent-pre.log +exec 2>> /var/log/confluent/confluent-pre.log +chmod 600 /var/log/confluent/confluent-pre.log cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //') if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then @@ -23,7 +34,17 @@ echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf systemctl restart sshd +mkdir -p /etc/confluent +export nodename confluent_profile confluent_mgr +curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions +. /etc/confluent/functions +run_remote_parts pre.d curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes -curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk -python3 /custom-installation/getinstalldisk +if [ ! -e /tmp/installdisk ]; then + curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk + python3 /custom-installation/getinstalldisk +fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml +) & +tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console + From 6502573d905eae76b31b67369c8af71f69bdbaa3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 18 Mar 2024 09:15:11 -0400 Subject: [PATCH 114/319] Bring ubuntu 22 versions of firstboot and post to 20 --- .../profiles/default/scripts/firstboot.sh | 10 ++++++++-- .../ubuntu20.04/profiles/default/scripts/post.sh | 16 +++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh index d14269cf..c0ba44ab 100755 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh @@ -2,7 +2,10 @@ echo "Confluent first boot is running" HOME=$(getent passwd $(whoami)|cut -d: -f 6) export HOME -seems a potentially relevant thing to put i... by Jarrod Johnson +( +exec >> /target/var/log/confluent/confluent-firstboot.log +exec 2>> /target/var/log/confluent/confluent-firstboot.log +chmod 600 /target/var/log/confluent/confluent-firstboot.log cp -a /etc/confluent/ssh/* /etc/ssh/ systemctl restart sshd rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}') @@ -18,7 +21,10 @@ done hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}') touch /etc/cloud/cloud-init.disabled source /etc/confluent/functions - +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') +export confluent_mgr confluent_profile run_remote_parts firstboot.d run_remote_config firstboot.d curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus +) & +tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh index 16a624c3..d9730889 100755 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh @@ -8,7 +8,6 @@ chmod go-rwx /etc/confluent/* for i in /custom-installation/ssh/*.ca; do echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts done - cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/ cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/ sshconf=/target/etc/ssh/ssh_config @@ -19,10 +18,15 @@ echo 'Host *' >> $sshconf echo ' HostbasedAuthentication yes' >> $sshconf echo ' EnableSSHKeysign yes' >> $sshconf echo ' HostbasedKeyTypes *ed25519*' >> $sshconf - +cp /etc/confluent/functions /target/etc/confluent/functions +source /etc/confluent/functions +mkdir -p /target/var/log/confluent +cp /var/log/confluent/* /target/var/log/confluent/ +( +exec >> /target/var/log/confluent/confluent-post.log +exec 2>> /target/var/log/confluent/confluent-post.log +chmod 600 /target/var/log/confluent/confluent-post.log curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh -curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions -source /target/etc/confluent/functions chmod +x /target/etc/confluent/firstboot.sh cp /tmp/allnodes /target/root/.shosts cp /tmp/allnodes /target/etc/ssh/shosts.equiv @@ -84,6 +88,8 @@ chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d source /target/etc/confluent/functions run_remote_config post +python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus -d 'status: staged' umount /target/sys /target/dev /target/proc - +) & +tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console From 7a6b03097b53e60aeb0d9845075d283a61904219 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 18 Mar 2024 12:24:25 -0400 Subject: [PATCH 115/319] Fixup Ubuntu 22 ARM support --- .../ubuntu22.04/profiles/default/initprofile.sh | 9 ++++++++- confluent_server/confluent/osimage.py | 4 +--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh index 20e12471..28d7e74c 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh @@ -3,5 +3,12 @@ sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \ ln -s $1/casper/vmlinuz $2/boot/kernel && \ ln -s $1/casper/initrd $2/boot/initramfs/distribution && \ mkdir -p $2/boot/efi/boot && \ -ln -s $1/EFI/boot/* $2/boot/efi/boot +if [ -d $1/EFI/boot/ ]; then + ln -s $1/EFI/boot/* $2/boot/efi/boot +elif [ -d $1/efi/boot/ ]; then + ln -s $1/efi/boot/* $2/boot/efi/boot +else + echo "Unrecogrized boot contents in media" > &2 + exit 1 +fi diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 2289a048..e0c1a8cb 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -411,9 +411,7 @@ def check_ubuntu(isoinfo): ] return {'name': 'ubuntu-{0}-{1}'.format(ver, arch), 'method': EXTRACT|COPY, - 'extractlist': ['casper/vmlinuz', 'casper/initrd', - 'efi/boot/bootx64.efi', 'efi/boot/grubx64.efi' - ], + 'extractlist': exlist, 'copyto': 'install.iso', 'category': 'ubuntu{0}'.format(major)} From 5f801e6683481f32c82c483f60bf8d3bf2097088 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 18 Mar 2024 12:45:25 -0400 Subject: [PATCH 116/319] Correct syntax error in ubuntu arm profile init --- confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh index 28d7e74c..cebcd41d 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh @@ -8,7 +8,7 @@ if [ -d $1/EFI/boot/ ]; then elif [ -d $1/efi/boot/ ]; then ln -s $1/efi/boot/* $2/boot/efi/boot else - echo "Unrecogrized boot contents in media" > &2 + echo "Unrecogrized boot contents in media" >&2 exit 1 fi From 559e88b14459e8655180c11628acc59b0a472b85 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 19 Mar 2024 09:41:20 -0400 Subject: [PATCH 117/319] Correct vgname for hyphenated node names --- .../el9-diskless/profiles/default/scripts/image2disk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py index 79d0008e..425e5177 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py @@ -326,8 +326,9 @@ def install_to_disk(imgpath): biggestsize = fs['initsize'] if fs['device'].startswith('/dev/mapper'): oldvgname = fs['device'].rsplit('/', 1)[-1] + # if node has - then /dev/mapper will double up the hypen if '_' in oldvgname and '-' in oldvgname.split('_')[-1]: - oldvgname = oldvgname.rsplit('-', 1)[0] + oldvgname = oldvgname.rsplit('-', 1)[0].replace('--', '-') osname = oldvgname.split('_')[0] nodename = socket.gethostname().split('.')[0] vgname = '{}_{}'.format(osname, nodename) From 13fc5d9f37deabb6b38ea1595a3944ff58e90b32 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 20 Mar 2024 09:49:25 -0400 Subject: [PATCH 118/319] Capture better error data on failed syncfiles syncfiles can often hang up in unexpected ways, provide a catch-all. --- confluent_server/confluent/syncfiles.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 70e5bdaf..94b74eea 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -24,6 +24,7 @@ import confluent.noderange as noderange import eventlet import pwd import grp +import sys def mkdirp(path): try: @@ -193,9 +194,8 @@ def sync_list_to_node(sl, node, suffixes, peerip=None): targip = node if peerip: targip = peerip - #BOOO, need stderr!!! - output = util.run( - ['rsync', '-rvLD', targdir + '/', 'root@[{}]:/'.format(targip)])[0] + output, stderr = util.run( + ['rsync', '-rvLD', targdir + '/', 'root@[{}]:/'.format(targip)]) except Exception as e: if 'CalledProcessError' not in repr(e): # https://github.com/eventlet/eventlet/issues/413 @@ -215,6 +215,9 @@ def sync_list_to_node(sl, node, suffixes, peerip=None): raise Exception("Syncing failed due to unreadable files: " + ','.join(unreadablefiles)) elif hasattr(e, 'stderr') and e.stderr and b'Permission denied, please try again.' in e.stderr: raise Exception('Syncing failed due to authentication error, is the confluent automation key not set up (osdeploy initialize -a) or is there some process replacing authorized_keys on the host?') + elif hasattr(e, 'stderr') and e.stderr: + sys.stderr.write(e.stderr.decode('utf8')) + raise else: raise finally: From 5a7d98c6b81618db5f4e76a813e565550923bd62 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 21 Mar 2024 16:09:37 -0400 Subject: [PATCH 119/319] Enhance error reporting For one, when using confluent expressions, induce {} to be an error to trigger an error for someone trying to xargs something. Another is to add warnings when clear does something deliberately, but is something that might surprise a user, steering them toward what they possibly might want to do instead. --- confluent_client/bin/nodeattrib | 3 ++- confluent_client/confluent/client.py | 3 +++ .../confluent/config/configmanager.py | 20 ++++++++++++++++--- .../plugins/configuration/attributes.py | 18 ++++++++++------- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index f4b0331f..265fe917 100755 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -126,13 +126,14 @@ elif options.set: argset = argset.strip() if argset: arglist += shlex.split(argset) - argset = argfile.readline() + argset = argfile.readline() session.stop_if_noderange_over(noderange, options.maxnodes) exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None) if exitcode != 0: sys.exit(exitcode) # Lists all attributes + if len(args) > 0: # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank. if requestargs is None: diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py index ad29ff02..a7c13cd3 100644 --- a/confluent_client/confluent/client.py +++ b/confluent_client/confluent/client.py @@ -668,6 +668,9 @@ def updateattrib(session, updateargs, nodetype, noderange, options, dictassign=N for attrib in updateargs[1:]: keydata[attrib] = None for res in session.update(targpath, keydata): + for node in res.get('databynode', {}): + for warnmsg in res['databynode'][node].get('_warnings', []): + sys.stderr.write('Warning: ' + warnmsg + '\n') if 'error' in res: if 'errorcode' in res: exitcode = res['errorcode'] diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 9419e7fe..dce692fd 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -1089,6 +1089,11 @@ class _ExpressionFormat(string.Formatter): self._nodename = nodename self._numbers = None + def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, + auto_arg_index=False): + super()._vformat(format_string, args, kwargs, used_args, + recursion_depth, auto_arg_index) + def get_field(self, field_name, args, kwargs): return field_name, field_name @@ -2197,16 +2202,16 @@ class ConfigManager(object): self._notif_attribwatchers(changeset) self._bg_sync_to_file() - def clear_node_attributes(self, nodes, attributes): + def clear_node_attributes(self, nodes, attributes, warnings=None): if cfgleader: return exec_on_leader('_rpc_master_clear_node_attributes', self.tenant, nodes, attributes) if cfgstreams: exec_on_followers('_rpc_clear_node_attributes', self.tenant, nodes, attributes) - self._true_clear_node_attributes(nodes, attributes) + self._true_clear_node_attributes(nodes, attributes, warnings) - def _true_clear_node_attributes(self, nodes, attributes): + def _true_clear_node_attributes(self, nodes, attributes, warnings): # accumulate all changes into a changeset and push in one go changeset = {} realattributes = [] @@ -2229,8 +2234,17 @@ class ConfigManager(object): # delete it and check for inheritence to backfil data del nodek[attrib] self._do_inheritance(nodek, attrib, node, changeset) + if not warnings is None: + if attrib in nodek: + warnings.append('The attribute "{}" was defined specifically for the node and clearing now has a value inherited from the group "{}"'.format(attrib, nodek[attrib]['inheritedfrom'])) _addchange(changeset, node, attrib) _mark_dirtykey('nodes', node, self.tenant) + elif attrib in nodek: + if not warnings is None: + warnings.append('The attribute "{0}" is inherited from group "{1}", leaving the inherited value alone (use "{0}=" with no value to explicitly blank the value if desired)'.format(attrib, nodek[attrib]['inheritedfrom'])) + else: + if not warnings is None: + warnings.append('Attribute "{}" is either already cleared, or does not match a defined attribute (if referencing an attribute group, try a wildcard)'.format(attrib)) if ('_expressionkeys' in nodek and attrib in nodek['_expressionkeys']): recalcexpressions = True diff --git a/confluent_server/confluent/plugins/configuration/attributes.py b/confluent_server/confluent/plugins/configuration/attributes.py index c2ea83d9..a56a1aee 100644 --- a/confluent_server/confluent/plugins/configuration/attributes.py +++ b/confluent_server/confluent/plugins/configuration/attributes.py @@ -21,16 +21,16 @@ import confluent.util as util from fnmatch import fnmatch -def retrieve(nodes, element, configmanager, inputdata): +def retrieve(nodes, element, configmanager, inputdata, clearwarnbynode=None): configmanager.check_quorum() if nodes is not None: - return retrieve_nodes(nodes, element, configmanager, inputdata) + return retrieve_nodes(nodes, element, configmanager, inputdata, clearwarnbynode) elif element[0] == 'nodegroups': return retrieve_nodegroup( - element[1], element[3], configmanager, inputdata) + element[1], element[3], configmanager, inputdata, clearwarnbynode) -def retrieve_nodegroup(nodegroup, element, configmanager, inputdata): +def retrieve_nodegroup(nodegroup, element, configmanager, inputdata, clearwarnbynode): try: grpcfg = configmanager.get_nodegroup_attributes(nodegroup) except KeyError: @@ -106,10 +106,12 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata): raise Exception("BUGGY ATTRIBUTE FOR NODEGROUP") -def retrieve_nodes(nodes, element, configmanager, inputdata): +def retrieve_nodes(nodes, element, configmanager, inputdata, clearwarnbynode): attributes = configmanager.get_node_attributes(nodes) if element[-1] == 'all': for node in util.natural_sort(nodes): + if clearwarnbynode and node in clearwarnbynode: + yield msg.Attributes(node, {'_warnings': clearwarnbynode[node]}) theattrs = set(allattributes.node).union(set(attributes[node])) for attribute in sorted(theattrs): if attribute in attributes[node]: # have a setting for it @@ -266,6 +268,7 @@ def update_nodes(nodes, element, configmanager, inputdata): namemap[node] = rename['rename'] configmanager.rename_nodes(namemap) return yield_rename_resources(namemap, isnode=True) + clearwarnbynode = {} for node in nodes: updatenode = inputdata.get_attributes(node, allattributes.node) clearattribs = [] @@ -299,10 +302,11 @@ def update_nodes(nodes, element, configmanager, inputdata): markup = (e.text[:e.offset-1] + '-->' + e.text[e.offset-1] + '<--' + e.text[e.offset:]).strip() raise exc.InvalidArgumentException('Syntax error in attribute name: "{0}"'.format(markup)) if len(clearattribs) > 0: - configmanager.clear_node_attributes([node], clearattribs) + clearwarnbynode[node] = [] + configmanager.clear_node_attributes([node], clearattribs, warnings=clearwarnbynode[node]) updatedict[node] = updatenode try: configmanager.set_node_attributes(updatedict) except ValueError as e: raise exc.InvalidArgumentException(str(e)) - return retrieve(nodes, element, configmanager, inputdata) + return retrieve(nodes, element, configmanager, inputdata, clearwarnbynode) From 6ad0e773de4bafa489da4587e6829dc3cf67b413 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 21 Mar 2024 16:28:49 -0400 Subject: [PATCH 120/319] Actually have the vformat override return Performing the super() is hardly helpful if it doesn't actually copy the return behavior. --- confluent_server/confluent/config/configmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index dce692fd..9e7818b5 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -1091,8 +1091,8 @@ class _ExpressionFormat(string.Formatter): def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, auto_arg_index=False): - super()._vformat(format_string, args, kwargs, used_args, - recursion_depth, auto_arg_index) + return super()._vformat(format_string, args, kwargs, used_args, + recursion_depth, auto_arg_index) def get_field(self, field_name, args, kwargs): return field_name, field_name From 838c0920eb8e159a293e4ba3983d5a85c9fa042e Mon Sep 17 00:00:00 2001 From: tkucherera Date: Fri, 22 Mar 2024 11:37:12 -0400 Subject: [PATCH 121/319] l2traceroute --- confluent_client/doc/man/l2traceroute.ronn | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 confluent_client/doc/man/l2traceroute.ronn diff --git a/confluent_client/doc/man/l2traceroute.ronn b/confluent_client/doc/man/l2traceroute.ronn new file mode 100644 index 00000000..7e660094 --- /dev/null +++ b/confluent_client/doc/man/l2traceroute.ronn @@ -0,0 +1,34 @@ +l2traceroute(8) -- returns the layer 2 route through an Ethernet network managed by confluent given 2 end points. +============================== +## SYNOPSIS +`l2traceroute [options] ` + +## DESCRIPTION +**l2traceroute** is a command that returns the layer 2 route for the configered interfaces in nodeattrib. +It can also be used with the -i and -e options to check against specific interfaces on the endpoints. + +Note the net..switch attributes have to be set on the end points + + +## OPTIONS +* ` -e` EFACE, --eface=INTERFACE + interface to check against for the second end point +* ` -i` INTERFACE, --interface=INTERFACE + interface to check against for the first end point +* `-h`, `--help`: + Show help message and exit + + +## EXAMPLES + * Checking route between two nodes: + `# l2traceroute_client n244 n1851` + `n244 to n1851: ['switch114']` + +* Checking route from one node to multiple nodes: + `# l2traceroute_client n244 n1833,n1851` + `n244 to n1833: ['switch114', 'switch7', 'switch32', 'switch253', 'switch85', 'switch72', 'switch21', 'switch2', 'switch96', 'switch103', 'switch115'] + n244 to n1851: ['switch114']` + + + + From 466ed7496123c4808c767a272af37fd5a8814ac6 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Fri, 22 Mar 2024 11:37:51 -0400 Subject: [PATCH 122/319] l2traceroute --- confluent_client/bin/l2traceroute | 154 ++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100755 confluent_client/bin/l2traceroute diff --git a/confluent_client/bin/l2traceroute b/confluent_client/bin/l2traceroute new file mode 100755 index 00000000..7b9ad4ac --- /dev/null +++ b/confluent_client/bin/l2traceroute @@ -0,0 +1,154 @@ +#!/usr/libexec/platform-python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2017 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'tkucherera' + +import optparse +import os +import signal +import sys +import subprocess + +try: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) +except AttributeError: + pass +path = os.path.dirname(os.path.realpath(__file__)) +path = os.path.realpath(os.path.join(path, '..', 'lib', 'python')) +if path.startswith('/opt'): + sys.path.append(path) + +import confluent.client as client + +argparser = optparse.OptionParser( + usage="Usage: %prog -i -e ", +) +argparser.add_option('-i', '--interface', type='str', + help='interface to check path against for the start node') +argparser.add_option('-e', '--eface', type='str', + help='interface to check path against for the end node') + +(options, args) = argparser.parse_args() + + +session = client.Command() + +def get_neighbors(switch): + switch_neigbors = [] + url = 'networking/neighbors/by-switch/{0}/by-peername/'.format(switch) + for neighbor in session.read(url): + if neighbor['item']['href'].startswith('switch'): + switch = neighbor['item']['href'].strip('/') + switch_neigbors.append(switch) + return switch_neigbors + + + +def find_path(start, end, path=[]): + path = path + [start] + if start == end: + return path # If start and end are the same, return the path + + for node in get_neighbors(start): + if node not in path and node.startswith('switch'): + new_path = find_path(node, end, path) + if new_path: + return new_path # If a path is found, return it + + return None # If no path is found, return None + + + +def is_cumulus(switch): + try: + read_attrib = subprocess.check_output(['nodeattrib', switch, 'hardwaremanagement.method']) + except subprocess.CalledProcessError: + return False + for attribs in read_attrib.decode('utf-8').split('\n'): + if len(attribs.split(':')) > 1: + attrib = attribs.split(':') + if attrib[2].strip() == 'affluent': + return True + else: + return False + else: + return False + + +def host_to_switch(node, interface=None): + # first check the the node config to see what switches are connected + # if host is in rhel can use nmstate package + cummulus_switches = [] + netarg = 'net.*.switch' + if interface: + netarg = 'net.{0}.switch'.format(interface) + read_attrib = subprocess.check_output(['nodeattrib', node, netarg]) + for attribs in read_attrib.decode('utf-8').split('\n'): + attrib = attribs.split(':') + try: + if ' net.mgt.switch' in attrib or attrib[2] == '': + continue + except IndexError: + continue + switch = attrib[2].strip() + if is_cumulus(switch): + cummulus_switches.append(switch) + return cummulus_switches + +try: + start_node = args[0] + end_node = args[1] + interface = options.interface + eface = options.eface +except IndexError: + argparser.print_help() + sys.exit(1) + +def path_between_nodes(start_switches, end_switches): + for start_switch in start_switches: + for end_switch in end_switches: + if start_switch == end_switch: + return [start_switch] + else: + path = find_path(start_switch, end_switch) + if path: + return path + else: + return 'No path found' + +end_nodeslist = [] +nodelist = '/noderange/{0}/nodes/'.format(end_node) +for res in session.read(nodelist): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + else: + elem=(res['item']['href'].replace('/', '')) + end_nodeslist.append(elem) + +start_switches = host_to_switch(start_node, interface) +for end_node in end_nodeslist: + if end_node: + end_switches = host_to_switch(end_node, eface) + path = path_between_nodes(start_switches, end_switches) + print(f'{start_node} to {end_node}: {path}') + + + + + + From c60bf68cbc34c6f389b53ef0a8c69086f7b555dd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 22 Mar 2024 12:56:09 -0400 Subject: [PATCH 123/319] Logout prior to renaming user Some firmware cannot tolerate a web session being active during a rename. Make sure logout has been done, and give a retry if needed to let the session close out after logging out. --- .../confluent/discovery/handlers/xcc.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index 23f31fb0..ff7ca042 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -247,6 +247,10 @@ class NodeHandler(immhandler.NodeHandler): if rsp.status == 200: pwdchanged = True password = newpassword + wc.set_header('Authorization', 'Bearer ' + rspdata['access_token']) + if '_csrf_token' in wc.cookies: + wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token']) + wc.grab_json_response_with_status('/api/providers/logout') else: if rspdata.get('locktime', 0) > 0: raise LockedUserException( @@ -280,6 +284,7 @@ class NodeHandler(immhandler.NodeHandler): rsp.read() if rsp.status != 200: return (None, None) + wc.grab_json_response_with_status('/api/providers/logout') self._currcreds = (username, newpassword) wc.set_basic_credentials(username, newpassword) pwdchanged = True @@ -434,6 +439,7 @@ class NodeHandler(immhandler.NodeHandler): '/api/function', {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)}) if status == 200 and rsp.get('return', 0) == 13: + wc.grab_json_response('/api/providers/logout') wc.set_basic_credentials(self._currcreds[0], self._currcreds[1]) status = 503 while status != 200: @@ -442,10 +448,13 @@ class NodeHandler(immhandler.NodeHandler): {'UserName': username}, method='PATCH') if status != 200: rsp = json.loads(rsp) - if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError'): - eventlet.sleep(10) + if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'): + eventlet.sleep(4) else: break + self.tmppasswd = None + self._currcreds = (username, passwd) + return self.tmppasswd = None wc.grab_json_response('/api/providers/logout') self._currcreds = (username, passwd) @@ -632,3 +641,4 @@ def remote_nodecfg(nodename, cfm): info = {'addresses': [ipaddr]} nh = NodeHandler(info, cfm) nh.config(nodename) + From 296a0e88b4962f5e93e43f954414fb3c5532b6d8 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Sun, 24 Mar 2024 11:41:23 -0400 Subject: [PATCH 124/319] making the use cases more generic --- confluent_client/bin/l2traceroute | 65 ++++++++++++++-------- confluent_client/doc/man/l2traceroute.ronn | 6 +- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/confluent_client/bin/l2traceroute b/confluent_client/bin/l2traceroute index 7b9ad4ac..e8f9705e 100755 --- a/confluent_client/bin/l2traceroute +++ b/confluent_client/bin/l2traceroute @@ -41,38 +41,44 @@ argparser.add_option('-i', '--interface', type='str', help='interface to check path against for the start node') argparser.add_option('-e', '--eface', type='str', help='interface to check path against for the end node') +argparser.add_option('-c', '--cumulus', action="store_true", dest="cumulus", + help='return layer 2 route through cumulus switches only') (options, args) = argparser.parse_args() +try: + start_node = args[0] + end_node = args[1] + interface = options.interface + eface = options.eface +except IndexError: + argparser.print_help() + sys.exit(1) session = client.Command() def get_neighbors(switch): switch_neigbors = [] - url = 'networking/neighbors/by-switch/{0}/by-peername/'.format(switch) + url = '/networking/neighbors/by-switch/{0}/by-peername/'.format(switch) for neighbor in session.read(url): - if neighbor['item']['href'].startswith('switch'): - switch = neighbor['item']['href'].strip('/') + switch = neighbor['item']['href'].strip('/') + if switch in all_switches: switch_neigbors.append(switch) return switch_neigbors - - def find_path(start, end, path=[]): path = path + [start] if start == end: return path # If start and end are the same, return the path for node in get_neighbors(start): - if node not in path and node.startswith('switch'): + if node not in path: new_path = find_path(node, end, path) if new_path: return new_path # If a path is found, return it return None # If no path is found, return None - - def is_cumulus(switch): try: read_attrib = subprocess.check_output(['nodeattrib', switch, 'hardwaremanagement.method']) @@ -92,11 +98,16 @@ def is_cumulus(switch): def host_to_switch(node, interface=None): # first check the the node config to see what switches are connected # if host is in rhel can use nmstate package - cummulus_switches = [] + if node in all_switches: + return [node] + switches = [] netarg = 'net.*.switch' if interface: netarg = 'net.{0}.switch'.format(interface) - read_attrib = subprocess.check_output(['nodeattrib', node, netarg]) + try: + read_attrib = subprocess.check_output(['nodeattrib', node, netarg]) + except subprocess.CalledProcessError: + return False for attribs in read_attrib.decode('utf-8').split('\n'): attrib = attribs.split(':') try: @@ -105,18 +116,11 @@ def host_to_switch(node, interface=None): except IndexError: continue switch = attrib[2].strip() - if is_cumulus(switch): - cummulus_switches.append(switch) - return cummulus_switches - -try: - start_node = args[0] - end_node = args[1] - interface = options.interface - eface = options.eface -except IndexError: - argparser.print_help() - sys.exit(1) + if is_cumulus(switch) and options.cumulus: + switches.append(switch) + else: + switches.append(switch) + return switches def path_between_nodes(start_switches, end_switches): for start_switch in start_switches: @@ -129,7 +133,17 @@ def path_between_nodes(start_switches, end_switches): return path else: return 'No path found' - + + +all_switches = [] +for res in session.read('/networking/neighbors/by-switch/'): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + else: + switch = (res['item']['href'].replace('/', '')) + all_switches.append(switch) + end_nodeslist = [] nodelist = '/noderange/{0}/nodes/'.format(end_node) for res in session.read(nodelist): @@ -144,9 +158,14 @@ start_switches = host_to_switch(start_node, interface) for end_node in end_nodeslist: if end_node: end_switches = host_to_switch(end_node, eface) + if not end_switches: + print('Error: net.{0}.switch attribute is not valid') + continue path = path_between_nodes(start_switches, end_switches) print(f'{start_node} to {end_node}: {path}') +# TODO dont put switches that are connected through management interfaces. + diff --git a/confluent_client/doc/man/l2traceroute.ronn b/confluent_client/doc/man/l2traceroute.ronn index 7e660094..16318567 100644 --- a/confluent_client/doc/man/l2traceroute.ronn +++ b/confluent_client/doc/man/l2traceroute.ronn @@ -7,7 +7,9 @@ l2traceroute(8) -- returns the layer 2 route through an Ethernet network managed **l2traceroute** is a command that returns the layer 2 route for the configered interfaces in nodeattrib. It can also be used with the -i and -e options to check against specific interfaces on the endpoints. -Note the net..switch attributes have to be set on the end points + +## PREREQUISITES +**l2traceroute** the net..switch attributes have to be set on the end points if endpoint is not a switch ## OPTIONS @@ -15,6 +17,8 @@ Note the net..switch attributes have to be set on the end points interface to check against for the second end point * ` -i` INTERFACE, --interface=INTERFACE interface to check against for the first end point +* ` -c` CUMULUS, --cumulus=CUMULUS + return layer 2 route through cumulus switches only * `-h`, `--help`: Show help message and exit From f7a2e51f9c284ab9c42d7c28b62b76df4ed0e711 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 26 Mar 2024 10:31:29 -0400 Subject: [PATCH 125/319] fstab fixup for hyphenated lvm vg names --- .../el9-diskless/profiles/default/scripts/image2disk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py index 425e5177..6a924964 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py @@ -103,7 +103,7 @@ def fixup(rootdir, vols): uuid = uuid.strip() entry[0] = 'UUID={}'.format(uuid) elif entry[2] == 'swap': - entry[0] = '/dev/mapper/{}-swap'.format(vgname) + entry[0] = '/dev/mapper/{}-swap'.format(vgname.replace('-', '--')) entry[0] = entry[0].ljust(42) entry[1] = entry[1].ljust(16) entry[3] = entry[3].ljust(28) From f1d3e47439f901075f2d02728f478705d88feac7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 22 Mar 2024 12:56:09 -0400 Subject: [PATCH 126/319] Logout prior to renaming user Some firmware cannot tolerate a web session being active during a rename. Make sure logout has been done, and give a retry if needed to let the session close out after logging out. --- .../confluent/discovery/handlers/xcc.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index 23f31fb0..ff7ca042 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -247,6 +247,10 @@ class NodeHandler(immhandler.NodeHandler): if rsp.status == 200: pwdchanged = True password = newpassword + wc.set_header('Authorization', 'Bearer ' + rspdata['access_token']) + if '_csrf_token' in wc.cookies: + wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token']) + wc.grab_json_response_with_status('/api/providers/logout') else: if rspdata.get('locktime', 0) > 0: raise LockedUserException( @@ -280,6 +284,7 @@ class NodeHandler(immhandler.NodeHandler): rsp.read() if rsp.status != 200: return (None, None) + wc.grab_json_response_with_status('/api/providers/logout') self._currcreds = (username, newpassword) wc.set_basic_credentials(username, newpassword) pwdchanged = True @@ -434,6 +439,7 @@ class NodeHandler(immhandler.NodeHandler): '/api/function', {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)}) if status == 200 and rsp.get('return', 0) == 13: + wc.grab_json_response('/api/providers/logout') wc.set_basic_credentials(self._currcreds[0], self._currcreds[1]) status = 503 while status != 200: @@ -442,10 +448,13 @@ class NodeHandler(immhandler.NodeHandler): {'UserName': username}, method='PATCH') if status != 200: rsp = json.loads(rsp) - if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError'): - eventlet.sleep(10) + if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'): + eventlet.sleep(4) else: break + self.tmppasswd = None + self._currcreds = (username, passwd) + return self.tmppasswd = None wc.grab_json_response('/api/providers/logout') self._currcreds = (username, passwd) @@ -632,3 +641,4 @@ def remote_nodecfg(nodename, cfm): info = {'addresses': [ipaddr]} nh = NodeHandler(info, cfm) nh.config(nodename) + From e38cd5d3e56ef0621f5449501719c810975ff096 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 15 Mar 2024 15:50:58 -0400 Subject: [PATCH 127/319] Fallback to unverified noderange on candidate manager check in PXE When doing pxe and the noderange of the candidate managers fails, try again without validation in case the user omitted collective members from nodelist, but still used ',' to enumerate them. --- confluent_server/confluent/discovery/protocols/pxe.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index a9a07963..ff473ebc 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -587,7 +587,10 @@ def get_deployment_profile(node, cfg, cfd=None): return None candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) if candmgrs: - candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: # fallback to unverified noderange + candmgrs = noderange.NodeRange(candmgrs).nodes if collective.get_myname() not in candmgrs: return None return profile From 19e9c6910d609f856cbc0422271e108cea5839e0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 Mar 2024 15:32:44 -0400 Subject: [PATCH 128/319] Fix nodeapply redoing a single node multiple times --- confluent_client/bin/nodeapply | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/bin/nodeapply b/confluent_client/bin/nodeapply index e39447bc..2e798742 100755 --- a/confluent_client/bin/nodeapply +++ b/confluent_client/bin/nodeapply @@ -102,9 +102,9 @@ def run(): cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0] if currprocs < concurrentprocs: currprocs += 1 - run_cmdv(node, cmdv, all, pipedesc) + run_cmdv(sshnode, cmdv, all, pipedesc) else: - pendingexecs.append((node, cmdv)) + pendingexecs.append((sshnode, cmdv)) if not all or exitcode: sys.exit(exitcode) rdy, _, _ = select.select(all, [], [], 10) From ac1f7c57b606674a1e05788d382e470349ef53b7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 Mar 2024 09:36:40 -0400 Subject: [PATCH 129/319] Fix lldp when peername is null Some neighbors result in a null name, handle that. --- confluent_server/confluent/networking/lldp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/networking/lldp.py b/confluent_server/confluent/networking/lldp.py index e1fd8d4e..e181d46f 100644 --- a/confluent_server/confluent/networking/lldp.py +++ b/confluent_server/confluent/networking/lldp.py @@ -381,9 +381,10 @@ def list_info(parms, requestedparameter): break else: candidate = info[requestedparameter] - candidate = candidate.strip() - if candidate != '': - results.add(_api_sanitize_string(candidate)) + if candidate: + candidate = candidate.strip() + if candidate != '': + results.add(_api_sanitize_string(candidate)) return [msg.ChildCollection(x + suffix) for x in util.natural_sort(results)] def _handle_neighbor_query(pathcomponents, configmanager): From c1afc144cb32d900fe4b9962e7581bd3612174de Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Feb 2024 15:05:56 -0500 Subject: [PATCH 130/319] Change to unix domain for vtbuffer communication The semaphore arbitrated single channel sharing was proving to be too slow. Make the communication lockless by having dedicated sockets per request. --- confluent_server/confluent/consoleserver.py | 56 +++--- confluent_vtbufferd/vtbufferd.c | 179 +++++++++++++++----- 2 files changed, 161 insertions(+), 74 deletions(-) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index 37274792..783d77de 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -62,39 +62,38 @@ def chunk_output(output, n): yield output[i:i + n] def get_buffer_output(nodename): - out = _bufferdaemon.stdin - instream = _bufferdaemon.stdout + out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) + out.connect("\x00confluent-vtbuffer") if not isinstance(nodename, bytes): nodename = nodename.encode('utf8') outdata = bytearray() - with _bufferlock: - out.write(struct.pack('I', len(nodename))) - out.write(nodename) - out.flush() - select.select((instream,), (), (), 30) - while not outdata or outdata[-1]: - try: - chunk = os.read(instream.fileno(), 128) - except IOError: - chunk = None - if chunk: - outdata.extend(chunk) - else: - select.select((instream,), (), (), 0) - return bytes(outdata[:-1]) + out.send(struct.pack('I', len(nodename))) + out.send(nodename) + select.select((out,), (), (), 30) + while not outdata or outdata[-1]: + try: + chunk = os.read(out.fileno(), 128) + except IOError: + chunk = None + if chunk: + outdata.extend(chunk) + else: + select.select((out,), (), (), 0) + return bytes(outdata[:-1]) def send_output(nodename, output): if not isinstance(nodename, bytes): nodename = nodename.encode('utf8') - with _bufferlock: - _bufferdaemon.stdin.write(struct.pack('I', len(nodename) | (1 << 29))) - _bufferdaemon.stdin.write(nodename) - _bufferdaemon.stdin.flush() - for chunk in chunk_output(output, 8192): - _bufferdaemon.stdin.write(struct.pack('I', len(chunk) | (2 << 29))) - _bufferdaemon.stdin.write(chunk) - _bufferdaemon.stdin.flush() + out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) + out.connect("\x00confluent-vtbuffer") + out.send(struct.pack('I', len(nodename) | (1 << 29))) + out.send(nodename) + for chunk in chunk_output(output, 8192): + out.send(struct.pack('I', len(chunk) | (2 << 29))) + out.send(chunk) def _utf8_normalize(data, decoder): # first we give the stateful decoder a crack at the byte stream, @@ -604,11 +603,8 @@ def initialize(): _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen( - ['/opt/confluent/bin/vtbufferd'], bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL) - fcntl.fcntl(_bufferdaemon.stdout.fileno(), - fcntl.F_SETFL, fl | os.O_NONBLOCK) + ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) def start_console_sessions(): configmodule.hook_new_configmanagers(_start_tenant_sessions) diff --git a/confluent_vtbufferd/vtbufferd.c b/confluent_vtbufferd/vtbufferd.c index e89269b4..055a5263 100644 --- a/confluent_vtbufferd/vtbufferd.c +++ b/confluent_vtbufferd/vtbufferd.c @@ -1,8 +1,14 @@ +#include +#define _GNU_SOURCE #include #include #include #include #include +#include +#include +#include +#include #include "tmt.h" #define HASHSIZE 2053 #define MAXNAMELEN 256 @@ -10,13 +16,17 @@ struct terment { struct terment *next; char *name; + int fd; TMT *vt; }; #define SETNODE 1 #define WRITE 2 #define READBUFF 0 +#define CLOSECONN 3 +#define MAXEVTS 16 static struct terment *buffers[HASHSIZE]; +static char* nodenames[HASHSIZE]; unsigned long hash(char *str) /* djb2a */ @@ -37,10 +47,13 @@ TMT *get_termentbyname(char *name) { return NULL; } -TMT *set_termentbyname(char *name) { +TMT *set_termentbyname(char *name, int fd) { struct terment *ret; int idx; + if (nodenames[fd] == NULL) { + nodenames[fd] = strdup(name); + } idx = hash(name); for (ret = buffers[idx]; ret != NULL; ret = ret->next) if (strcmp(name, ret->name) == 0) @@ -48,12 +61,13 @@ TMT *set_termentbyname(char *name) { ret = (struct terment *)malloc(sizeof(*ret)); ret->next = buffers[idx]; ret->name = strdup(name); + ret->fd = fd; ret->vt = tmt_open(31, 100, NULL, NULL, L"→←↑↓■◆▒°±▒┘┐┌└┼⎺───⎽├┤┴┬│≤≥π≠£•"); buffers[idx] = ret; return ret->vt; } -void dump_vt(TMT* outvt) { +void dump_vt(TMT* outvt, int outfd) { const TMTSCREEN *out = tmt_screen(outvt); const TMTPOINT *curs = tmt_cursor(outvt); int line, idx, maxcol, maxrow; @@ -67,9 +81,10 @@ void dump_vt(TMT* outvt) { tmt_color_t fg = TMT_COLOR_DEFAULT; tmt_color_t bg = TMT_COLOR_DEFAULT; wchar_t sgrline[30]; + char strbuffer[128]; size_t srgidx = 0; char colorcode = 0; - wprintf(L"\033c"); + write(outfd, "\033c", 2); maxcol = 0; maxrow = 0; for (line = out->nline - 1; line >= 0; --line) { @@ -148,60 +163,136 @@ void dump_vt(TMT* outvt) { } if (sgrline[0] != 0) { sgrline[wcslen(sgrline) - 1] = 0; // Trim last ; - wprintf(L"\033[%lsm", sgrline); + + snprintf(strbuffer, sizeof(strbuffer), "\033[%lsm", sgrline); + write(outfd, strbuffer, strlen(strbuffer)); + write(outfd, "\033[]", 3); } - wprintf(L"%lc", out->lines[line]->chars[idx].c); + snprintf(strbuffer, sizeof(strbuffer), "%lc", out->lines[line]->chars[idx].c); + write(outfd, strbuffer, strlen(strbuffer)); } if (line < maxrow) - wprintf(L"\r\n"); + write(outfd, "\r\n", 2); } - fflush(stdout); - wprintf(L"\x1b[%ld;%ldH", curs->r + 1, curs->c + 1); - fflush(stdout); + //fflush(stdout); + snprintf(strbuffer, sizeof(strbuffer), "\x1b[%ld;%ldH", curs->r + 1, curs->c + 1); + write(outfd, strbuffer, strlen(strbuffer)); + //fflush(stdout); +} + +int handle_traffic(int fd) { + int cmd, length; + char currnode[MAXNAMELEN]; + char cmdbuf[MAXDATALEN]; + char *nodename; + TMT *currvt = NULL; + TMT *outvt = NULL; + length = read(fd, &cmd, 4); + if (length <= 0) { + return 0; + } + length = cmd & 536870911; + cmd = cmd >> 29; + if (cmd == SETNODE) { + cmd = read(fd, currnode, length); + currnode[length] = 0; + if (cmd < 0) + return 0; + currvt = set_termentbyname(currnode, fd); + } else if (cmd == WRITE) { + if (currvt == NULL) { + nodename = nodenames[fd]; + currvt = set_termentbyname(nodename, fd); + } + cmd = read(fd, cmdbuf, length); + cmdbuf[length] = 0; + if (cmd < 0) + return 0; + tmt_write(currvt, cmdbuf, length); + } else if (cmd == READBUFF) { + cmd = read(fd, cmdbuf, length); + cmdbuf[length] = 0; + if (cmd < 0) + return 0; + outvt = get_termentbyname(cmdbuf); + if (outvt != NULL) + dump_vt(outvt, fd); + length = write(fd, "\x00", 1); + if (length < 0) + return 0; + } else if (cmd == CLOSECONN) { + return 0; + } + return 1; } int main(int argc, char* argv[]) { - int cmd, length; setlocale(LC_ALL, ""); - char cmdbuf[MAXDATALEN]; - char currnode[MAXNAMELEN]; - TMT *currvt = NULL; - TMT *outvt = NULL; + struct sockaddr_un addr; + int numevts; + int status; + int poller; + int n; + socklen_t len; + int ctlsock, currsock; + socklen_t addrlen; + struct ucred ucr; + + struct epoll_event epvt, evts[MAXEVTS]; stdin = freopen(NULL, "rb", stdin); if (stdin == NULL) { exit(1); } + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path + 1, argv[1], sizeof(addr.sun_path) - 2); // abstract namespace socket + ctlsock = socket(AF_UNIX, SOCK_STREAM, 0); + status = bind(ctlsock, (const struct sockaddr*)&addr, sizeof(sa_family_t) + strlen(argv[1]) + 1); //sizeof(struct sockaddr_un)); + if (status < 0) { + perror("Unable to open unix socket - "); + exit(1); + } + listen(ctlsock, 128); + poller = epoll_create(1); + memset(&epvt, 0, sizeof(struct epoll_event)); + epvt.events = EPOLLIN; + epvt.data.fd = ctlsock; + if (epoll_ctl(poller, EPOLL_CTL_ADD, ctlsock, &epvt) < 0) { + perror("Unable to poll the socket"); + exit(1); + } + // create a unix domain socket for accepting, each connection is only allowed to either read or write, not both while (1) { - length = fread(&cmd, 4, 1, stdin); - if (length < 0) - continue; - length = cmd & 536870911; - cmd = cmd >> 29; - if (cmd == SETNODE) { - cmd = fread(currnode, 1, length, stdin); - currnode[length] = 0; - if (cmd < 0) - continue; - currvt = set_termentbyname(currnode); - } else if (cmd == WRITE) { - if (currvt == NULL) - currvt = set_termentbyname(""); - cmd = fread(cmdbuf, 1, length, stdin); - cmdbuf[length] = 0; - if (cmd < 0) - continue; - tmt_write(currvt, cmdbuf, length); - } else if (cmd == READBUFF) { - cmd = fread(cmdbuf, 1, length, stdin); - cmdbuf[length] = 0; - if (cmd < 0) - continue; - outvt = get_termentbyname(cmdbuf); - if (outvt != NULL) - dump_vt(outvt); - length = write(1, "\x00", 1); - if (length < 0) - continue; + numevts = epoll_wait(poller, evts, MAXEVTS, -1); + if (numevts < 0) { + perror("Failed wait"); + exit(1); + } + for (n = 0; n < numevts; ++n) { + if (evts[n].data.fd == ctlsock) { + currsock = accept(ctlsock, (struct sockaddr *) &addr, &addrlen); + len = sizeof(ucr); + getsockopt(currsock, SOL_SOCKET, SO_PEERCRED, &ucr, &len); + if (ucr.uid != getuid()) { // block access for other users + close(currsock); + continue; + } + memset(&epvt, 0, sizeof(struct epoll_event)); + epvt.events = EPOLLIN; + epvt.data.fd = currsock; + epoll_ctl(poller, EPOLL_CTL_ADD, currsock, &epvt); + } else { + if (!handle_traffic(evts[n].data.fd)) { + epoll_ctl(poller, EPOLL_CTL_DEL, evts[n].data.fd, NULL); + close(evts[n].data.fd); + if (nodenames[evts[n].data.fd] != NULL) { + free(nodenames[evts[n].data.fd]); + nodenames[evts[n].data.fd] = NULL; + } + } + } } } } + + From b3b3627bf926c3f18e0578fd3f8033463af4855a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Feb 2024 15:07:12 -0500 Subject: [PATCH 131/319] Remove disused bufferlock We no longer use a lock on buffer communication, eliminate the stale variable. --- confluent_server/confluent/consoleserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index 783d77de..7b9530f2 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -49,7 +49,6 @@ _handled_consoles = {} _tracelog = None _bufferdaemon = None -_bufferlock = None try: range = xrange @@ -599,8 +598,6 @@ def _start_tenant_sessions(cfm): def initialize(): global _tracelog global _bufferdaemon - global _bufferlock - _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen( ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL, From d183a3f99cb452c2a8e0a38dcc9a2c30bb834db4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 1 Feb 2024 08:50:44 -0500 Subject: [PATCH 132/319] Fix problem where one multicast/broadcast attempt could tank other interfaces Carrying over change from ssdp, ignore failures on transmit, particularly if firewall --- confluent_server/confluent/discovery/protocols/slp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/slp.py b/confluent_server/confluent/discovery/protocols/slp.py index e42c1577..ac332def 100644 --- a/confluent_server/confluent/discovery/protocols/slp.py +++ b/confluent_server/confluent/discovery/protocols/slp.py @@ -246,11 +246,11 @@ def _find_srvtype(net, net4, srvtype, addresses, xid): try: net4.sendto(data, ('239.255.255.253', 427)) except socket.error as se: - # On occasion, multicasting may be disabled - # tolerate this scenario and move on - if se.errno != 101: - raise - net4.sendto(data, (bcast, 427)) + pass + try: + net4.sendto(data, (bcast, 427)) + except socket.error as se: + pass def _grab_rsps(socks, rsps, interval, xidmap, deferrals): From 7b3129a1a2a736d8d3672f2f983eb2ccd0062400 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 09:08:28 -0500 Subject: [PATCH 133/319] Fix FFDC preflight checks The code was comparing two string constants, instead of a variable to a constant. Correct the problem to enable the preflight checks to work as intended. --- confluent_server/confluent/firmwaremanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/firmwaremanager.py b/confluent_server/confluent/firmwaremanager.py index a7713943..eb5d4c86 100644 --- a/confluent_server/confluent/firmwaremanager.py +++ b/confluent_server/confluent/firmwaremanager.py @@ -53,7 +53,7 @@ def execupdate(handler, filename, updateobj, type, owner, node, datfile): return if type == 'ffdc' and os.path.isdir(filename): filename += '/' + node - if 'type' == 'ffdc': + if type == 'ffdc': errstr = False if os.path.exists(filename): errstr = '{0} already exists on {1}, cannot overwrite'.format( From 17fff4997baa5b344013b9f4003bbe166bd94db3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jan 2024 10:52:49 -0500 Subject: [PATCH 134/319] Fix omission of info dir in plugins --- confluent_server/setup.py.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index e6bd08b2..871497e3 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -19,6 +19,7 @@ setup( 'confluent/plugins/hardwaremanagement/', 'confluent/plugins/deployment/', 'confluent/plugins/console/', + 'confluent/plugins/info/', 'confluent/plugins/shell/', 'confluent/collective/', 'confluent/plugins/configuration/'], From ddb8c4cce44ac8d0d30385cea7d466ea862d3c73 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 1 Dec 2023 15:55:17 -0500 Subject: [PATCH 135/319] Fix a few noderange abbreviations Also, add some test cases on abbreviation to help sanity check things in the future. --- confluent_server/confluent/noderange.py | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index df4552b8..cf99dd72 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -96,6 +96,7 @@ class Bracketer(object): txtnums = getnumbers_nodename(nodename) nums = [int(x) for x in txtnums] for n in range(self.count): + # First pass to see if we have exactly one different number padto = len(txtnums[n]) needpad = (padto != len('{}'.format(nums[n]))) if self.sequences[n] is None: @@ -105,7 +106,24 @@ class Bracketer(object): elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: continue # new nodename has no new number, keep going else: # if self.sequences[n][2] != nums[n] or : - if self.diffn is not None and (n != self.diffn or + if self.diffn is not None and (n != self.diffn or + (padto < self.numlens[n][1]) or + (needpad and padto != self.numlens[n][1])): + self.flush_current() + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [padto, padto] + self.diffn = n + for n in range(self.count): + padto = len(txtnums[n]) + needpad = (padto != len('{}'.format(nums[n]))) + if self.sequences[n] is None: + # We initialize to text pieces, 'currstart', and 'prev' number + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] + elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: + continue # new nodename has no new number, keep going + else: # if self.sequences[n][2] != nums[n] or : + if self.diffn is not None and (n != self.diffn or (padto < self.numlens[n][1]) or (needpad and padto != self.numlens[n][1])): self.flush_current() @@ -449,3 +467,29 @@ class NodeRange(object): if self.cfm is None: return set([element]) raise Exception(element + ' not a recognized node, group, or alias') + +if __name__ == '__main__': + cases = [ + (['r3u4', 'r5u6'], 'r3u4,r5u6'), # should not erroneously gather + (['r3u4s1', 'r5u6s3'], 'r3u4s1,r5u6s3'), # should not erroneously gather + (['r3u4s1', 'r3u4s2', 'r5u4s3'], 'r3u4s[1:2],r5u4s3'), # should not erroneously gather + (['r3u4', 'r3u5', 'r3u6', 'r3u9', 'r4u1'], 'r3u[4:6,9],r4u1'), + (['n01', 'n2', 'n03'], 'n01,n2,n03'), + (['n7', 'n8', 'n09', 'n10', 'n11', 'n12', 'n13', 'n14', 'n15', 'n16', + 'n17', 'n18', 'n19', 'n20'], 'n[7:8],n[09:20]') + ] + for case in cases: + gc = case[0] + bracketer = Bracketer(gc[0]) + for chnk in gc[1:]: + bracketer.extend(chnk) + br = bracketer.range + resnodes = NodeRange(br).nodes + if set(resnodes) != set(gc): + print('FAILED: ' + repr(sorted(gc))) + print('RESULT: ' + repr(sorted(resnodes))) + print('EXPECTED: ' + repr(case[1])) + print('ACTUAL: ' + br) + + + From 661b2ae81542deffa9c53b7df844389515470eef Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 27 Nov 2023 08:34:34 -0500 Subject: [PATCH 136/319] Filter out nvme 'c' devnames, that are used to refer to paths to nvme Some versions start manifesting nvme devnames with 'c', which are to be used to interact with multipath to have raw devices backing a traditional nvme device. --- .../el7-diskless/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk | 2 ++ .../el8-diskless/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk | 2 ++ .../el9-diskless/profiles/default/scripts/getinstalldisk | 2 ++ .../rhvh4/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk | 2 ++ .../suse15/profiles/server/scripts/getinstalldisk | 2 ++ .../profiles/default/scripts/getinstalldisk | 2 ++ .../ubuntu20.04/profiles/default/scripts/getinstalldisk | 2 ++ .../ubuntu22.04/profiles/default/scripts/getinstalldisk | 2 ++ 11 files changed, 22 insertions(+) diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk +++ b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None From 03bdbfc8ed43f64e25a9b103fa1874e774dbe362 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 13 Feb 2024 15:58:08 -0500 Subject: [PATCH 137/319] Provide more useful error messages on mistakes within [] --- confluent_server/confluent/noderange.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index cf99dd72..4a5cb808 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -402,12 +402,16 @@ class NodeRange(object): def _expandstring(self, element, filternodes=None): prefix = '' if element[0][0] in ('/', '~'): + if self.purenumeric: + raise Exception('Regular expression not supported within "[]"') element = ''.join(element) nameexpression = element[1:] if self.cfm is None: raise Exception('Verification configmanager required') return set(self.cfm.filter_nodenames(nameexpression, filternodes)) elif '=' in element[0] or '!~' in element[0]: + if self.purenumeric: + raise Exception('The "=" character is invalid within "[]"') element = ''.join(element) if self.cfm is None: raise Exception('Verification configmanager required') From 12bb5d583a7d58d2326521d455426ea887e1449e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 13 Feb 2024 16:00:50 -0500 Subject: [PATCH 138/319] Correct the equality message in better messagesw --- confluent_server/confluent/noderange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 4a5cb808..7657292c 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -411,7 +411,7 @@ class NodeRange(object): return set(self.cfm.filter_nodenames(nameexpression, filternodes)) elif '=' in element[0] or '!~' in element[0]: if self.purenumeric: - raise Exception('The "=" character is invalid within "[]"') + raise Exception('Equality/Inequality operators (=, !=, =~, !~) are invalid within "[]"') element = ''.join(element) if self.cfm is None: raise Exception('Verification configmanager required') From 34804b2d5f70e3f1835ab5368da589053f188ca7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 1 Apr 2024 12:13:21 -0400 Subject: [PATCH 139/319] Provide components for cert management with modern XCC Refresh getcsr and installcert to handle latest firmware. Also add ability to have pre-existing CSR, and trust the SAN on the way through. If this becomes more properly a feature, then would likely impose a SAN on certs, similar to the SSH principals, rather than deferring to the CSR to get it right. --- confluent_server/confluent/certutil.py | 76 +++++++++++++++++++------- misc/getcsr.py | 54 ++++++++++++++---- misc/installcert.py | 15 +++++ 3 files changed, 112 insertions(+), 33 deletions(-) diff --git a/confluent_server/confluent/certutil.py b/confluent_server/confluent/certutil.py index 2e788bad..9a478787 100644 --- a/confluent_server/confluent/certutil.py +++ b/confluent_server/confluent/certutil.py @@ -206,7 +206,7 @@ def create_simple_ca(keyout, certout): finally: os.remove(tmpconfig) -def create_certificate(keyout=None, certout=None): +def create_certificate(keyout=None, certout=None, csrout=None): if not keyout: keyout, certout = get_certificate_paths() if not keyout: @@ -214,9 +214,10 @@ def create_certificate(keyout=None, certout=None): assure_tls_ca() shortname = socket.gethostname().split('.')[0] longname = shortname # socket.getfqdn() - subprocess.check_call( - ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out', - keyout]) + if not csrout: + subprocess.check_call( + ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out', + keyout]) san = ['IP:{0}'.format(x) for x in get_ip_addresses()] # It is incorrect to put IP addresses as DNS type. However # there exists non-compliant clients that fail with them as IP @@ -229,21 +230,34 @@ def create_certificate(keyout=None, certout=None): os.close(tmphdl) tmphdl, extconfig = tempfile.mkstemp() os.close(tmphdl) - tmphdl, csrout = tempfile.mkstemp() - os.close(tmphdl) + needcsr = False + if csrout is None: + needcsr = True + tmphdl, csrout = tempfile.mkstemp() + os.close(tmphdl) shutil.copy2(sslcfg, tmpconfig) - serialnum = '0x' + ''.join(['{:02x}'.format(x) for x in bytearray(os.urandom(20))]) try: - with open(tmpconfig, 'a') as cfgfile: - cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san)) - with open(extconfig, 'a') as cfgfile: - cfgfile.write('\nbasicConstraints=CA:false\nsubjectAltName={0}'.format(san)) - subprocess.check_call([ - 'openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj', - '/CN={0}'.format(longname), - '-extensions', 'SAN', '-config', tmpconfig - ]) + if needcsr: + with open(tmpconfig, 'a') as cfgfile: + cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san)) + with open(extconfig, 'a') as cfgfile: + cfgfile.write('\nbasicConstraints=CA:false\nsubjectAltName={0}'.format(san)) + subprocess.check_call([ + 'openssl', 'req', '-new', '-key', keyout, '-out', csrout, '-subj', + '/CN={0}'.format(longname), + '-extensions', 'SAN', '-config', tmpconfig + ]) + else: + # when used manually, allow the csr SAN to stand + # may add explicit subj/SAN argument, in which case we would skip copy + with open(tmpconfig, 'a') as cfgfile: + cfgfile.write('\ncopy_extensions=copy\n') + with open(extconfig, 'a') as cfgfile: + cfgfile.write('\nbasicConstraints=CA:false\n') if os.path.exists('/etc/confluent/tls/cakey.pem'): + # simple style CA in effect, make a random serial number and + # hope for the best, and accept inability to backdate the cert + serialnum = '0x' + ''.join(['{:02x}'.format(x) for x in bytearray(os.urandom(20))]) subprocess.check_call([ 'openssl', 'x509', '-req', '-in', csrout, '-CA', '/etc/confluent/tls/cacert.pem', @@ -252,20 +266,40 @@ def create_certificate(keyout=None, certout=None): '-extfile', extconfig ]) else: + # we moved to a 'proper' CA, mainly for access to backdating + # start of certs for finicky system clocks + # this also provides a harder guarantee of serial uniqueness, but + # not of practical consequence (160 bit random value is as good as + # guaranteed unique) + # downside is certificate generation is serialized + cacfgfile = '/etc/confluent/tls/ca/openssl.cfg' + if needcsr: + tmphdl, tmpcafile = tempfile.mkstemp() + shutil.copy2(cacfgfile, tmpcafile) + os.close(tmphdl) + cacfgfile = tmpcafile + # with realcalock: # if we put it in server, we must lock it subprocess.check_call([ - 'openssl', 'ca', '-config', '/etc/confluent/tls/ca/openssl.cfg', + 'openssl', 'ca', '-config', cacfgfile, '-in', csrout, '-out', certout, '-batch', '-notext', '-startdate', '19700101010101Z', '-enddate', '21000101010101Z', '-extfile', extconfig ]) finally: os.remove(tmpconfig) - os.remove(csrout) - os.remove(extconfig) + if needcsr: + os.remove(csrout) + print(extconfig) # os.remove(extconfig) if __name__ == '__main__': + import sys outdir = os.getcwd() keyout = os.path.join(outdir, 'key.pem') - certout = os.path.join(outdir, 'cert.pem') - create_certificate(keyout, certout) + certout = os.path.join(outdir, sys.argv[2] + 'cert.pem') + csrout = None + try: + csrout = sys.argv[1] + except IndexError: + csrout = None + create_certificate(keyout, certout, csrout) diff --git a/misc/getcsr.py b/misc/getcsr.py index 253bfcd8..6f956b2d 100644 --- a/misc/getcsr.py +++ b/misc/getcsr.py @@ -12,11 +12,40 @@ ap.add_argument('--state', help='State or Province') ap.add_argument('--city', help='City or Locality') ap.add_argument('--org', help='Organization name') ap.add_argument('--name', help='Common/Host Name') +ap.add_argument('outcsr', help='CSR filename to save') args = ap.parse_args() c = cmd.Command(args.xcc, os.environ['XCCUSER'], os.environ['XCCPASS'], verifycallback=lambda x: True) -params = [ + +overview = c._do_web_request('/redfish/v1/') +cs = overview.get('CertificateService', {}).get('@odata.id', None) +if cs: + csinfo = c._do_web_request(cs) + gcsr = csinfo.get('Actions', {}).get('#CertificateService.GenerateCSR', {}).get('target', None) + if gcsr: + #https://n241-bmc/redfish/v1/Managers/1/NetworkProtocol HTTPS + #/redfish/v1/Managers/1/NetworkProtocol/HTTPS/Certificates + #/redfish/v1/CertificateService/CertificateLocations + csrargs = { + 'City': args.city, + 'State': args.state, + 'Organization': args.org, + 'Country': args.country, + 'CommonName': args.name, + 'KeyPairAlgorithm': 'TPM_ALG_ECDH', + 'KeyCurveId': 'TPM_ECC_NIST_P384', + 'CertificateCollection': { '@odata.id': '/redfish/v1/Managers/1/NetworkProtocol/HTTPS/Certificates'} + } + + csrinfo = c._do_web_request(gcsr, csrargs) + if 'CSRString' in csrinfo: + with open(args.outcsr, 'w') as csrout: + csrout.write(csrinfo['CSRString']) + sys.exit(0) + +else: + params = [ '0', # 'serviceType' args.country, args.state, @@ -32,15 +61,16 @@ params = [ '', '', '', -] -wc = c.oem.wc -rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_GenKeyAndCSR': ','.join(params)}) -rsp, status = wc.grab_json_response_with_status('/api/dataset', {'CSR_Format': '1'}) -rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_DownloadCSRANDCert': '0,4,0'}) -wc.request('GET', '/download/{0}'.format(rsp['FileName'])) -rsp = wc.getresponse() -csr = rsp.read() -if rsp.getheader('Content-Encoding', None) == 'gzip': - csr = gzip.GzipFile(fileobj=io.BytesIO(csr)).read() -print(csr) + ] + + wc = c.oem.wc + rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_GenKeyAndCSR': ','.join(params)}) + rsp, status = wc.grab_json_response_with_status('/api/dataset', {'CSR_Format': '1'}) + rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_DownloadCSRANDCert': '0,4,0'}) + wc.request('GET', '/download/{0}'.format(rsp['FileName'])) + rsp = wc.getresponse() + csr = rsp.read() + if rsp.getheader('Content-Encoding', None) == 'gzip': + csr = gzip.GzipFile(fileobj=io.BytesIO(csr)).read() + print(csr) diff --git a/misc/installcert.py b/misc/installcert.py index 9654bf54..2d53e800 100644 --- a/misc/installcert.py +++ b/misc/installcert.py @@ -8,8 +8,23 @@ ap.add_argument('xcc', help='XCC address') ap.add_argument('cert', help='Certificate in PEM format') args = ap.parse_args() +cert = open(args.cert, 'r').read() c = cmd.Command(args.xcc, os.environ['XCCUSER'], os.environ['XCCPASS'], verifycallback=lambda x: True) +overview = c._do_web_request('/redfish/v1/') +cs = overview.get('CertificateService', {}).get('@odata.id', None) +if cs: + csinfo = c._do_web_request(cs) + gcsr = csinfo.get('Actions', {}).get('#CertificateService.ReplaceCertificate', {}).get('target', None) + if gcsr: + repcertargs = { + 'CertificateUri': { '@odata.id': '/redfish/v1/Managers/1/NetworkProtocol/HTTPS/Certificates/1' }, + 'CertificateType': 'PEM', + 'CertificateString': cert } + print(repr(c._do_web_request(gcsr, repcertargs))) + sys.exit(0) + + #CertificateService.ReplaceCertificate wc = c.oem.wc cert = open(args.cert, 'rb').read() res = wc.grab_json_response_with_status('/api/function', {'Sec_ImportCert': '0,1,0,0,,{0}'.format(cert)}) From 894290f577cf450a9e702d8af8f6cffa96558ede Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 3 Apr 2024 18:46:37 -0400 Subject: [PATCH 140/319] nodebmcpassword --- confluent_client/bin/nodebmcpassword | 91 ++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100755 confluent_client/bin/nodebmcpassword diff --git a/confluent_client/bin/nodebmcpassword b/confluent_client/bin/nodebmcpassword new file mode 100755 index 00000000..ccaebac4 --- /dev/null +++ b/confluent_client/bin/nodebmcpassword @@ -0,0 +1,91 @@ +#!/usr/libexec/platform-python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2015-2017 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'tkucherera' + +import optparse +import os +import signal +import sys +try: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) +except AttributeError: + pass + +path = os.path.dirname(os.path.realpath(__file__)) +path = os.path.realpath(os.path.join(path, '..', 'lib', 'python')) +if path.startswith('/opt'): + sys.path.append(path) + +import confluent.client as client + +argparser = optparse.OptionParser(usage="Usage: %prog ") +argparser.add_option('-m', '--maxnodes', type='int', + help='Number of nodes to affect before prompting for confirmation') + + + +(options, args) = argparser.parse_args() +try: + noderange = args[0] + username = args[1] + new_password = args[2] +except IndexError: + argparser.print_help() + sys.exit(1) +client.check_globbing(noderange) +session = client.Command() +exitcode = 0 + + + +errorNodes = set([]) + +uid_dict = {} +session.stop_if_noderange_over(noderange, options.maxnodes) + +for rsp in session.read('/noderange/{0}/configuration/management_controller/users/all'.format(noderange)): + databynode = rsp["databynode"] + for node in databynode: + if 'error' in rsp['databynode'][node]: + print(node, ':', rsp['databynode'][node]['error']) + continue + for user in rsp['databynode'][node]['users']: + if user['username'] == username: + if not user['uid'] in uid_dict: + uid_dict[user['uid']] = node + continue + uid_dict[user['uid']] = uid_dict[user['uid']] + ',{}'.format(node) + break + +for uid in uid_dict: + success = session.simple_noderange_command(uid_dict[uid], 'configuration/management_controller/users/{0}'.format(uid), new_password, key='password', errnodes=errorNodes) # = 0 if successful + +allNodes = set([]) + +for node in session.read('/noderange/{0}/nodes/'.format(noderange)): + if 'error' in node and success != 0: + sys.exit(success) + allNodes.add(node['item']['href'].replace("/", "")) + +goodNodes = allNodes - errorNodes + +for node in goodNodes: + print(node + ": Password Change Successful") + + +sys.exit(success) \ No newline at end of file From 272c456435a4b3a3545298a508593cc38cdfb19a Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 3 Apr 2024 18:50:46 -0400 Subject: [PATCH 141/319] nodebmcpassword man page --- confluent_client/doc/man/nodebmcpassword.ronn | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 confluent_client/doc/man/nodebmcpassword.ronn diff --git a/confluent_client/doc/man/nodebmcpassword.ronn b/confluent_client/doc/man/nodebmcpassword.ronn new file mode 100644 index 00000000..5927f670 --- /dev/null +++ b/confluent_client/doc/man/nodebmcpassword.ronn @@ -0,0 +1,28 @@ +nodebmcpassword(8) -- Change management controller password for a specified user +========================================================= + +## SYNOPSIS + +`nodebmcpassword ` + +## DESCRIPTION + +`nodebmcpassword` allows you to change the management controller password for a user on a specified noderange + +## OPTIONS + +* `-m MAXNODES`, `--maxnodes=MAXNODES`: + Number of nodes to affect before prompting for + confirmation + +* `-h`, `--help`: + Show help message and exit + +## EXAMPLES: + +* Reset the management controller for nodes n1 through n4: + `# nodebmcreset n1-n4` + `n1: Password Change Successful` + `n2: Password Change Successful` + `n3: Password Change Successful` + `n4: Password Change Successful` \ No newline at end of file From a4e152c17d226cb613a7b684bb4ff215a7e2e131 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 10:31:46 -0400 Subject: [PATCH 142/319] Defer disarm until after successful client notification It is theoretically possible for a client to get disconnected right in the middle. In such a scenario, err on the side of letting the mechanism stay armed for the sake of a retry being possible. --- confluent_server/confluent/credserver.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/credserver.py b/confluent_server/confluent/credserver.py index c569bc4d..390179f8 100644 --- a/confluent_server/confluent/credserver.py +++ b/confluent_server/confluent/credserver.py @@ -127,14 +127,15 @@ class CredServer(object): if hmacval != hmac.new(hmackey, etok, hashlib.sha256).digest(): client.close() return - cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': echotoken}, 'deployment.sealedapikey': '', 'deployment.apiarmed': ''}} - if hmackey and apiarmed != 'continuous': - self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken']) - if apiarmed == 'continuous': - del cfgupdate[nodename]['deployment.apiarmed'] + cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': echotoken}}} self.cfm.set_node_attributes(cfgupdate) client.recv(2) # drain end of message client.send(b'\x05\x00') # report success + if hmackey and apiarmed != 'continuous': + self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken']) + if apiarmed != 'continuous': + tokclear = {nodename: {'deployment.sealedapikey': '', 'deployment.apiarmed': ''}} + self.cfm.set_node_attributes(tokclear) finally: client.close() From f68f9f46939ed115b501b125212359777cf045c6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 11:07:11 -0400 Subject: [PATCH 143/319] Make syncfile step robust or pause If syncfiles fails, keep it retrying. Also, slow down sync checking to avoid hammering the system. Further, randomized delay to spread highly synchronized requestors. Block attempts to do multiple concurrent syncfile runs. --- .../profiles/default/scripts/syncfileclient | 2 ++ .../profiles/default/scripts/syncfileclient | 2 ++ .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../suse15/profiles/hpc/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/server/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- confluent_server/confluent/syncfiles.py | 2 ++ 13 files changed, 176 insertions(+), 10 deletions(-) diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient index 8d37d43a..cca0f57d 100644 --- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,5 @@ #!/usr/bin/python +import time import importlib import tempfile import json @@ -223,6 +224,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: + time.sleep(2) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient index 8d37d43a..02dbcc4d 100644 --- a/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient @@ -5,6 +5,7 @@ import json import os import shutil import pwd +import time import grp try: from importlib.machinery import SourceFileLoader @@ -223,6 +224,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: + time.sleep(2) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 94b74eea..df5574e3 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -289,6 +289,8 @@ syncrunners = {} def start_syncfiles(nodename, cfg, suffixes, principals=[]): peerip = None + if nodename in syncrunners: + return '503 Synchronization already in progress ' if 'myips' in suffixes: targips = suffixes['myips'] del suffixes['myips'] From 33271451d711d3bc2a26a10cf22381d948e8aed2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 13:17:19 -0400 Subject: [PATCH 144/319] Support SHA384 if used as fingerprint --- confluent_server/confluent/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/util.py b/confluent_server/confluent/util.py index 96d2291b..462ec930 100644 --- a/confluent_server/confluent/util.py +++ b/confluent_server/confluent/util.py @@ -168,7 +168,7 @@ def cert_matches(fingerprint, certificate): return algo(certificate).digest() == fingerprint algo, _, fp = fingerprint.partition('$') newfp = None - if algo in ('sha512', 'sha256'): + if algo in ('sha512', 'sha256', 'sha384'): newfp = get_fingerprint(certificate, algo) return newfp and fingerprint == newfp From 02f301b5d08eaa20a9e02a4c1bd39be019f073a4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 13:41:27 -0400 Subject: [PATCH 145/319] Fix mistakes in syncfileclient change --- .../el8-diskless/profiles/default/scripts/syncfileclient | 3 ++- confluent_osdeploy/el8/profiles/default/scripts/syncfileclient | 3 ++- .../el9-diskless/profiles/default/scripts/syncfileclient | 3 ++- .../genesis/profiles/default/scripts/syncfileclient | 3 ++- .../suse15-diskless/profiles/default/scripts/syncfileclient | 3 ++- confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient | 3 ++- .../suse15/profiles/server/scripts/syncfileclient | 3 ++- .../profiles/default/scripts/syncfileclient | 3 ++- .../ubuntu20.04/profiles/default/scripts/syncfileclient | 3 ++- .../ubuntu22.04/profiles/default/scripts/syncfileclient | 3 ++- 10 files changed, 20 insertions(+), 10 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') From 8ca9a44476de8da95894d7a7ed3324232f12bc05 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 14:27:00 -0400 Subject: [PATCH 146/319] Provide more interesting response body to syncfileclient --- confluent_server/confluent/selfservice.py | 4 ++-- confluent_server/confluent/syncfiles.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index 3d7feebb..a166e0fb 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -517,8 +517,8 @@ def handle_request(env, start_response): pals = get_extra_names(nodename, cfg, myip) result = syncfiles.start_syncfiles( nodename, cfg, json.loads(reqbody), pals) - start_response(result, ()) - yield '' + start_response(result[0], ()) + yield result[1] return if 'GET' == operation: status, output = syncfiles.get_syncresult(nodename) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index df5574e3..ed99fedf 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -290,7 +290,7 @@ syncrunners = {} def start_syncfiles(nodename, cfg, suffixes, principals=[]): peerip = None if nodename in syncrunners: - return '503 Synchronization already in progress ' + return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) if 'myips' in suffixes: targips = suffixes['myips'] del suffixes['myips'] @@ -313,13 +313,13 @@ def start_syncfiles(nodename, cfg, suffixes, principals=[]): raise Exception('Cannot perform syncfiles without profile assigned') synclist = '/var/lib/confluent/public/os/{}/syncfiles'.format(profile) if not os.path.exists(synclist): - return '200 OK' # not running + return '200 OK', 'No synclist' # not running sl = SyncList(synclist, nodename, cfg) if not (sl.appendmap or sl.mergemap or sl.replacemap or sl.appendoncemap): - return '200 OK' # the synclist has no actual entries + return '200 OK', 'Empty synclist' # the synclist has no actual entries syncrunners[nodename] = eventlet.spawn( sync_list_to_node, sl, nodename, suffixes, peerip) - return '202 Queued' # backgrounded + return '202 Queued', 'Background synchronization initiated' # backgrounded def get_syncresult(nodename): if nodename not in syncrunners: From 67b3c48dc9b322817f2a556e7afdf463be4c1ee8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 14:58:38 -0400 Subject: [PATCH 147/319] Clean up error output on syncfileclient execution --- .../el8-diskless/profiles/default/scripts/syncfileclient | 2 ++ confluent_osdeploy/el8/profiles/default/scripts/syncfileclient | 2 ++ .../el9-diskless/profiles/default/scripts/syncfileclient | 2 ++ .../genesis/profiles/default/scripts/syncfileclient | 2 ++ .../suse15-diskless/profiles/default/scripts/syncfileclient | 2 ++ confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient | 2 ++ .../suse15/profiles/server/scripts/syncfileclient | 2 ++ .../profiles/default/scripts/syncfileclient | 2 ++ .../ubuntu20.04/profiles/default/scripts/syncfileclient | 2 ++ .../ubuntu22.04/profiles/default/scripts/syncfileclient | 2 ++ 10 files changed, 20 insertions(+) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) From 1da27083cc8a64510009331b8853fcbc8a804c0f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:08:56 -0400 Subject: [PATCH 148/319] Another cleanup of syncfileclient output --- .../el8-diskless/profiles/default/scripts/syncfileclient | 4 +++- .../el8/profiles/default/scripts/syncfileclient | 4 +++- .../el9-diskless/profiles/default/scripts/syncfileclient | 4 +++- .../genesis/profiles/default/scripts/syncfileclient | 4 +++- .../suse15-diskless/profiles/default/scripts/syncfileclient | 4 +++- confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient | 4 +++- .../suse15/profiles/server/scripts/syncfileclient | 4 +++- .../profiles/default/scripts/syncfileclient | 4 +++- .../ubuntu20.04/profiles/default/scripts/syncfileclient | 4 +++- .../ubuntu22.04/profiles/default/scripts/syncfileclient | 4 +++- 10 files changed, 30 insertions(+), 10 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' From eaffb342b2af2878c1a8aaad00c79b7873d23f74 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:19:38 -0400 Subject: [PATCH 149/319] Reap stale sync runners after a minute dead If the client never claims the result, delete the sync task. --- confluent_server/confluent/syncfiles.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index ed99fedf..1f739ba1 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -285,9 +285,10 @@ def mkpathorlink(source, destination, appendexist=False): syncrunners = {} - +cleaner = None def start_syncfiles(nodename, cfg, suffixes, principals=[]): + global cleaner peerip = None if nodename in syncrunners: return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) @@ -319,7 +320,26 @@ def start_syncfiles(nodename, cfg, suffixes, principals=[]): return '200 OK', 'Empty synclist' # the synclist has no actual entries syncrunners[nodename] = eventlet.spawn( sync_list_to_node, sl, nodename, suffixes, peerip) - return '202 Queued', 'Background synchronization initiated' # backgrounded + if not cleaner: + cleaner = eventlet.spawn(cleanit) + return '202 Queued', 'Background synchronization initiated' # backgrounded + + +def cleanit(): + toreap = {} + while True: + for nn in list(syncrunners): + if syncrunners[nn].dead: + if nn in toreap: + syncrunners[nn].wait() + del syncrunners[nn] + del toreap[nn] + else: + toreap[nn] = 1 + elif nn is in toreap: + del toreap[nn] + eventlet.sleep(30) + def get_syncresult(nodename): if nodename not in syncrunners: From 8fb889ba736609b6ee70be835836d8691e91f4ff Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:27:20 -0400 Subject: [PATCH 150/319] Correct syntax error --- confluent_server/confluent/syncfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 1f739ba1..068d1ae4 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -336,7 +336,7 @@ def cleanit(): del toreap[nn] else: toreap[nn] = 1 - elif nn is in toreap: + elif nn in toreap: del toreap[nn] eventlet.sleep(30) From 01722c18c4e1e99d56e4bd465eb8005a69a1ae58 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:40:40 -0400 Subject: [PATCH 151/319] Fix location of idle sleep in syncfiles cleaner --- confluent_server/confluent/syncfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 068d1ae4..9c96e533 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -338,7 +338,7 @@ def cleanit(): toreap[nn] = 1 elif nn in toreap: del toreap[nn] - eventlet.sleep(30) + eventlet.sleep(30) def get_syncresult(nodename): From ceaf641c1a31cd03da1ecab66557bfb6d244d73f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 16:18:24 -0400 Subject: [PATCH 152/319] Keep reap loop going on error --- confluent_server/confluent/syncfiles.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 9c96e533..f1e638f8 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -331,7 +331,11 @@ def cleanit(): for nn in list(syncrunners): if syncrunners[nn].dead: if nn in toreap: - syncrunners[nn].wait() + try: + syncrunners[nn].wait() + except Exception as e: + print(repr(e)) + pass del syncrunners[nn] del toreap[nn] else: From 8e5ee6c9d8490d0275d9e84a118048d1e783744b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 10 Apr 2024 13:54:06 -0400 Subject: [PATCH 153/319] Make orphaned sync runner retire on new sync request --- confluent_server/confluent/syncfiles.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index f1e638f8..16cf4c49 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -290,8 +290,6 @@ cleaner = None def start_syncfiles(nodename, cfg, suffixes, principals=[]): global cleaner peerip = None - if nodename in syncrunners: - return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) if 'myips' in suffixes: targips = suffixes['myips'] del suffixes['myips'] @@ -318,6 +316,11 @@ def start_syncfiles(nodename, cfg, suffixes, principals=[]): sl = SyncList(synclist, nodename, cfg) if not (sl.appendmap or sl.mergemap or sl.replacemap or sl.appendoncemap): return '200 OK', 'Empty synclist' # the synclist has no actual entries + if nodename in syncrunners: + if syncrunners[nodename].dead: + syncrunners[nodename].wait() + else: + return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) syncrunners[nodename] = eventlet.spawn( sync_list_to_node, sl, nodename, suffixes, peerip) if not cleaner: From a6a2f2f2de333b863c3d3ed926f2942d7b92e7b3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Apr 2024 11:46:11 -0400 Subject: [PATCH 154/319] Fixes for attribute clear warning behavior Correct collective behavior for failing to clear on followers. Also, connect the warnings from the leader to the member issuing the RPC. --- .../confluent/config/configmanager.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 9e7818b5..6c7ebd71 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -252,10 +252,12 @@ def _rpc_master_rename_nodegroups(tenant, renamemap): def _rpc_master_clear_node_attributes(tenant, nodes, attributes): - ConfigManager(tenant).clear_node_attributes(nodes, attributes) + warnings = [] + ConfigManager(tenant).clear_node_attributes(nodes, attributes, warnings) + return warnings -def _rpc_clear_node_attributes(tenant, nodes, attributes): +def _rpc_clear_node_attributes(tenant, nodes, attributes): # master has to do the warnings ConfigManager(tenant)._true_clear_node_attributes(nodes, attributes) @@ -348,9 +350,9 @@ def exec_on_leader(function, *args): rpclen = len(rpcpayload) cfgleader.sendall(struct.pack('!Q', rpclen)) cfgleader.sendall(rpcpayload) - _pendingchangesets[xid].wait() + retv = _pendingchangesets[xid].wait() del _pendingchangesets[xid] - return + return retv def exec_on_followers(fnname, *args): @@ -714,8 +716,9 @@ def relay_slaved_requests(name, listener): exc = None if not (rpc['function'].startswith('_rpc_') or rpc['function'].endswith('_collective_member')): raise Exception('Unsupported function {0} called'.format(rpc['function'])) + retv = None try: - globals()[rpc['function']](*rpc['args']) + retv = globals()[rpc['function']](*rpc['args']) except ValueError as ve: exc = ['ValueError', str(ve)] except Exception as e: @@ -723,7 +726,7 @@ def relay_slaved_requests(name, listener): exc = ['Exception', str(e)] if 'xid' in rpc: res = _push_rpc(listener, msgpack.packb({'xid': rpc['xid'], - 'exc': exc}, use_bin_type=False)) + 'exc': exc, 'ret': retv}, use_bin_type=False)) if not res: break try: @@ -929,7 +932,7 @@ def follow_channel(channel): exc = Exception(excstr) _pendingchangesets[rpc['xid']].send_exception(exc) else: - _pendingchangesets[rpc['xid']].send() + _pendingchangesets[rpc['xid']].send(rpc.get('ret', None)) if 'quorum' in rpc: _hasquorum = rpc['quorum'] res = _push_rpc(channel, b'') # use null as ACK @@ -2204,14 +2207,17 @@ class ConfigManager(object): def clear_node_attributes(self, nodes, attributes, warnings=None): if cfgleader: - return exec_on_leader('_rpc_master_clear_node_attributes', + mywarnings = exec_on_leader('_rpc_master_clear_node_attributes', self.tenant, nodes, attributes) + if warnings is not None: + warnings.extend(mywarnings) + return if cfgstreams: exec_on_followers('_rpc_clear_node_attributes', self.tenant, nodes, attributes) self._true_clear_node_attributes(nodes, attributes, warnings) - def _true_clear_node_attributes(self, nodes, attributes, warnings): + def _true_clear_node_attributes(self, nodes, attributes, warnings=None): # accumulate all changes into a changeset and push in one go changeset = {} realattributes = [] @@ -2234,16 +2240,16 @@ class ConfigManager(object): # delete it and check for inheritence to backfil data del nodek[attrib] self._do_inheritance(nodek, attrib, node, changeset) - if not warnings is None: + if warnings is not None: if attrib in nodek: warnings.append('The attribute "{}" was defined specifically for the node and clearing now has a value inherited from the group "{}"'.format(attrib, nodek[attrib]['inheritedfrom'])) _addchange(changeset, node, attrib) _mark_dirtykey('nodes', node, self.tenant) elif attrib in nodek: - if not warnings is None: + if warnings is not None: warnings.append('The attribute "{0}" is inherited from group "{1}", leaving the inherited value alone (use "{0}=" with no value to explicitly blank the value if desired)'.format(attrib, nodek[attrib]['inheritedfrom'])) else: - if not warnings is None: + if warnings is not None: warnings.append('Attribute "{}" is either already cleared, or does not match a defined attribute (if referencing an attribute group, try a wildcard)'.format(attrib)) if ('_expressionkeys' in nodek and attrib in nodek['_expressionkeys']): From 3ba3394a542422433cbb0b726103f4742e4ba363 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Apr 2024 17:32:13 -0400 Subject: [PATCH 155/319] Fix None return by exec_on_leader with warnings --- confluent_server/confluent/config/configmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 6c7ebd71..528924e8 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -2209,7 +2209,7 @@ class ConfigManager(object): if cfgleader: mywarnings = exec_on_leader('_rpc_master_clear_node_attributes', self.tenant, nodes, attributes) - if warnings is not None: + if mywarnings and warnings is not None: warnings.extend(mywarnings) return if cfgstreams: From 3b55f500cea756f37837dbdc14a399af9624778d Mon Sep 17 00:00:00 2001 From: tkucherera Date: Tue, 16 Apr 2024 03:16:15 -0400 Subject: [PATCH 156/319] sample post scripts directory --- .../profile/scripts/sample/consoleredirect | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 confluent_osdeploy/common/profile/scripts/sample/consoleredirect diff --git a/confluent_osdeploy/common/profile/scripts/sample/consoleredirect b/confluent_osdeploy/common/profile/scripts/sample/consoleredirect new file mode 100644 index 00000000..4ebc3a8f --- /dev/null +++ b/confluent_osdeploy/common/profile/scripts/sample/consoleredirect @@ -0,0 +1,49 @@ +is_suse=false +is_rhel=false + +if test -f /boot/efi/EFI/redhat/grub.cfg; then + grubcfg="/boot/efi/EFI/redhat/grub.cfg" + grub2-mkconfig -o $grubcfg + is_rhel=true +elif test -f /boot/efi/EFI/sle_hpc/grub.cfg; then + grubcfg="/boot/efi/EFI/sle_hpc/grub.cfg" + grub2-mkconfig -o $grubcfg + is_suse=true +else + echo "Expected File missing: Check if os sle_hpc or redhat" + exit +fi + +# working on SUSE +if $is_suse; then + start=false + num_line=0 + lines_to_edit=() + while read line; do + ((num_line++)) + if [[ $line == *"grub_platform"* ]]; then + start=true + fi + if $start; then + if [[ $line != "#"* ]];then + lines_to_edit+=($num_line) + fi + fi + if [[ ${#line} -eq 2 && $line == *"fi" ]]; then + if $start; then + start=false + fi + fi + done < grub_cnf.cfg + + for line_num in "${lines_to_edit[@]}"; do + line_num+="s" + sed -i "${line_num},^,#," $grubcfg + done + sed -i 's,^terminal,#terminal,' $grubcfg +fi + +# Working on Redhat +if $is_rhel; then + sed -i 's,^serial,#serial, ; s,^terminal,#terminal,' $grubcfg +fi \ No newline at end of file From 10f0fabb8cfe81b71fb4933c078f58cea729e096 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 17 Apr 2024 15:18:45 -0400 Subject: [PATCH 157/319] Fix nodegroup retrieval nodegroup information was broken by clear warning support. --- confluent_server/confluent/plugins/configuration/attributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/configuration/attributes.py b/confluent_server/confluent/plugins/configuration/attributes.py index a56a1aee..2c9a6ac9 100644 --- a/confluent_server/confluent/plugins/configuration/attributes.py +++ b/confluent_server/confluent/plugins/configuration/attributes.py @@ -30,7 +30,7 @@ def retrieve(nodes, element, configmanager, inputdata, clearwarnbynode=None): element[1], element[3], configmanager, inputdata, clearwarnbynode) -def retrieve_nodegroup(nodegroup, element, configmanager, inputdata, clearwarnbynode): +def retrieve_nodegroup(nodegroup, element, configmanager, inputdata, clearwarnbynode=None): try: grpcfg = configmanager.get_nodegroup_attributes(nodegroup) except KeyError: From b6068823271c96b4ab344dde03813aae8b1a72c3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 19 Apr 2024 08:22:32 -0400 Subject: [PATCH 158/319] Have collate preserve relative whitespace The change to tolerate either a space or no space ended up greedily consuming whitespace. Do best possible in two cases: For log, use the first line as a clue, and consistently pad or not pad according to first line. It won't catch different pad strategies, or handle first line being indented but other lines not being indented. For the textgroup variant, allow subsequent lines to revise the pad downward, and accept any whitespace, not just space. --- confluent_client/bin/collate | 11 +++++++++-- confluent_client/confluent/textgroup.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/confluent_client/bin/collate b/confluent_client/bin/collate index 07095901..2a086303 100755 --- a/confluent_client/bin/collate +++ b/confluent_client/bin/collate @@ -21,6 +21,7 @@ import optparse import os +import re import select import sys @@ -84,6 +85,7 @@ fullline = sys.stdin.readline() printpending = True clearpending = False holdoff = 0 +padded = None while fullline: for line in fullline.split('\n'): if not line: @@ -92,13 +94,18 @@ while fullline: line = 'UNKNOWN: ' + line if options.log: node, output = line.split(':', 1) - output = output.lstrip() + if padded is None: + if output.startswith(' '): + padded = True + else: + padded = False + if padded: + output = re.sub(r'^ ', '', output) currlog = options.log.format(node=node, nodename=node) with open(currlog, mode='a') as log: log.write(output + '\n') continue node, output = line.split(':', 1) - output = output.lstrip() grouped.add_line(node, output) if options.watch: if not holdoff: diff --git a/confluent_client/confluent/textgroup.py b/confluent_client/confluent/textgroup.py index cd35b6fa..e2f0dc7f 100644 --- a/confluent_client/confluent/textgroup.py +++ b/confluent_client/confluent/textgroup.py @@ -98,17 +98,24 @@ class GroupedData(object): self.byoutput = {} self.header = {} self.client = confluentconnection + self.detectedpad = None def generate_byoutput(self): self.byoutput = {} + thepad = self.detectedpad if self.detectedpad else '' for n in self.bynode: - output = '\n'.join(self.bynode[n]) + output = '' + for ln in self.bynode[n]: + output += ln.replace(thepad, '', 1) + '\n' if output not in self.byoutput: self.byoutput[output] = set([n]) else: self.byoutput[output].add(n) def add_line(self, node, line): + wspc = re.search(r'^\s*', line).group() + if self.detectedpad is None or len(wspc) < len(self.detectedpad): + self.detectedpad = wspc if node not in self.bynode: self.bynode[node] = [line] else: @@ -219,4 +226,4 @@ if __name__ == '__main__': if not line: continue groupoutput.add_line(*line.split(': ', 1)) - groupoutput.print_deviants() \ No newline at end of file + groupoutput.print_deviants() From 6f2be355efa738c08e617d49fceb5aeafc240931 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Apr 2024 11:32:27 -0400 Subject: [PATCH 159/319] Source from "local" media if present Some environments may want to load the bulk of the media via USB rather than over the network. This prefers that source if that scheme is detected. --- .../usr/lib/dracut/hooks/pre-trigger/01-confluent.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index 6db95276..355a5ad7 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -227,7 +227,13 @@ if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; the fi fi -echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf +. /etc/os-release +ISOSRC=$(blkid -t TYPE=iso9660|grep -Ei ' LABEL="'$ID-$VERSION_ID|sed -e s/:.*//) +if [ -z "$ISOSRC" ]; then + echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf +else + echo inst.repo=cdrom:$ISOSRC >> /etc/cmdline.d/01-confluent.conf +fi echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution From 86e612b4bf0887a748c9a2c67a2f0e095507862c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Apr 2024 12:47:52 -0400 Subject: [PATCH 160/319] Move anaconda netroot to not be specified in media sourced When sourcing from media, do not trigger anaconda netroot behavior. --- .../usr/lib/dracut/hooks/pre-trigger/01-confluent.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index 355a5ad7..a1778e08 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -231,14 +231,14 @@ fi ISOSRC=$(blkid -t TYPE=iso9660|grep -Ei ' LABEL="'$ID-$VERSION_ID|sed -e s/:.*//) if [ -z "$ISOSRC" ]; then echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf + root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution + export root else echo inst.repo=cdrom:$ISOSRC >> /etc/cmdline.d/01-confluent.conf fi echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart -root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution export kickstart -export root autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg) autoconfigmethod=${autoconfigmethod#ipv4_method: } if [ "$autoconfigmethod" = "dhcp" ]; then From 9954b227a95b33c53c2869e29c091ee68f81cfc9 Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Tue, 23 Apr 2024 15:06:28 +0200 Subject: [PATCH 161/319] Revert "nodeconfig ntp servers for smms" This reverts commit 44405baaf39af36d1b695d1f8be6836319257816. --- confluent_client/bin/nodeconfig | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/confluent_client/bin/nodeconfig b/confluent_client/bin/nodeconfig index 0ba8d351..06d512c7 100755 --- a/confluent_client/bin/nodeconfig +++ b/confluent_client/bin/nodeconfig @@ -1,4 +1,4 @@ -#!/usr/libexec/platform-python +#!/usr/bin/python2 # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2017 Lenovo @@ -98,17 +98,8 @@ cfgpaths = { 'static_v6_gateway'), 'bmc.hostname': ( 'configuration/management_controller/hostname', 'hostname'), - 'bmc.ntp': ( - 'configuration/management_controller/ntp/enabled', 'state'), - 'bmc.ntpServer1': ( - 'configuration/management_controller/ntp/servers/1', 'server'), - 'bmc.ntpServer2': ( - 'configuration/management_controller/ntp/servers/2', 'server'), - 'bmc.ntpServer3': ( - 'configuration/management_controller/ntp/servers/3', 'server') } - autodeps = { 'bmc.ipv4_address': (('bmc.ipv4_method', 'static'),) } @@ -122,7 +113,6 @@ client.check_globbing(noderange) setmode = None assignment = {} queryparms = {} - printsys = [] printbmc = [] printextbmc = [] @@ -141,6 +131,7 @@ if len(args) == 1 or options.exclude: queryparms[path] = {} queryparms[path][attrib] = candidate + def _assign_value(): if key not in cfgpaths: setsys[key] = value @@ -242,7 +233,6 @@ else: parse_config_line(args[1:]) session = client.Command() rcode = 0 - if options.restoredefault: session.stop_if_noderange_over(noderange, options.maxnodes) if options.restoredefault.lower() in ( @@ -305,10 +295,8 @@ else: for path in queryparms: if options.comparedefault: continue - - rcode |= client.print_attrib_path(path, session, list(queryparms[path]),NullOpt(), queryparms[path]) - - + rcode |= client.print_attrib_path(path, session, list(queryparms[path]), + NullOpt(), queryparms[path]) if printsys == 'all' or printextbmc or printbmc or printallbmc: if printbmc or not printextbmc: rcode |= client.print_attrib_path( From 3b69c542034a7311942537c5398c2ac334e88d4e Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Tue, 23 Apr 2024 15:08:36 +0200 Subject: [PATCH 162/319] Revert "new eaton pdu power readings" This reverts commit ac68f1f22c90a7852459996bd87e07804900d90e. --- .../plugins/hardwaremanagement/eatonpdu.py | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py index 9b5ade5b..da60b182 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py @@ -156,7 +156,6 @@ def get_sensor_data(element, node, configmanager): gc.logout() _sensors_by_node[node] = [sdata, time.time() + 1] sn = _sensors_by_node.get(node, None) -# print(sn) for outlet in sn[0]: for sensename in sn[0][outlet]: myname = '{0} {1}'.format(outlet, sensename) @@ -278,35 +277,22 @@ class PDUClient(object): if outdata[0] == outlet: return 'on' if outdata[3] else 'off' return - def get_outlet_sensors(self): - rsp = self.do_request('cgi_pdu_outlets') - data = sanitize_json(rsp[0]) - data = json.loads(data) - data = data['data'][0] - - - return data def get_sensor_data(self): rsp = self.do_request1('cgi_overview') - + data = sanitize_json(rsp[0]) data = json.loads(data) - data1 = data['data'][4][0][8] - - data = self.get_outlet_sensors() + data = data['data'][0] sdata = {} - for outdata in data: - + outsense = {} - outletname = outdata[0][1] - - + outletname = outdata[3] outsense['Power'] = { - 'value': outdata[4], - 'units': 'W', + 'value': outdata[5], + 'units': 'kW', 'type': 'Power', } sdata[outletname] = outsense From 21b8534fefaaee5ebee23247da6dbacf94d3f506 Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Tue, 23 Apr 2024 15:29:35 +0200 Subject: [PATCH 163/319] Revert "my changes to Eaton PDU sensors" This reverts commit 6d87d11f5e6decd854995d4b5bd7188c049c418a. --- .../plugins/hardwaremanagement/eatonpdu.py | 49 +++++-------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py index da60b182..16be5b38 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py @@ -158,7 +158,7 @@ def get_sensor_data(element, node, configmanager): sn = _sensors_by_node.get(node, None) for outlet in sn[0]: for sensename in sn[0][outlet]: - myname = '{0} {1}'.format(outlet, sensename) + myname = 'Outlet {0} {1}'.format(outlet, sensename) measurement = sn[0][outlet][sensename] if name == 'all' or simplify_name(myname) == name: readings.append({ @@ -259,11 +259,6 @@ class PDUClient(object): url = '/config/gateway?page={}&sessionId={}&_dc={}'.format(suburl, self.sessid, int(time.time())) return wc.grab_response(url) - def do_request1(self, suburl): - wc = self.wc - url = '/config/gateway?page={}&sessionId={}'.format(suburl, self.sessid) - return wc.grab_response(url) - def logout(self): self.do_request('cgi_logout') @@ -279,51 +274,29 @@ class PDUClient(object): return def get_sensor_data(self): - rsp = self.do_request1('cgi_overview') - + rsp = self.do_request('cgi_pdu_outlets') data = sanitize_json(rsp[0]) data = json.loads(data) - data1 = data['data'][4][0][8] data = data['data'][0] sdata = {} for outdata in data: - outsense = {} - outletname = outdata[3] + outletname = outdata[0][0] + outsense['Energy'] = { + 'value': float(outdata[11] / 1000), + 'units': 'kwh', + 'type': 'Energy' + } outsense['Power'] = { - 'value': outdata[5], - 'units': 'kW', + 'value': float(outdata[4]), + 'units': 'w', 'type': 'Power', } sdata[outletname] = outsense - for outdata in data1: - - outsense = {} - outletname = outdata[0] - if type(outdata[1]) == str : - splitter = outdata[1].split(" ") - - if len(splitter) == 1: - splitter.append('w') - outsense['Power'] = { - 'value': splitter[0], - 'units': splitter[1], - 'type': 'Power', - } - elif type(outdata[1]) == list: - if type(outdata[1][1]) == float: - outletname=outletname.strip('
Since') - - outsense['Energy'] = { - 'value': outdata[1][0] / 1000, - 'units': 'kWh', - 'type': 'Energy', - } - sdata[outletname] = outsense return sdata def set_outlet(self, outlet, state): - rsp = self.do_request('cgi_overview') + rsp = self.do_request('cgi_pdu_outlets') data = sanitize_json(rsp[0]) data = json.loads(data) data = data['data'][0] From bf004fb7b97e426217337848b21f39a62bd86aad Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Tue, 23 Apr 2024 15:53:35 +0200 Subject: [PATCH 164/319] cleaned geist.py --- .../plugins/hardwaremanagement/geist.py | 277 ++++++++++-------- 1 file changed, 156 insertions(+), 121 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/geist.py b/confluent_server/confluent/plugins/hardwaremanagement/geist.py index 06af8675..d3fce9e1 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/geist.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/geist.py @@ -19,25 +19,29 @@ import confluent.exceptions as exc import eventlet.green.time as time import eventlet import eventlet.greenpool as greenpool -import json + + def simplify_name(name): - return name.lower().replace(' ', '_').replace('/', '-').replace( - '_-_', '-') + return name.lower().replace(" ", "_").replace("/", "-").replace("_-_", "-") + pdupool = greenpool.GreenPool(128) + def data_by_type(indata): databytype = {} for keyname in indata: obj = indata[keyname] - objtype = obj.get('type', None) + objtype = obj.get("type", None) if not objtype: continue if objtype in databytype: - raise Exception("Multiple instances of type {} not yet supported".format(objtype)) + raise Exception( + "Multiple instances of type {} not yet supported".format(objtype) + ) databytype[objtype] = obj - obj['keyname'] = keyname + obj["keyname"] = keyname return databytype @@ -59,198 +63,229 @@ class GeistClient(object): def wc(self): if self._wc: return self._wc - targcfg = self.configmanager.get_node_attributes(self.node, - ['hardwaremanagement.manager'], - decrypt=True) + targcfg = self.configmanager.get_node_attributes( + self.node, ["hardwaremanagement.manager"], decrypt=True + ) targcfg = targcfg.get(self.node, {}) - target = targcfg.get( - 'hardwaremanagement.manager', {}).get('value', None) + target = targcfg.get("hardwaremanagement.manager", {}).get("value", None) if not target: target = self.node - target = target.split('/', 1)[0] + target = target.split("/", 1)[0] cv = util.TLSCertVerifier( - self.configmanager, self.node, - 'pubkeys.tls_hardwaremanager').verify_cert + self.configmanager, self.node, "pubkeys.tls_hardwaremanager" + ).verify_cert self._wc = wc.SecureHTTPConnection(target, port=443, verifycallback=cv) return self._wc def login(self, configmanager): - credcfg = configmanager.get_node_attributes(self.node, - ['secret.hardwaremanagementuser', - 'secret.hardwaremanagementpassword'], - decrypt=True) + credcfg = configmanager.get_node_attributes( + self.node, + ["secret.hardwaremanagementuser", "secret.hardwaremanagementpassword"], + decrypt=True, + ) credcfg = credcfg.get(self.node, {}) - username = credcfg.get( - 'secret.hardwaremanagementuser', {}).get('value', None) - passwd = credcfg.get( - 'secret.hardwaremanagementpassword', {}).get('value', None) + username = credcfg.get("secret.hardwaremanagementuser", {}).get("value", None) + passwd = credcfg.get("secret.hardwaremanagementpassword", {}).get("value", None) if not isinstance(username, str): - username = username.decode('utf8') + username = username.decode("utf8") if not isinstance(passwd, str): - passwd = passwd.decode('utf8') + passwd = passwd.decode("utf8") if not username or not passwd: - raise Exception('Missing username or password') + raise Exception("Missing username or password") self.username = username rsp = self.wc.grab_json_response( - '/api/auth/{0}'.format(username), - {'cmd': 'login', 'data': {'password': passwd}}) + "/api/auth/{0}".format(username), + {"cmd": "login", "data": {"password": passwd}}, + ) - token = rsp['data']['token'] + token = rsp["data"]["token"] return token def logout(self): if self._token: - self.wc.grab_json_response('/api/auth/{0}'.format(self.username), - {'cmd': 'logout', 'token': self.token}) + self.wc.grab_json_response( + "/api/auth/{0}".format(self.username), + {"cmd": "logout", "token": self.token}, + ) self._token = None def get_outlet(self, outlet): - rsp = self.wc.grab_json_response('/api/dev') - rsp = rsp['data'] + rsp = self.wc.grab_json_response("/api/dev") + rsp = rsp["data"] dbt = data_by_type(rsp) - if 't3hd' in dbt: - del dbt['t3hd'] + if "t3hd" in dbt: + del dbt["t3hd"] if len(dbt) != 1: - raise Exception('Multiple PDUs not supported per pdu') + raise Exception("Multiple PDUs not supported per pdu") pdutype = list(dbt)[0] - outlet = dbt[pdutype]['outlet'][ str(int(outlet)- 1) ] -# print(outlet) - state = outlet['state'].split('2')[-1] + outlet = dbt[pdutype]["outlet"][str(int(outlet) - 1)] + + state = outlet["state"].split("2")[-1] return state def set_outlet(self, outlet, state): - rsp = self.wc.grab_json_response('/api/dev') - dbt = data_by_type(rsp['data']) - if 't3hd' in dbt: - del dbt['t3hd'] + rsp = self.wc.grab_json_response("/api/dev") + dbt = data_by_type(rsp["data"]) + if "t3hd" in dbt: + del dbt["t3hd"] if len(dbt) != 1: self.logout() - raise Exception('Multiple PDUs per endpoint not supported') - pdu = dbt[list(dbt)[0]]['keyname'] - outlet = int(outlet) - 1 + raise Exception("Multiple PDUs per endpoint not supported") + pdu = dbt[list(dbt)[0]]["keyname"] + outlet = int(outlet) - 1 rsp = self.wc.grab_json_response( - '/api/dev/{0}/outlet/{1}'.format(pdu, outlet), - {'cmd': 'control', 'token': self.token, - 'data': {'action': state, 'delay': False}}) + "/api/dev/{0}/outlet/{1}".format(pdu, outlet), + { + "cmd": "control", + "token": self.token, + "data": {"action": state, "delay": False}, + }, + ) -def process_measurement(keyname, name, enttype, entname, measurement, readings, category): - if measurement['type'] == 'realPower': - if category not in ('all', 'power'): + +def process_measurement( + keyname, name, enttype, entname, measurement, readings, category +): + if measurement["type"] == "realPower": + if category not in ("all", "power"): return - readtype = 'Real Power' - elif measurement['type'] == 'apparentPower': - if category not in ('all', 'power'): + readtype = "Real Power" + elif measurement["type"] == "apparentPower": + if category not in ("all", "power"): return - readtype = 'Apparent Power' - elif measurement['type'] == 'energy': - if category not in ('all', 'energy'): + readtype = "Apparent Power" + elif measurement["type"] == "energy": + if category not in ("all", "energy"): return - readtype = 'Energy' - elif measurement['type'] == 'voltage': - if category not in ('all',): + readtype = "Energy" + elif measurement["type"] == "voltage": + if category not in ("all",): return - readtype = 'Voltage' - elif measurement['type'] == 'current': - if category not in ('all',): + readtype = "Voltage" + elif measurement["type"] == "current": + if category not in ("all",): return - readtype = 'Current' - elif measurement['type'] == 'temperature': - readtype = 'Temperature' - elif measurement['type'] == 'dewpoint': - readtype = 'Dewpoint' - elif measurement['type'] == 'humidity': - readtype = 'Humidity' + readtype = "Current" + elif measurement["type"] == "temperature": + readtype = "Temperature" + elif measurement["type"] == "dewpoint": + readtype = "Dewpoint" + elif measurement["type"] == "humidity": + readtype = "Humidity" else: return - myname = entname + ' ' + readtype - if name != 'all' and simplify_name(myname) != name: + myname = entname + " " + readtype + if name != "all" and simplify_name(myname) != name: return - readings.append({ - 'name': myname, - 'value': float(measurement['value']), - 'units': measurement['units'], - 'type': readtype.split()[-1] - }) - + readings.append( + { + "name": myname, + "value": float(measurement["value"]), + "units": measurement["units"], + "type": readtype.split()[-1], + } + ) + def process_measurements(name, category, measurements, enttype, readings): for measure in util.natural_sort(list(measurements)): - measurement = measurements[measure]['measurement'] - entname = measurements[measure]['name'] + measurement = measurements[measure]["measurement"] + entname = measurements[measure]["name"] for measureid in measurement: - process_measurement(measure, name, enttype, entname, measurement[measureid], readings, category) - + process_measurement( + measure, + name, + enttype, + entname, + measurement[measureid], + readings, + category, + ) + _sensors_by_node = {} + + def read_sensors(element, node, configmanager): category, name = element[-2:] justnames = False if len(element) == 3: # just get names category = name - name = 'all' + name = "all" justnames = True - if category in ('leds, fans', 'temperature'): + if category in ("leds, fans", "temperature"): return sn = _sensors_by_node.get(node, None) if not sn or sn[1] < time.time(): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response('/api/dev') + adev = gc.wc.grab_json_response("/api/dev") _sensors_by_node[node] = (adev, time.time() + 1) sn = _sensors_by_node.get(node, None) - dbt = data_by_type(sn[0]['data']) + dbt = data_by_type(sn[0]["data"]) readings = [] - for datatype in dbt: + for datatype in dbt: datum = dbt[datatype] - process_measurements(name, category, datum['entity'], 'entity', readings) + process_measurements(name, category, datum["entity"], "entity", readings) - if 'outlet' in datum: - process_measurements(name, category, datum['outlet'], 'outlet', readings) + if "outlet" in datum: + process_measurements(name, category, datum["outlet"], "outlet", readings) if justnames: for reading in readings: - yield msg.ChildCollection(simplify_name(reading['name'])) + yield msg.ChildCollection(simplify_name(reading["name"])) else: yield msg.SensorReadings(readings, name=node) + def get_outlet(element, node, configmanager): gc = GeistClient(node, configmanager) state = gc.get_outlet(element[-1]) return msg.PowerState(node=node, state=state) + def read_firmware(node, configmanager): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response('/api/sys') - myversion = adev['data']['version'] - yield msg.Firmware([{'PDU Firmware': {'version': myversion}}], node) + adev = gc.wc.grab_json_response("/api/sys") + myversion = adev["data"]["version"] + yield msg.Firmware([{"PDU Firmware": {"version": myversion}}], node) def read_inventory(element, node, configmanager): _inventory = {} inventory = {} gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response('/api/sys') - basedata = adev['data'] - inventory['present'] = True - inventory['name'] = 'PDU' + adev = gc.wc.grab_json_response("/api/sys") + basedata = adev["data"] + inventory["present"] = True + inventory["name"] = "PDU" for elem in basedata.items(): - if elem[0] !='component' and elem[0] !='locale' and elem[0] !='state' and elem[0] !='contact' and elem[0] !='appVersion' and elem[0] !='build' and elem[0] !='version' and elem[0] !='apiVersion': + if ( + elem[0] != "component" + and elem[0] != "locale" + and elem[0] != "state" + and elem[0] != "contact" + and elem[0] != "appVersion" + and elem[0] != "build" + and elem[0] != "version" + and elem[0] != "apiVersion" + ): temp = elem[0] if elem[0] == "serialNumber": temp = "Serial" - elif elem[0] == "partNumber": + elif elem[0] == "partNumber": temp = "P/N" - elif elem[0] == "modelNumber": - temp= "Lenovo P/N and Serial" + elif elem[0] == "modelNumber": + temp = "Lenovo P/N and Serial" _inventory[temp] = elem[1] - elif elem[0] =='component': - tempname = '' - for component in basedata['component'].items(): + elif elem[0] == "component": + tempname = "" + for component in basedata["component"].items(): for item in component: if type(item) == str: @@ -258,28 +293,27 @@ def read_inventory(element, node, configmanager): else: for entry in item.items(): temp = entry[0] - if temp == 'sn': + if temp == "sn": temp = "Serial" - _inventory[tempname + ' ' + temp] = entry[1] - + _inventory[tempname + " " + temp] = entry[1] - inventory['information']= _inventory + inventory["information"] = _inventory + + yield msg.KeyValueData({"inventory": [inventory]}, node) - yield msg.KeyValueData({'inventory': [inventory]}, node) - def retrieve(nodes, element, configmanager, inputdata): - if 'outlets' in element: + if "outlets" in element: gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(get_outlet, element, node, configmanager) for res in gp: yield res - + return - elif element[0] == 'sensors': + elif element[0] == "sensors": gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_sensors, element, node, configmanager) @@ -287,7 +321,7 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum return - elif '/'.join(element).startswith('inventory/firmware/all'): + elif "/".join(element).startswith("inventory/firmware/all"): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_firmware, node, configmanager) @@ -295,7 +329,7 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum - elif '/'.join(element).startswith('inventory/hardware/all'): + elif "/".join(element).startswith("inventory/hardware/all"): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_inventory, element, node, configmanager) @@ -304,12 +338,13 @@ def retrieve(nodes, element, configmanager, inputdata): yield datum else: for node in nodes: - yield msg.ConfluentResourceUnavailable(node, 'Not implemented') + yield msg.ConfluentResourceUnavailable(node, "Not implemented") return - + + def update(nodes, element, configmanager, inputdata): - if 'outlets' not in element: - yield msg.ConfluentResourceUnavailable(node, 'Not implemented') + if "outlets" not in element: + yield msg.ConfluentResourceUnavailable(node, "Not implemented") return for node in nodes: gc = GeistClient(node, configmanager) From d779f015d6c35d1f459886f9ab9f9b63b808b374 Mon Sep 17 00:00:00 2001 From: Wera Grzeda Date: Tue, 23 Apr 2024 18:14:48 +0200 Subject: [PATCH 165/319] sed to use single quotes --- .../plugins/hardwaremanagement/geist.py | 222 +++++++++--------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/geist.py b/confluent_server/confluent/plugins/hardwaremanagement/geist.py index d3fce9e1..3f086115 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/geist.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/geist.py @@ -1,13 +1,13 @@ # Copyright 2022 Lenovo # -# Licensed under the Apache License, Version 2.0 (the "License"); +# Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, +# distributed under the License is distributed on an 'AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @@ -23,7 +23,7 @@ import eventlet.greenpool as greenpool def simplify_name(name): - return name.lower().replace(" ", "_").replace("/", "-").replace("_-_", "-") + return name.lower().replace(' ', '_').replace('/', '-').replace('_-_', '-') pdupool = greenpool.GreenPool(128) @@ -33,15 +33,15 @@ def data_by_type(indata): databytype = {} for keyname in indata: obj = indata[keyname] - objtype = obj.get("type", None) + objtype = obj.get('type', None) if not objtype: continue if objtype in databytype: raise Exception( - "Multiple instances of type {} not yet supported".format(objtype) + 'Multiple instances of type {} not yet supported'.format(objtype) ) databytype[objtype] = obj - obj["keyname"] = keyname + obj['keyname'] = keyname return databytype @@ -64,15 +64,15 @@ class GeistClient(object): if self._wc: return self._wc targcfg = self.configmanager.get_node_attributes( - self.node, ["hardwaremanagement.manager"], decrypt=True + self.node, ['hardwaremanagement.manager'], decrypt=True ) targcfg = targcfg.get(self.node, {}) - target = targcfg.get("hardwaremanagement.manager", {}).get("value", None) + target = targcfg.get('hardwaremanagement.manager', {}).get('value', None) if not target: target = self.node - target = target.split("/", 1)[0] + target = target.split('/', 1)[0] cv = util.TLSCertVerifier( - self.configmanager, self.node, "pubkeys.tls_hardwaremanager" + self.configmanager, self.node, 'pubkeys.tls_hardwaremanager' ).verify_cert self._wc = wc.SecureHTTPConnection(target, port=443, verifycallback=cv) return self._wc @@ -80,68 +80,68 @@ class GeistClient(object): def login(self, configmanager): credcfg = configmanager.get_node_attributes( self.node, - ["secret.hardwaremanagementuser", "secret.hardwaremanagementpassword"], + ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True, ) credcfg = credcfg.get(self.node, {}) - username = credcfg.get("secret.hardwaremanagementuser", {}).get("value", None) - passwd = credcfg.get("secret.hardwaremanagementpassword", {}).get("value", None) + username = credcfg.get('secret.hardwaremanagementuser', {}).get('value', None) + passwd = credcfg.get('secret.hardwaremanagementpassword', {}).get('value', None) if not isinstance(username, str): - username = username.decode("utf8") + username = username.decode('utf8') if not isinstance(passwd, str): - passwd = passwd.decode("utf8") + passwd = passwd.decode('utf8') if not username or not passwd: - raise Exception("Missing username or password") + raise Exception('Missing username or password') self.username = username rsp = self.wc.grab_json_response( - "/api/auth/{0}".format(username), - {"cmd": "login", "data": {"password": passwd}}, + '/api/auth/{0}'.format(username), + {'cmd': 'login', 'data': {'password': passwd}}, ) - token = rsp["data"]["token"] + token = rsp['data']['token'] return token def logout(self): if self._token: self.wc.grab_json_response( - "/api/auth/{0}".format(self.username), - {"cmd": "logout", "token": self.token}, + '/api/auth/{0}'.format(self.username), + {'cmd': 'logout', 'token': self.token}, ) self._token = None def get_outlet(self, outlet): - rsp = self.wc.grab_json_response("/api/dev") - rsp = rsp["data"] + rsp = self.wc.grab_json_response('/api/dev') + rsp = rsp['data'] dbt = data_by_type(rsp) - if "t3hd" in dbt: - del dbt["t3hd"] + if 't3hd' in dbt: + del dbt['t3hd'] if len(dbt) != 1: - raise Exception("Multiple PDUs not supported per pdu") + raise Exception('Multiple PDUs not supported per pdu') pdutype = list(dbt)[0] - outlet = dbt[pdutype]["outlet"][str(int(outlet) - 1)] + outlet = dbt[pdutype]['outlet'][str(int(outlet) - 1)] - state = outlet["state"].split("2")[-1] + state = outlet['state'].split('2')[-1] return state def set_outlet(self, outlet, state): - rsp = self.wc.grab_json_response("/api/dev") - dbt = data_by_type(rsp["data"]) - if "t3hd" in dbt: - del dbt["t3hd"] + rsp = self.wc.grab_json_response('/api/dev') + dbt = data_by_type(rsp['data']) + if 't3hd' in dbt: + del dbt['t3hd'] if len(dbt) != 1: self.logout() - raise Exception("Multiple PDUs per endpoint not supported") - pdu = dbt[list(dbt)[0]]["keyname"] + raise Exception('Multiple PDUs per endpoint not supported') + pdu = dbt[list(dbt)[0]]['keyname'] outlet = int(outlet) - 1 rsp = self.wc.grab_json_response( - "/api/dev/{0}/outlet/{1}".format(pdu, outlet), + '/api/dev/{0}/outlet/{1}'.format(pdu, outlet), { - "cmd": "control", - "token": self.token, - "data": {"action": state, "delay": False}, + 'cmd': 'control', + 'token': self.token, + 'data': {'action': state, 'delay': False}, }, ) @@ -149,51 +149,51 @@ class GeistClient(object): def process_measurement( keyname, name, enttype, entname, measurement, readings, category ): - if measurement["type"] == "realPower": - if category not in ("all", "power"): + if measurement['type'] == 'realPower': + if category not in ('all', 'power'): return - readtype = "Real Power" - elif measurement["type"] == "apparentPower": - if category not in ("all", "power"): + readtype = 'Real Power' + elif measurement['type'] == 'apparentPower': + if category not in ('all', 'power'): return - readtype = "Apparent Power" - elif measurement["type"] == "energy": - if category not in ("all", "energy"): + readtype = 'Apparent Power' + elif measurement['type'] == 'energy': + if category not in ('all', 'energy'): return - readtype = "Energy" - elif measurement["type"] == "voltage": - if category not in ("all",): + readtype = 'Energy' + elif measurement['type'] == 'voltage': + if category not in ('all',): return - readtype = "Voltage" - elif measurement["type"] == "current": - if category not in ("all",): + readtype = 'Voltage' + elif measurement['type'] == 'current': + if category not in ('all',): return - readtype = "Current" - elif measurement["type"] == "temperature": - readtype = "Temperature" - elif measurement["type"] == "dewpoint": - readtype = "Dewpoint" - elif measurement["type"] == "humidity": - readtype = "Humidity" + readtype = 'Current' + elif measurement['type'] == 'temperature': + readtype = 'Temperature' + elif measurement['type'] == 'dewpoint': + readtype = 'Dewpoint' + elif measurement['type'] == 'humidity': + readtype = 'Humidity' else: return - myname = entname + " " + readtype - if name != "all" and simplify_name(myname) != name: + myname = entname + ' ' + readtype + if name != 'all' and simplify_name(myname) != name: return readings.append( { - "name": myname, - "value": float(measurement["value"]), - "units": measurement["units"], - "type": readtype.split()[-1], + 'name': myname, + 'value': float(measurement['value']), + 'units': measurement['units'], + 'type': readtype.split()[-1], } ) def process_measurements(name, category, measurements, enttype, readings): for measure in util.natural_sort(list(measurements)): - measurement = measurements[measure]["measurement"] - entname = measurements[measure]["name"] + measurement = measurements[measure]['measurement'] + entname = measurements[measure]['name'] for measureid in measurement: process_measurement( measure, @@ -215,28 +215,28 @@ def read_sensors(element, node, configmanager): if len(element) == 3: # just get names category = name - name = "all" + name = 'all' justnames = True - if category in ("leds, fans", "temperature"): + if category in ('leds, fans', 'temperature'): return sn = _sensors_by_node.get(node, None) if not sn or sn[1] < time.time(): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response("/api/dev") + adev = gc.wc.grab_json_response('/api/dev') _sensors_by_node[node] = (adev, time.time() + 1) sn = _sensors_by_node.get(node, None) - dbt = data_by_type(sn[0]["data"]) + dbt = data_by_type(sn[0]['data']) readings = [] for datatype in dbt: datum = dbt[datatype] - process_measurements(name, category, datum["entity"], "entity", readings) + process_measurements(name, category, datum['entity'], 'entity', readings) - if "outlet" in datum: - process_measurements(name, category, datum["outlet"], "outlet", readings) + if 'outlet' in datum: + process_measurements(name, category, datum['outlet'], 'outlet', readings) if justnames: for reading in readings: - yield msg.ChildCollection(simplify_name(reading["name"])) + yield msg.ChildCollection(simplify_name(reading['name'])) else: yield msg.SensorReadings(readings, name=node) @@ -250,42 +250,42 @@ def get_outlet(element, node, configmanager): def read_firmware(node, configmanager): gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response("/api/sys") - myversion = adev["data"]["version"] - yield msg.Firmware([{"PDU Firmware": {"version": myversion}}], node) + adev = gc.wc.grab_json_response('/api/sys') + myversion = adev['data']['version'] + yield msg.Firmware([{'PDU Firmware': {'version': myversion}}], node) def read_inventory(element, node, configmanager): _inventory = {} inventory = {} gc = GeistClient(node, configmanager) - adev = gc.wc.grab_json_response("/api/sys") - basedata = adev["data"] - inventory["present"] = True - inventory["name"] = "PDU" + adev = gc.wc.grab_json_response('/api/sys') + basedata = adev['data'] + inventory['present'] = True + inventory['name'] = 'PDU' for elem in basedata.items(): if ( - elem[0] != "component" - and elem[0] != "locale" - and elem[0] != "state" - and elem[0] != "contact" - and elem[0] != "appVersion" - and elem[0] != "build" - and elem[0] != "version" - and elem[0] != "apiVersion" + elem[0] != 'component' + and elem[0] != 'locale' + and elem[0] != 'state' + and elem[0] != 'contact' + and elem[0] != 'appVersion' + and elem[0] != 'build' + and elem[0] != 'version' + and elem[0] != 'apiVersion' ): temp = elem[0] - if elem[0] == "serialNumber": - temp = "Serial" - elif elem[0] == "partNumber": - temp = "P/N" - elif elem[0] == "modelNumber": - temp = "Lenovo P/N and Serial" + if elem[0] == 'serialNumber': + temp = 'Serial' + elif elem[0] == 'partNumber': + temp = 'P/N' + elif elem[0] == 'modelNumber': + temp = 'Lenovo P/N and Serial' _inventory[temp] = elem[1] - elif elem[0] == "component": - tempname = "" - for component in basedata["component"].items(): + elif elem[0] == 'component': + tempname = '' + for component in basedata['component'].items(): for item in component: if type(item) == str: @@ -293,18 +293,18 @@ def read_inventory(element, node, configmanager): else: for entry in item.items(): temp = entry[0] - if temp == "sn": - temp = "Serial" - _inventory[tempname + " " + temp] = entry[1] + if temp == 'sn': + temp = 'Serial' + _inventory[tempname + ' ' + temp] = entry[1] - inventory["information"] = _inventory + inventory['information'] = _inventory - yield msg.KeyValueData({"inventory": [inventory]}, node) + yield msg.KeyValueData({'inventory': [inventory]}, node) def retrieve(nodes, element, configmanager, inputdata): - if "outlets" in element: + if 'outlets' in element: gp = greenpool.GreenPile(pdupool) for node in nodes: @@ -313,7 +313,7 @@ def retrieve(nodes, element, configmanager, inputdata): yield res return - elif element[0] == "sensors": + elif element[0] == 'sensors': gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_sensors, element, node, configmanager) @@ -321,7 +321,7 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum return - elif "/".join(element).startswith("inventory/firmware/all"): + elif '/'.join(element).startswith('inventory/firmware/all'): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_firmware, node, configmanager) @@ -329,7 +329,7 @@ def retrieve(nodes, element, configmanager, inputdata): for datum in rsp: yield datum - elif "/".join(element).startswith("inventory/hardware/all"): + elif '/'.join(element).startswith('inventory/hardware/all'): gp = greenpool.GreenPile(pdupool) for node in nodes: gp.spawn(read_inventory, element, node, configmanager) @@ -338,13 +338,13 @@ def retrieve(nodes, element, configmanager, inputdata): yield datum else: for node in nodes: - yield msg.ConfluentResourceUnavailable(node, "Not implemented") + yield msg.ConfluentResourceUnavailable(node, 'Not implemented') return def update(nodes, element, configmanager, inputdata): - if "outlets" not in element: - yield msg.ConfluentResourceUnavailable(node, "Not implemented") + if 'outlets' not in element: + yield msg.ConfluentResourceUnavailable(node, 'Not implemented') return for node in nodes: gc = GeistClient(node, configmanager) From 8f01f22bb5cae398cffd6d631232485c6993c409 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 24 Apr 2024 10:00:49 -0400 Subject: [PATCH 166/319] add password prompting and env var --- confluent_client/bin/nodebmcpassword | 31 +++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/confluent_client/bin/nodebmcpassword b/confluent_client/bin/nodebmcpassword index ccaebac4..f76b076c 100755 --- a/confluent_client/bin/nodebmcpassword +++ b/confluent_client/bin/nodebmcpassword @@ -17,6 +17,7 @@ __author__ = 'tkucherera' +from getpass import getpass import optparse import os import signal @@ -36,25 +37,49 @@ import confluent.client as client argparser = optparse.OptionParser(usage="Usage: %prog ") argparser.add_option('-m', '--maxnodes', type='int', help='Number of nodes to affect before prompting for confirmation') +argparser.add_option('-p', '--prompt', action='store_true', + help='Prompt for password values interactively') +argparser.add_option('-e', '--environment', action='store_true', + help='Set passwod, but from environment variable of ' + 'same name') (options, args) = argparser.parse_args() + + try: noderange = args[0] username = args[1] - new_password = args[2] except IndexError: argparser.print_help() sys.exit(1) + client.check_globbing(noderange) session = client.Command() exitcode = 0 +if options.prompt: + oneval = 1 + twoval = 2 + while oneval != twoval: + oneval = getpass('Enter pass for {0}: '.format(username)) + twoval = getpass('Confirm pass for {0}: '.format(username)) + if oneval != twoval: + print('Values did not match.') + new_password = twoval - +elif len(args) == 3: + if options.environment: + key = args[2] + new_password = os.environ.get(key, os.environ[key.upper()]) + else: + new_password = args[2] +else: + argparser.print_help() + sys.exit(1) + errorNodes = set([]) - uid_dict = {} session.stop_if_noderange_over(noderange, options.maxnodes) From 39fb229ef1d5ba05f9ced53f04c69d6ec66b9ff6 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Thu, 2 May 2024 10:29:20 -0400 Subject: [PATCH 167/319] check update args before setting anything --- confluent_client/confluent/client.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py index a7c13cd3..f9887d4e 100644 --- a/confluent_client/confluent/client.py +++ b/confluent_client/confluent/client.py @@ -705,6 +705,14 @@ def updateattrib(session, updateargs, nodetype, noderange, options, dictassign=N noderange, 'attributes/all', dictassign[key], key) else: if "=" in updateargs[1]: + update_ready = True + for arg in updateargs[1:]: + if not '=' in arg: + update_ready = False + sys.stderr.write('Error: {0} not a valid expression\n'.format(str(arg))) + exitcode = 1 + if not update_ready: + sys.exit(exitcode) try: for val in updateargs[1:]: val = val.split('=', 1) From 930ff3e20d97776223f7b9585373ef9514bcf172 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Thu, 2 May 2024 10:41:30 -0400 Subject: [PATCH 168/319] fix error message --- confluent_client/confluent/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py index f9887d4e..a9957b96 100644 --- a/confluent_client/confluent/client.py +++ b/confluent_client/confluent/client.py @@ -595,7 +595,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref else: printmissing.add(attr) for missing in printmissing: - sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) + sys.stderr.write('Error: {0} not a valid attribute\n'.format(missing)) return exitcode @@ -709,9 +709,9 @@ def updateattrib(session, updateargs, nodetype, noderange, options, dictassign=N for arg in updateargs[1:]: if not '=' in arg: update_ready = False - sys.stderr.write('Error: {0} not a valid expression\n'.format(str(arg))) exitcode = 1 if not update_ready: + sys.stderr.write('Error: {0} Can not set and read at the same time!\n'.format(str(updateargs[1:]))) sys.exit(exitcode) try: for val in updateargs[1:]: From b7a5101a34cc03f70abc725a3ecfc259ff98e457 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 3 May 2024 10:27:01 -0400 Subject: [PATCH 169/319] Provide extra warning about redoing SSH materials --- confluent_server/bin/osdeploy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_server/bin/osdeploy b/confluent_server/bin/osdeploy index 47ebc4a8..0283553c 100644 --- a/confluent_server/bin/osdeploy +++ b/confluent_server/bin/osdeploy @@ -325,14 +325,14 @@ def initialize(cmdset): try: sshutil.initialize_ca() except sshutil.AlreadyExists: - emprint('Skipping generation of SSH CA, already present and would likely be more problematic to regenerate than to reuse (if absolutely sure you want to discard old CA, then delete /etc/confluent/ssh/ca*)') + emprint('Skipping generation of SSH CA, already present and would likely be more problematic to regenerate than to reuse (if absolutely sure you want to discard old CA, then delete /etc/confluent/ssh/ca* and restart confluent)') if cmdset.a: didsomething = True init_confluent_myname() try: sshutil.initialize_root_key(True, True) except sshutil.AlreadyExists: - emprint('Skipping generation of new automation key, already present and regeneration usually causes more problems. (If absolutely certain, delete /etc/confluent/ssh/automation*)') + emprint('Skipping generation of new automation key, already present and regeneration usually causes more problems. (If absolutely certain, delete /etc/confluent/ssh/automation* and restart confluent)') if cmdset.p: install_tftp_content() if cmdset.l: From d0e73c887b788936063b5be6439bf670d5836110 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 7 May 2024 10:05:50 -0400 Subject: [PATCH 170/319] Load firewall before esxi installation begins Parts of esxi install depend on firewall running. When we are done with 'odd' networking, restore firewall to meet that expectation. --- confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel b/confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel index 74d47104..f88c730d 100644 --- a/confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel +++ b/confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel @@ -116,4 +116,5 @@ profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^pro mv /etc/confluent/confluent.deploycfg.new /etc/confluent/confluent.deploycfg export node mgr profile . /tmp/modinstall +localcli network firewall load exec /bin/install From 62be16442ca4957374028894d5384f1f23037875 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 8 May 2024 11:51:00 -0400 Subject: [PATCH 171/319] Fix passive detection of SLP devices (e.g. SMM) A mistake in the python3 port caused passive SLP detection to break. Remedy that mistake. --- confluent_server/confluent/discovery/protocols/slp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/slp.py b/confluent_server/confluent/discovery/protocols/slp.py index ac332def..30acb475 100644 --- a/confluent_server/confluent/discovery/protocols/slp.py +++ b/confluent_server/confluent/discovery/protocols/slp.py @@ -411,7 +411,7 @@ def query_srvtypes(target): parsed = _parse_slp_header(rs) if parsed: payload = parsed['payload'] - if payload[:2] != '\x00\x00': + if payload[:2] != b'\x00\x00': return stypelen = struct.unpack('!H', bytes(payload[2:4]))[0] stypes = payload[4:4+stypelen].decode('utf-8') From 172c57c6f19f8a5bc593fb13ac76df9afee87bf4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 15 May 2024 12:28:41 -0400 Subject: [PATCH 172/319] Fix media location search for EL8 EL8 distributions marked the 'OS' as dracut, workaround by trying to use PRETTY_NAME --- .../usr/lib/dracut/hooks/pre-trigger/01-confluent.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index a1778e08..caa60997 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -228,6 +228,15 @@ if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; the fi . /etc/os-release +if [ "$ID" = "dracut" ]; then + ID=$(echo $PRETTY_NAME|awk '{print $1}') + VERSION_ID=$(echo $VERSION|awk '{print $1}') + if [ "$ID" = "Oracle" ]; then + ID=OL + elif [ "$ID" = "Red" ]; then + ID=RHEL + fi +fi ISOSRC=$(blkid -t TYPE=iso9660|grep -Ei ' LABEL="'$ID-$VERSION_ID|sed -e s/:.*//) if [ -z "$ISOSRC" ]; then echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf From c0bcc3791d0071c565d8b480d10b3e51b0261c5a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 15 May 2024 12:30:13 -0400 Subject: [PATCH 173/319] Fix handling some eatonpdu return values --- .../confluent/plugins/hardwaremanagement/eatonpdu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py index 16be5b38..4c3d4654 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py @@ -33,7 +33,8 @@ def simplify_name(name): def sanitize_json(data): if not isinstance(data, str): data = data.decode('utf8') - return re.sub(r'([^ {:,]*):', r'"\1":', data).replace("'", '"') + return re.sub(r'([^ {:,]*):', r'"\1":', data).replace("'", '"').replace(',,', ',null,') + def answer_challenge(username, password, data): realm = data[0] From 010c8a0a231f54df6ad44edac70e9a9e10bffa60 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 22 May 2024 15:44:05 -0400 Subject: [PATCH 174/319] Amend EL network bringup One issue is that there are multiple networkmanager connections, clean this up, though this seems not to be a functional issue. However, sometimes the lldpad usage screws up network configuration, disable the facility by forcibly disabling fcoe sincec that is what triggers lldpad. wq --- .../lib/dracut/hooks/cmdline/01-confluent.sh | 5 ++-- .../dracut/hooks/pre-trigger/01-confluent.sh | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh index f441504e..bc327610 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh @@ -7,8 +7,8 @@ if ! grep console= /proc/cmdline >& /dev/null; then if [ -n "$autocons" ]; then echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf autocons=${autocons%,*} - echo $autocons > /tmp/01-autocons.devnode - echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons + echo $autocons > /tmp/01-autocons.devnode + echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons echo "Initializing auto detected console when installer starts" > $autocons fi fi @@ -16,4 +16,5 @@ if grep console=ttyS /proc/cmdline >& /dev/null; then echo "Serial console has been requested in the kernel arguments, the local video may not show progress" > /dev/tty1 fi . /lib/anaconda-lib.sh +echo rd.fcoe=0 > /etc/cmdline.d/nofcoe.conf wait_for_kickstart diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index caa60997..8701cc7e 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -99,6 +99,10 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then fi fi done + for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do + ip addr flush dev $NICGUESS + ip link set $NICGUESS down + done NetworkManager --configure-and-quit=initrd --no-daemon hmackeyfile=/tmp/cnflnthmackeytmp echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile @@ -175,7 +179,7 @@ if [ ! -z "$autocons" ]; then errout="-e $autocons" fi while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg; do - sleep 10 + sleep 10 done ifidx=$(cat /tmp/confluent.ifidx 2> /dev/null) if [ -z "$ifname" ]; then @@ -216,15 +220,15 @@ proto=${proto#protocol: } textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg) textconsole=${textconsole#textconsole: } if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then - autocons=$(cat /tmp/01-autocons.devnode) - if [ ! -z "$autocons" ]; then - echo Auto-configuring installed system to use text console - echo Auto-configuring installed system to use text console > $autocons + autocons=$(cat /tmp/01-autocons.devnode) + if [ ! -z "$autocons" ]; then + echo Auto-configuring installed system to use text console + echo Auto-configuring installed system to use text console > $autocons /opt/confluent/bin/autocons -c > /dev/null - cp /tmp/01-autocons.conf /etc/cmdline.d/ - else - echo "Unable to automatically detect requested text console" - fi + cp /tmp/01-autocons.conf /etc/cmdline.d/ + else + echo "Unable to automatically detect requested text console" + fi fi . /etc/os-release @@ -327,4 +331,8 @@ if [ -e /lib/nm-lib.sh ]; then fi fi fi +for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do + ip addr flush dev $NICGUESS + ip link set $NICGUESS down +done From bab0a77cb6c3a9fec77dfc13e09d82620a6f251d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 23 May 2024 15:15:37 -0400 Subject: [PATCH 175/319] Refresh to CentOS stream 9 --- genesis/97genesis/install-base | 6 +- genesis/97genesis/install-python | 376 +++++++++++++++---------------- genesis/confluent-genesis.spec | 2 +- genesis/extracttmuxlicenses.py | 30 ++- genesis/getlicenses.py | 40 ++-- 5 files changed, 241 insertions(+), 213 deletions(-) diff --git a/genesis/97genesis/install-base b/genesis/97genesis/install-base index 5d43f9b8..7fe1a976 100644 --- a/genesis/97genesis/install-base +++ b/genesis/97genesis/install-base @@ -5,16 +5,16 @@ dracut_install tpm2_create tpm2_pcrread tpm2_createpolicy tpm2_createprimary dracut_install tpm2_load tpm2_unseal tpm2_getcap tpm2_evictcontrol dracut_install tpm2_pcrextend tpm2_policypcr tpm2_flushcontext tpm2_startauthsession dracut_install openssl tar ipmitool cpio xz gzip lsmod ethtool -dracut_install modprobe touch echo cut wc bash netstat uniq grep ip hostname +dracut_install modprobe touch echo cut wc bash uniq grep ip hostname dracut_install awk egrep dirname bc expr sort dracut_install ssh sshd vi reboot lspci parted tmux mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap dracut_install efibootmgr dracut_install du df ssh-keygen scp clear dhclient lldpd lldpcli tee -dracut_install /lib64/libnss_dns-2.28.so /lib64/libnss_dns.so.2 /lib64/libnss_myhostname.so.2 +dracut_install /lib64/libnss_dns.so.2 /lib64/libnss_dns.so.2 /lib64/libnss_myhostname.so.2 dracut_install ldd uptime /usr/lib64/libnl-3.so.200 dracut_install poweroff date /etc/nsswitch.conf /etc/services /etc/protocols dracut_install /usr/share/terminfo/x/xterm /usr/share/terminfo/l/linux /usr/share/terminfo/v/vt100 /usr/share/terminfo/x/xterm-color /usr/share/terminfo/s/screen /usr/share/terminfo/x/xterm-256color /usr/share/terminfo/p/putty-256color /usr/share/terminfo/p/putty /usr/share/terminfo/d/dumb -dracut_install chmod ifconfig whoami route head tail basename /etc/redhat-release ping tr lsusb /usr/share/hwdata/usb.ids +dracut_install chmod whoami head tail basename /etc/redhat-release ping tr /usr/share/hwdata/usb.ids dracut_install dmidecode /usr/lib64/libstdc++.so.6 dracut_install ps free find inst /bin/bash /bin/sh diff --git a/genesis/97genesis/install-python b/genesis/97genesis/install-python index afc16a44..c5dd6db7 100644 --- a/genesis/97genesis/install-python +++ b/genesis/97genesis/install-python @@ -2,36 +2,35 @@ #strace /usr/libexec/platform-python -c 'import hashlib; import socket; import argparse; import socket; import os; import http.client; import http.cookies; import subprocess; import base64; import ctypes; import struct; import urllib.parse; import shlex; import configparser' dracut_install /usr/libexec/platform-python dracut_install /etc/localtime -dracut_install /lib64/libffi.so.6 -dracut_install /lib64/libssl.so.1.1 -dracut_install /usr/lib64/python3.6/os.py +dracut_install /lib64/libffi.so.8 +dracut_install /lib64/libssl.so.3 +dracut_install /usr/lib64/python3.9/os.py dracut_install /usr/lib64/gconv/gconv-modules.cache -dracut_install /usr/lib64/python3.6 -dracut_install /usr/lib64/python3.6/collections -dracut_install /usr/lib64/python3.6/ctypes -dracut_install /usr/lib64/python3.6/email -dracut_install /usr/lib64/python3.6/encodings -dracut_install /usr/lib64/python3.6/http -dracut_install /usr/lib64/python3.6/lib-dynload -dracut_install /usr/lib64/python3.6/lib-dynload/fcntl.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/binascii.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_bisect.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_blake2.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_datetime.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_hashlib.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_heapq.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/math.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_posixsubprocess.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_random.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/select.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_sha3.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_socket.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_ssl.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/_struct.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/unicodedata.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/site-packages -dracut_install /usr/lib64/python3.6/urllib +dracut_install /usr/lib64/python3.9 +dracut_install /usr/lib64/python3.9/collections +dracut_install /usr/lib64/python3.9/ctypes +dracut_install /usr/lib64/python3.9/email +dracut_install /usr/lib64/python3.9/encodings +dracut_install /usr/lib64/python3.9/http +dracut_install /usr/lib64/python3.9/lib-dynload +dracut_install /usr/lib64/python3.9/lib-dynload/fcntl.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/binascii.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_bisect.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_blake2.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_ctypes.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_datetime.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_hashlib.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_heapq.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/math.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_posixsubprocess.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_random.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/select.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_socket.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_ssl.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/_struct.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/unicodedata.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/site-packages +dracut_install /usr/lib64/python3.9/urllib dracut_install /usr/lib/locale/en_US.utf8/LC_ADDRESS dracut_install /usr/lib/locale/en_US.utf8/LC_COLLATE dracut_install /usr/lib/locale/en_US.utf8/LC_CTYPE @@ -45,96 +44,94 @@ dracut_install /usr/lib/locale/en_US.utf8/LC_NUMERIC dracut_install /usr/lib/locale/en_US.utf8/LC_PAPER dracut_install /usr/lib/locale/en_US.utf8/LC_TELEPHONE dracut_install /usr/lib/locale/en_US.utf8/LC_TIME -dracut_install /usr/lib/python3.6/site-packages -dracut_install /usr/lib64/python3.6/argparse.py -dracut_install /usr/lib64/python3.6/codecs.py -dracut_install /usr/lib64/python3.6/encodings/aliases.py -dracut_install /usr/lib64/python3.6/encodings/utf_8.py -dracut_install /usr/lib64/python3.6/encodings/latin_1.py -dracut_install /usr/lib64/python3.6/encodings/ascii.py -dracut_install /usr/lib64/python3.6/encodings/idna.py -dracut_install /usr/lib64/python3.6/io.py -dracut_install /usr/lib64/python3.6/abc.py -dracut_install /usr/lib64/python3.6/_weakrefset.py -dracut_install /usr/lib64/python3.6/weakref.py -dracut_install /usr/lib64/python3.6/site.py -dracut_install /usr/lib64/python3.6/stat.py -dracut_install /usr/lib64/python3.6/posixpath.py -dracut_install /usr/lib64/python3.6/genericpath.py -dracut_install /usr/lib64/python3.6/_collections_abc.py -dracut_install /usr/lib64/python3.6/_sitebuiltins.py -dracut_install /usr/lib64/python3.6/sysconfig.py -dracut_install /usr/lib64/python3.6/_sysconfigdata_m_linux_x86_64-linux-gnu.py -dracut_install /usr/lib64/python3.6/encodings/__init__.py -dracut_install /usr/lib64/python3.6/socket.py -dracut_install /usr/lib64/python3.6/selectors.py +dracut_install /usr/lib/python3.9/site-packages +dracut_install /usr/lib64/python3.9/argparse.py +dracut_install /usr/lib64/python3.9/codecs.py +dracut_install /usr/lib64/python3.9/encodings/aliases.py +dracut_install /usr/lib64/python3.9/encodings/utf_8.py +dracut_install /usr/lib64/python3.9/encodings/latin_1.py +dracut_install /usr/lib64/python3.9/encodings/ascii.py +dracut_install /usr/lib64/python3.9/encodings/idna.py +dracut_install /usr/lib64/python3.9/io.py +dracut_install /usr/lib64/python3.9/abc.py +dracut_install /usr/lib64/python3.9/_weakrefset.py +dracut_install /usr/lib64/python3.9/weakref.py +dracut_install /usr/lib64/python3.9/site.py +dracut_install /usr/lib64/python3.9/stat.py +dracut_install /usr/lib64/python3.9/posixpath.py +dracut_install /usr/lib64/python3.9/genericpath.py +dracut_install /usr/lib64/python3.9/_collections_abc.py +dracut_install /usr/lib64/python3.9/_sitebuiltins.py +dracut_install /usr/lib64/python3.9/sysconfig.py +dracut_install /usr/lib64/python3.9/_sysconfigdata_d_linux_x86_64-linux-gnu.py +dracut_install /usr/lib64/python3.9/encodings/__init__.py +dracut_install /usr/lib64/python3.9/socket.py +dracut_install /usr/lib64/python3.9/selectors.py dracut_install /usr/share/locale/locale.alias -dracut_install /usr/lib64/python3.6/collections/__init__.py -dracut_install /usr/lib64/python3.6/operator.py -dracut_install /usr/lib64/python3.6/keyword.py -dracut_install /usr/lib64/python3.6/heapq.py -dracut_install /usr/lib64/python3.6/reprlib.py -dracut_install /usr/lib64/python3.6/enum.py -dracut_install /usr/lib64/python3.6/types.py -dracut_install /usr/lib64/python3.6/functools.py -dracut_install /usr/lib64/python3.6/collections/abc.py -dracut_install /usr/lib64/python3.6/http/client.py -dracut_install /usr/lib64/python3.6/email/parser.py -dracut_install /usr/lib64/python3.6/email/feedparser.py -dracut_install /usr/lib64/python3.6/re.py -dracut_install /usr/lib64/python3.6/sre_compile.py -dracut_install /usr/lib64/python3.6/sre_parse.py -dracut_install /usr/lib64/python3.6/sre_constants.py -dracut_install /usr/lib64/python3.6/copyreg.py -dracut_install /usr/lib64/python3.6/email/errors.py -dracut_install /usr/lib64/python3.6/email/_policybase.py -dracut_install /usr/lib64/python3.6/email/header.py -dracut_install /usr/lib64/python3.6/email/quoprimime.py -dracut_install /usr/lib64/python3.6/string.py -dracut_install /usr/lib64/python3.6/stringprep.py -dracut_install /usr/lib64/python3.6/email/base64mime.py -dracut_install /usr/lib64/python3.6/base64.py -dracut_install /usr/lib64/python3.6/struct.py -dracut_install /usr/lib64/python3.6/email/charset.py -dracut_install /usr/lib64/python3.6/email/encoders.py -dracut_install /usr/lib64/python3.6/quopri.py -dracut_install /usr/lib64/python3.6/email/utils.py -dracut_install /usr/lib64/python3.6/random.py -dracut_install /usr/lib64/python3.6/warnings.py -dracut_install /usr/lib64/python3.6/hashlib.py -dracut_install /usr/lib64/python3.6/bisect.py -dracut_install /usr/lib64/python3.6/datetime.py -dracut_install /usr/lib64/python3.6/urllib/parse.py -dracut_install /usr/lib64/python3.6/email/_parseaddr.py -dracut_install /usr/lib64/python3.6/calendar.py -dracut_install /usr/lib64/python3.6/locale.py -dracut_install /usr/lib64/python3.6/email/message.py -dracut_install /usr/lib64/python3.6/uu.py -dracut_install /usr/lib64/python3.6/email/_encoded_words.py -dracut_install /usr/lib64/python3.6/email/iterators.py -dracut_install /usr/lib64/python3.6/http/__init__.py -dracut_install /usr/lib64/python3.6/http/cookies.py -dracut_install /usr/lib64/python3.6/argparse.py -dracut_install /usr/lib64/python3.6/copy.py -dracut_install /usr/lib64/python3.6/textwrap.py -dracut_install /usr/lib64/python3.6/gettext.py -dracut_install /usr/lib64/python3.6/subprocess.py -dracut_install /usr/lib64/python3.6/signal.py -dracut_install /usr/lib64/python3.6/threading.py -dracut_install /usr/lib64/python3.6/traceback.py -dracut_install /usr/lib64/python3.6/dummy_threading.py -dracut_install /usr/lib64/python3.6/_dummy_thread.py -dracut_install /usr/lib64/python3.6/linecache.py -dracut_install /usr/lib64/python3.6/tokenize.py -dracut_install /usr/lib64/python3.6/token.py -dracut_install /usr/lib64/python3.6/shlex.py -dracut_install /usr/lib64/python3.6/configparser.py -dracut_install /usr/lib64/python3.6/lib-dynload/readline.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/ctypes/__init__.py -dracut_install /usr/lib64/python3.6/ctypes/_endian.py -dracut_install /usr/lib64/python3.6/ctypes/util.py -dracut_install /usr/lib64/python3.6/ssl.py -dracut_install /usr/lib64/python3.6/ipaddress.py +dracut_install /usr/lib64/python3.9/collections/__init__.py +dracut_install /usr/lib64/python3.9/operator.py +dracut_install /usr/lib64/python3.9/keyword.py +dracut_install /usr/lib64/python3.9/heapq.py +dracut_install /usr/lib64/python3.9/reprlib.py +dracut_install /usr/lib64/python3.9/enum.py +dracut_install /usr/lib64/python3.9/types.py +dracut_install /usr/lib64/python3.9/functools.py +dracut_install /usr/lib64/python3.9/collections/abc.py +dracut_install /usr/lib64/python3.9/http/client.py +dracut_install /usr/lib64/python3.9/email/parser.py +dracut_install /usr/lib64/python3.9/email/feedparser.py +dracut_install /usr/lib64/python3.9/re.py +dracut_install /usr/lib64/python3.9/sre_compile.py +dracut_install /usr/lib64/python3.9/sre_parse.py +dracut_install /usr/lib64/python3.9/sre_constants.py +dracut_install /usr/lib64/python3.9/copyreg.py +dracut_install /usr/lib64/python3.9/email/errors.py +dracut_install /usr/lib64/python3.9/email/_policybase.py +dracut_install /usr/lib64/python3.9/email/header.py +dracut_install /usr/lib64/python3.9/email/quoprimime.py +dracut_install /usr/lib64/python3.9/string.py +dracut_install /usr/lib64/python3.9/stringprep.py +dracut_install /usr/lib64/python3.9/email/base64mime.py +dracut_install /usr/lib64/python3.9/base64.py +dracut_install /usr/lib64/python3.9/struct.py +dracut_install /usr/lib64/python3.9/email/charset.py +dracut_install /usr/lib64/python3.9/email/encoders.py +dracut_install /usr/lib64/python3.9/quopri.py +dracut_install /usr/lib64/python3.9/email/utils.py +dracut_install /usr/lib64/python3.9/random.py +dracut_install /usr/lib64/python3.9/warnings.py +dracut_install /usr/lib64/python3.9/hashlib.py +dracut_install /usr/lib64/python3.9/bisect.py +dracut_install /usr/lib64/python3.9/datetime.py +dracut_install /usr/lib64/python3.9/urllib/parse.py +dracut_install /usr/lib64/python3.9/email/_parseaddr.py +dracut_install /usr/lib64/python3.9/calendar.py +dracut_install /usr/lib64/python3.9/locale.py +dracut_install /usr/lib64/python3.9/email/message.py +dracut_install /usr/lib64/python3.9/uu.py +dracut_install /usr/lib64/python3.9/email/_encoded_words.py +dracut_install /usr/lib64/python3.9/email/iterators.py +dracut_install /usr/lib64/python3.9/http/__init__.py +dracut_install /usr/lib64/python3.9/http/cookies.py +dracut_install /usr/lib64/python3.9/argparse.py +dracut_install /usr/lib64/python3.9/copy.py +dracut_install /usr/lib64/python3.9/textwrap.py +dracut_install /usr/lib64/python3.9/gettext.py +dracut_install /usr/lib64/python3.9/subprocess.py +dracut_install /usr/lib64/python3.9/signal.py +dracut_install /usr/lib64/python3.9/threading.py +dracut_install /usr/lib64/python3.9/traceback.py +dracut_install /usr/lib64/python3.9/linecache.py +dracut_install /usr/lib64/python3.9/tokenize.py +dracut_install /usr/lib64/python3.9/token.py +dracut_install /usr/lib64/python3.9/shlex.py +dracut_install /usr/lib64/python3.9/configparser.py +dracut_install /usr/lib64/python3.9/lib-dynload/readline.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/ctypes/__init__.py +dracut_install /usr/lib64/python3.9/ctypes/_endian.py +dracut_install /usr/lib64/python3.9/ctypes/util.py +dracut_install /usr/lib64/python3.9/ssl.py +dracut_install /usr/lib64/python3.9/ipaddress.py dracut_install /usr/lib/locale/en_US.utf8/LC_ADDRESS dracut_install /usr/lib/locale/en_US.utf8/LC_IDENTIFICATION dracut_install /usr/lib/locale/en_US.utf8/LC_MEASUREMENT @@ -147,78 +144,77 @@ dracut_install /usr/lib/locale/en_US.utf8/LC_CTYPE dracut_install /usr/lib/locale/en_US.utf8/LC_NAME dracut_install /usr/lib/locale/en_US.utf8/LC_NUMERIC dracut_install /usr/lib/locale/en_US.utf8/LC_PAPER -dracut_install /usr/lib64/python3.6/json/__init__.py /usr/lib64/python3.6/json/decoder.py /usr/lib64/python3.6/json/encoder.py /usr/lib64/python3.6/json/scanner.py /usr/lib64/python3.6/json/tool.py /usr/lib64/python3.6/lib-dynload/_json.cpython-36m-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/json/__init__.py /usr/lib64/python3.9/json/decoder.py /usr/lib64/python3.9/json/encoder.py /usr/lib64/python3.9/json/scanner.py /usr/lib64/python3.9/json/tool.py /usr/lib64/python3.9/lib-dynload/_json.cpython-39-x86_64-linux-gnu.so # ansible dependencies -dracut_install /usr/lib64/python3.6/runpy.py -dracut_install /usr/lib64/python3.6/importlib/__init__.py -dracut_install /usr/lib64/python3.6/importlib/_bootstrap.py -dracut_install /usr/lib64/python3.6/importlib/_bootstrap_external.py -dracut_install /usr/lib64/python3.6/importlib/abc.py -dracut_install /usr/lib64/python3.6/importlib/machinery.py -dracut_install /usr/lib64/python3.6/importlib/util.py -dracut_install /usr/lib64/python3.6/contextlib.py -dracut_install /usr/lib64/python3.6/pkgutil.py -dracut_install /usr/lib64/python3.6/shutil.py -dracut_install /usr/lib64/python3.6/fnmatch.py -dracut_install /usr/lib64/python3.6/tempfile.py -dracut_install /usr/lib64/python3.6/zipfile.py -dracut_install /usr/lib64/python3.6/encodings/cp437.py -dracut_install /usr/lib64/python3.6/lib-dynload/zlib.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/grp.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/lib-dynload/array.cpython-36m-x86_64-linux-gnu.so -dracut_install /usr/lib64/python3.6/__future__.py -dracut_install /usr/lib64/python3.6/platform.py -dracut_install /usr/lib64/python3.6/logging/__init__.py -dracut_install /usr/lib64/python3.6/logging/config.py -dracut_install /usr/lib64/python3.6/logging/handlers.py -dracut_install /usr/lib64/python3.6/optparse.py -dracut_install /usr/lib64/python3.6/ast.py -dracut_install /usr/lib64/python3.6/multiprocessing/__init__.py -dracut_install /usr/lib64/python3.6/multiprocessing/connection.py -dracut_install /usr/lib64/python3.6/multiprocessing/context.py -dracut_install /usr/lib64/python3.6/multiprocessing/dummy/__init__.py -dracut_install /usr/lib64/python3.6/multiprocessing/dummy/connection.py -dracut_install /usr/lib64/python3.6/multiprocessing/forkserver.py -dracut_install /usr/lib64/python3.6/multiprocessing/heap.py -dracut_install /usr/lib64/python3.6/multiprocessing/managers.py -dracut_install /usr/lib64/python3.6/multiprocessing/pool.py -dracut_install /usr/lib64/python3.6/multiprocessing/popen_fork.py -dracut_install /usr/lib64/python3.6/multiprocessing/popen_forkserver.py -dracut_install /usr/lib64/python3.6/multiprocessing/popen_spawn_posix.py -dracut_install /usr/lib64/python3.6/multiprocessing/popen_spawn_win32.py -dracut_install /usr/lib64/python3.6/multiprocessing/process.py -dracut_install /usr/lib64/python3.6/multiprocessing/queues.py -dracut_install /usr/lib64/python3.6/multiprocessing/reduction.py -dracut_install /usr/lib64/python3.6/multiprocessing/resource_sharer.py -dracut_install /usr/lib64/python3.6/multiprocessing/semaphore_tracker.py -dracut_install /usr/lib64/python3.6/multiprocessing/sharedctypes.py -dracut_install /usr/lib64/python3.6/multiprocessing/spawn.py -dracut_install /usr/lib64/python3.6/multiprocessing/synchronize.py -dracut_install /usr/lib64/python3.6/multiprocessing/util.py -dracut_install /usr/lib64/python3.6/pickle.py -dracut_install /usr/lib64/python3.6/_compat_pickle.py -dracut_install /usr/lib64/python3.6/queue.py -dracut_install /usr/lib64/python3.6/glob.py -dracut_install /usr/lib64/python3.6/distutils/__init__.py -dracut_install /usr/lib64/python3.6/distutils/archive_util.py -dracut_install /usr/lib64/python3.6/distutils/cmd.py -dracut_install /usr/lib64/python3.6/distutils/config.py -dracut_install /usr/lib64/python3.6/distutils/core.py -dracut_install /usr/lib64/python3.6/distutils/debug.py -dracut_install /usr/lib64/python3.6/distutils/dep_util.py -dracut_install /usr/lib64/python3.6/distutils/dir_util.py -dracut_install /usr/lib64/python3.6/distutils/errors.py -dracut_install /usr/lib64/python3.6/distutils/extension.py -dracut_install /usr/lib64/python3.6/distutils/fancy_getopt.py -dracut_install /usr/lib64/python3.6/distutils/file_util.py -dracut_install /usr/lib64/python3.6/distutils/filelist.py -dracut_install /usr/lib64/python3.6/distutils/log.py -dracut_install /usr/lib64/python3.6/distutils/spawn.py -dracut_install /usr/lib64/python3.6/distutils/sysconfig.py -dracut_install /usr/lib64/python3.6/distutils/text_file.py -dracut_install /usr/lib64/python3.6/distutils/util.py -dracut_install /usr/lib64/python3.6/distutils/version.py -dracut_install /usr/lib64/python3.6/distutils/versionpredicate.py -dracut_install /usr/lib64/python3.6/getpass.py +dracut_install /usr/lib64/python3.9/runpy.py +dracut_install /usr/lib64/python3.9/importlib/__init__.py +dracut_install /usr/lib64/python3.9/importlib/_bootstrap.py +dracut_install /usr/lib64/python3.9/importlib/_bootstrap_external.py +dracut_install /usr/lib64/python3.9/importlib/abc.py +dracut_install /usr/lib64/python3.9/importlib/machinery.py +dracut_install /usr/lib64/python3.9/importlib/util.py +dracut_install /usr/lib64/python3.9/contextlib.py +dracut_install /usr/lib64/python3.9/pkgutil.py +dracut_install /usr/lib64/python3.9/shutil.py +dracut_install /usr/lib64/python3.9/fnmatch.py +dracut_install /usr/lib64/python3.9/tempfile.py +dracut_install /usr/lib64/python3.9/zipfile.py +dracut_install /usr/lib64/python3.9/encodings/cp437.pyc +dracut_install /usr/lib64/python3.9/lib-dynload/zlib.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/grp.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/lib-dynload/array.cpython-39-x86_64-linux-gnu.so +dracut_install /usr/lib64/python3.9/__future__.py +dracut_install /usr/lib64/python3.9/platform.py +dracut_install /usr/lib64/python3.9/logging/__init__.py +dracut_install /usr/lib64/python3.9/logging/config.py +dracut_install /usr/lib64/python3.9/logging/handlers.py +dracut_install /usr/lib64/python3.9/optparse.py +dracut_install /usr/lib64/python3.9/ast.py +dracut_install /usr/lib64/python3.9/multiprocessing/__init__.py +dracut_install /usr/lib64/python3.9/multiprocessing/connection.py +dracut_install /usr/lib64/python3.9/multiprocessing/context.py +dracut_install /usr/lib64/python3.9/multiprocessing/dummy/__init__.py +dracut_install /usr/lib64/python3.9/multiprocessing/dummy/connection.py +dracut_install /usr/lib64/python3.9/multiprocessing/forkserver.py +dracut_install /usr/lib64/python3.9/multiprocessing/heap.py +dracut_install /usr/lib64/python3.9/multiprocessing/managers.py +dracut_install /usr/lib64/python3.9/multiprocessing/pool.py +dracut_install /usr/lib64/python3.9/multiprocessing/popen_fork.py +dracut_install /usr/lib64/python3.9/multiprocessing/popen_forkserver.py +dracut_install /usr/lib64/python3.9/multiprocessing/popen_spawn_posix.py +dracut_install /usr/lib64/python3.9/multiprocessing/popen_spawn_win32.py +dracut_install /usr/lib64/python3.9/multiprocessing/process.py +dracut_install /usr/lib64/python3.9/multiprocessing/queues.py +dracut_install /usr/lib64/python3.9/multiprocessing/reduction.py +dracut_install /usr/lib64/python3.9/multiprocessing/resource_sharer.py +dracut_install /usr/lib64/python3.9/multiprocessing/sharedctypes.py +dracut_install /usr/lib64/python3.9/multiprocessing/spawn.py +dracut_install /usr/lib64/python3.9/multiprocessing/synchronize.py +dracut_install /usr/lib64/python3.9/multiprocessing/util.py +dracut_install /usr/lib64/python3.9/pickle.py +dracut_install /usr/lib64/python3.9/_compat_pickle.py +dracut_install /usr/lib64/python3.9/queue.py +dracut_install /usr/lib64/python3.9/glob.py +dracut_install /usr/lib64/python3.9/distutils/__init__.py +dracut_install /usr/lib64/python3.9/distutils/archive_util.py +dracut_install /usr/lib64/python3.9/distutils/cmd.py +dracut_install /usr/lib64/python3.9/distutils/config.py +dracut_install /usr/lib64/python3.9/distutils/core.py +dracut_install /usr/lib64/python3.9/distutils/debug.py +dracut_install /usr/lib64/python3.9/distutils/dep_util.py +dracut_install /usr/lib64/python3.9/distutils/dir_util.py +dracut_install /usr/lib64/python3.9/distutils/errors.py +dracut_install /usr/lib64/python3.9/distutils/extension.py +dracut_install /usr/lib64/python3.9/distutils/fancy_getopt.py +dracut_install /usr/lib64/python3.9/distutils/file_util.py +dracut_install /usr/lib64/python3.9/distutils/filelist.py +dracut_install /usr/lib64/python3.9/distutils/log.py +dracut_install /usr/lib64/python3.9/distutils/spawn.py +dracut_install /usr/lib64/python3.9/distutils/sysconfig.py +dracut_install /usr/lib64/python3.9/distutils/text_file.py +dracut_install /usr/lib64/python3.9/distutils/util.py +dracut_install /usr/lib64/python3.9/distutils/version.py +dracut_install /usr/lib64/python3.9/distutils/versionpredicate.py +dracut_install /usr/lib64/python3.9/getpass.py dracut_install /usr/libexec/openssh/sftp-server diff --git a/genesis/confluent-genesis.spec b/genesis/confluent-genesis.spec index c1b7d45c..beaeb5cd 100644 --- a/genesis/confluent-genesis.spec +++ b/genesis/confluent-genesis.spec @@ -1,5 +1,5 @@ %define arch x86_64 -Version: 3.7.1 +Version: 3.10.0 Release: 1 Name: confluent-genesis-%{arch} BuildArch: noarch diff --git a/genesis/extracttmuxlicenses.py b/genesis/extracttmuxlicenses.py index 9c10922d..1a6eeba5 100644 --- a/genesis/extracttmuxlicenses.py +++ b/genesis/extracttmuxlicenses.py @@ -1,8 +1,18 @@ import glob +import os yearsbyname = {} namesbylicense = {} filesbylicense = {} -for source in glob.glob('*.c'): +allfiles = glob.glob('*.c') +allfiles.extend(glob.glob('*.y')) +allfiles.extend(glob.glob('compat/*.c')) +foundbin = False +for source in allfiles: # glob.glob('*.c'): + if 'cmd-parse.c' == source: + continue + if not os.path.exists(source.replace('.c', '.o')): + continue + foundbin = True with open(source, 'r') as sourcein: cap = False thelicense = '' @@ -19,13 +29,27 @@ for source in glob.glob('*.c'): line = line[3:] if line.startswith('Author: '): continue + + if line == 'Copyright (c) 1989, 1993\n': + name = 'The Regents of the University of California' + if name not in yearsbyname: + yearsbyname[name] = set([]) + yearsbyname[name].add('1989') + yearsbyname[name].add('1993') + continue if line.startswith('Copyright'): _, _, years, name = line.split(maxsplit=3) - name = name.split('>', 1)[0] + '>' + if '>' in name: + name = name.split('>', 1)[0] + '>' currnames.add(name) if name not in yearsbyname: yearsbyname[name] = set([]) - yearsbyname[name].add(years) + if '-' in years: + strt, end = years.split('-') + for x in range(int(strt), int(end) + 1): + print(str(x)) + else: + yearsbyname[name].add(years) continue thelicense += line if thelicense not in namesbylicense: diff --git a/genesis/getlicenses.py b/genesis/getlicenses.py index a0118c48..e87e2786 100644 --- a/genesis/getlicenses.py +++ b/genesis/getlicenses.py @@ -61,7 +61,7 @@ for lic in sorted(licenses): print(lic) manualrpms = [ 'ipmitool', - 'almalinux-release', + 'centos-stream-release', 'libaio', 'hwdata', 'snmp', @@ -92,7 +92,7 @@ manuallicenses = [ '/usr/share/licenses/bash/NOTICE', '/usr/share/licenses/libsepol/NOTICE', '/usr/share/licenses/perl/COPYING.regexec', # regexec.c - '/usr/share/doc/platform-python/README.rst', + '/usr/share/doc/python3/README.rst', '/usr/share/licenses/lz4/LICENSE', '/usr/share/licenses/lm_sensors/COPYING', '/usr/share/doc/libunistring/README', @@ -111,14 +111,6 @@ manuallicenses = [ '/usr/share/licenses/tmux/NOTICE', # built by extracttmuxlicenses.py '/usr/share/licenses/tmux/COPYING', # extracted from source '/usr/share/licenses/tmux/README', # extracted from source - '/usr/share/licenses/kernel-extra/exceptions/Linux-syscall-note', - '/usr/share/licenses/kernel-extra/other/Apache-2.0', - '/usr/share/licenses/kernel-extra/other/CC-BY-SA-4.0', - '/usr/share/licenses/kernel-extra/other/CDDL-1.0', - '/usr/share/licenses/kernel-extra/other/GPL-1.0', - '/usr/share/licenses/kernel-extra/other/Linux-OpenIB', - '/usr/share/licenses/kernel-extra/other/MPL-1.1', - '/usr/share/licenses/kernel-extra/other/X11', '/usr/share/licenses/kernel-extra/preferred/BSD-2-Clause', '/usr/share/licenses/kernel-extra/preferred/BSD-3-Clause', '/usr/share/licenses/kernel-extra/preferred/BSD-3-Clause-Clear', @@ -126,12 +118,25 @@ manuallicenses = [ '/usr/share/licenses/kernel-extra/preferred/LGPL-2.0', '/usr/share/licenses/kernel-extra/preferred/LGPL-2.1', '/usr/share/licenses/kernel-extra/preferred/MIT', + '/usr/share/licenses/kernel-extra/deprecated/GFDL-1.1', + '/usr/share/licenses/kernel-extra/deprecated/GFDL-1.2', + '/usr/share/licenses/kernel-extra/deprecated/GPL-1.0', + '/usr/share/licenses/kernel-extra/deprecated/ISC', + '/usr/share/licenses/kernel-extra/deprecated/Linux-OpenIB', + '/usr/share/licenses/kernel-extra/deprecated/X11', + '/usr/share/licenses/kernel-extra/deprecated/Zlib', + '/usr/share/licenses/kernel-extra/dual/Apache-2.0', + '/usr/share/licenses/kernel-extra/dual/CC-BY-4.0', + '/usr/share/licenses/kernel-extra/dual/CDDL-1.0', + '/usr/share/licenses/kernel-extra/dual/MPL-1.1', + '/usr/share/licenses/kernel-extra/exceptions/GCC-exception-2.0', + '/usr/share/licenses/kernel-extra/exceptions/Linux-syscall-note', '/usr/share/licenses/util-linux/COPYING.GPLv3', # extract from parse-date.c, from srpm - '/usr/share/licenses/kmod/tools/COPYING', # GPL not LGPL, must extract from kmod srpm + '/usr/share/licenses/kmod/COPYING', # GPL not LGPL, must extract from kmod srpm '/usr/share/licenses/krb5-libs/NOTICE', # copy it verbatim from LICENSE, exact same file '/usr/share/doc/less/README', - '/usr/share/almalinux-release/EULA', - '/usr/share/doc/almalinux-release/GPL', + '/usr/share/centos-release/EULA', + #'/usr/share/doc/almalinux-release/GPL', '/usr/share/licenses/libcap-ng-utils/COPYING', '/usr/share/licenses/libdb/copyright', # from libdb, db-5.3.28, lang/sql/odbc/debian/copyright '/usr/share/licenses/libgcrypt/LICENSES.ppc-aes-gcm', # libgcrypt license to carry forward @@ -140,6 +145,7 @@ manuallicenses = [ ] for lic in manuallicenses: print(lic) +missinglics = [] for rpm in rpmlist: if not rpm: continue @@ -148,7 +154,9 @@ for rpm in rpmlist: break else: if rpm not in licensesbyrpm: - raise Exception('Unresolved license info for ' + rpm) - print("UH OH: " + rpm) - + missinglics.append(rpm) +if missinglics: + for lic in missinglics: + print("Missing: " + lic) + raise Exception("Missing licenses: " + ','.join(missinglics)) From bb414524ad3745675c5db2afec4117ad1d19b0fe Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 30 May 2024 08:14:58 -0400 Subject: [PATCH 176/319] Add fallback if timedatectl can't run. --- confluent_server/confluent/selfservice.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index a166e0fb..438d5b3e 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -378,6 +378,8 @@ def handle_request(env, start_response): tdc = util.run(['timedatectl'])[0].split(b'\n') except subprocess.CalledProcessError: tdc = [] + currtzvintage = time.time() + ncfg['timezone'] = currtz for ent in tdc: ent = ent.strip() if ent.startswith(b'Time zone:'): From 6564f8de7257df9c1e801ab9507241264edfe240 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 5 Jun 2024 08:39:37 -0400 Subject: [PATCH 177/319] Update license material --- confluent_client/COPYRIGHT | 60 +++++++++++++++++++++++++- confluent_server/COPYRIGHT | 87 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 5 deletions(-) diff --git a/confluent_client/COPYRIGHT b/confluent_client/COPYRIGHT index 6bcd9986..6f435939 100644 --- a/confluent_client/COPYRIGHT +++ b/confluent_client/COPYRIGHT @@ -1,6 +1,8 @@ -Name: confluent-client Project: https://hpc.lenovo.com/users/ Source: https://github.com/lenovo/confluent +Upstream-Name: confluent-client + +All file of the Confluent-client software is distributed under the terms of an Apache-2.0 license as indicated below: Files: * Copyright: 2014-2019 Lenovo @@ -11,7 +13,8 @@ Copyright: 2014 IBM Corporation 2015-2019 Lenovo License: Apache-2.0 -File: sortutil.py +File: sortutil.py, + tlvdata.py Copyright: 2014 IBM Corporation 2015-2016 Lenovo License: Apache-2.0 @@ -19,3 +22,56 @@ License: Apache-2.0 File: tlv.py Copyright: 2014 IBM Corporation License: Apache-2.0 + +License: Apache-2.0 +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and +You must cause any modified files to carry prominent notices stating that You changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/confluent_server/COPYRIGHT b/confluent_server/COPYRIGHT index 6bcd9986..05d7bac2 100644 --- a/confluent_server/COPYRIGHT +++ b/confluent_server/COPYRIGHT @@ -1,21 +1,102 @@ -Name: confluent-client Project: https://hpc.lenovo.com/users/ Source: https://github.com/lenovo/confluent +Upstream-Name: confluent-client + +All file of the Confluent-client software is distributed under the terms of an Apache-2.0 license as indicated below: Files: * -Copyright: 2014-2019 Lenovo +Copyright: 2014-2023 Lenovo License: Apache-2.0 -File: client.py: +File: client.py + sockapi.py + redfish.py + attributes.py + httpapi.py + configmanager.py Copyright: 2014 IBM Corporation 2015-2019 Lenovo License: Apache-2.0 File: sortutil.py + log.py Copyright: 2014 IBM Corporation 2015-2016 Lenovo License: Apache-2.0 +File: util.py + noderange.py + main.py + exceptions.py +Copyright: 2014 IBM Corporation + 2015-2017 Lenovo +License: Apache-2.0 + +File: ipmi.py + plugin.py + core.py + consoleserver.py +Copyright: 2014 IBM Corporation + 2015-2018 Lenovo +License: Apache-2.0 + File: tlv.py + shellmodule.py + conf.py + auth.py Copyright: 2014 IBM Corporation License: Apache-2.0 + + +License: Apache-2.0 +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and +You must cause any modified files to carry prominent notices stating that You changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS \ No newline at end of file From 451e3ba224e572da4f9820f9ca1b0f487c1cbf7d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 6 Jun 2024 09:02:45 -0400 Subject: [PATCH 178/319] Add nodeapply man page --- confluent_client/doc/man/nodeapply.ronn | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 confluent_client/doc/man/nodeapply.ronn diff --git a/confluent_client/doc/man/nodeapply.ronn b/confluent_client/doc/man/nodeapply.ronn new file mode 100644 index 00000000..dae35025 --- /dev/null +++ b/confluent_client/doc/man/nodeapply.ronn @@ -0,0 +1,30 @@ +nodeapply(8) -- Execute command on many nodes in a noderange through ssh +========================================================================= + +## SYNOPSIS + +`nodeapply [options] ` + +## DESCRIPTION + +Provides shortcut access to a number of common operations against deployed +nodes. These operations include refreshing ssh certificates and configuration, +rerunning syncflies, and executing specified postscripts. + +## OPTIONS + +* `-k`, `--security` + Refresh SSH configuration (hosts.equiv and node SSH certificates) + +* `-F`, `--sync` + Rerun syncfiles from deployed profile + +* `-P SCRIPTS`, `--scripts=SCRIPTS` + Re-run specified scripts, with full path under scripts specified, e.g. post.d/scriptname,firstboot.d/otherscriptname + +* `-c COUNT`, `-f COUNT`, `--count=COUNT` + Specify the maximum number of instances to run concurrently + +* `-m MAXNODES`, `--maxnodes=MAXNODES` + Specify a maximum number of nodes to run remote ssh command to, prompting + if over the threshold From 5d416cb1b1e6df6b967c78ab9456f15d0c31d0d6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 6 Jun 2024 11:23:28 -0400 Subject: [PATCH 179/319] Remove disused dependency list --- confluent_server/requirements.txt | 3 --- confluent_server/setup.py.tmpl | 3 --- 2 files changed, 6 deletions(-) diff --git a/confluent_server/requirements.txt b/confluent_server/requirements.txt index d0be6908..e69de29b 100644 --- a/confluent_server/requirements.txt +++ b/confluent_server/requirements.txt @@ -1,3 +0,0 @@ -confluent_client>=0.1 -pycrypto>=2.6 -pyparsing diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index 871497e3..8987e19d 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -23,9 +23,6 @@ setup( 'confluent/plugins/shell/', 'confluent/collective/', 'confluent/plugins/configuration/'], - install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet', - 'dnspython', 'netifaces', 'pysnmp', 'pyparsing', - 'pyghmi>=1.0.44'], scripts=['bin/confluent', 'bin/confluent_selfcheck', 'bin/confluentdbutil', 'bin/collective', 'bin/osdeploy'], data_files=[('/etc/init.d', ['sysvinit/confluent']), ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']), From 38a95131f905f28ef3b4692e5176868f2e22c0c9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 7 Jun 2024 09:36:18 -0400 Subject: [PATCH 180/319] Add more error handling in vtbufferd --- confluent_vtbufferd/vtbufferd.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/confluent_vtbufferd/vtbufferd.c b/confluent_vtbufferd/vtbufferd.c index 055a5263..bbcaba6b 100644 --- a/confluent_vtbufferd/vtbufferd.c +++ b/confluent_vtbufferd/vtbufferd.c @@ -232,10 +232,11 @@ int main(int argc, char* argv[]) { int numevts; int status; int poller; - int n; + int n, rt; socklen_t len; - int ctlsock, currsock; - socklen_t addrlen; + int ctlsock = 0; + int currsock = 0; + socklen_t addrlen = 0; struct ucred ucr; struct epoll_event epvt, evts[MAXEVTS]; @@ -247,6 +248,10 @@ int main(int argc, char* argv[]) { addr.sun_family = AF_UNIX; strncpy(addr.sun_path + 1, argv[1], sizeof(addr.sun_path) - 2); // abstract namespace socket ctlsock = socket(AF_UNIX, SOCK_STREAM, 0); + if (ctlsock < 0) { + perror("Unable to open unix socket - "); + exit(1); + } status = bind(ctlsock, (const struct sockaddr*)&addr, sizeof(sa_family_t) + strlen(argv[1]) + 1); //sizeof(struct sockaddr_un)); if (status < 0) { perror("Unable to open unix socket - "); @@ -272,7 +277,11 @@ int main(int argc, char* argv[]) { if (evts[n].data.fd == ctlsock) { currsock = accept(ctlsock, (struct sockaddr *) &addr, &addrlen); len = sizeof(ucr); - getsockopt(currsock, SOL_SOCKET, SO_PEERCRED, &ucr, &len); + rt = getsockopt(currsock, SOL_SOCKET, SO_PEERCRED, &ucr, &len); + if (rt < 0) { + close(currsock); + continue; + } if (ucr.uid != getuid()) { // block access for other users close(currsock); continue; From c6adf8175a42c5d479d60142fb75651e88efee6e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 7 Jun 2024 11:03:00 -0400 Subject: [PATCH 181/319] Try processing driver disks before udevadm invocation --- .../lib/dracut/hooks/pre-trigger/01-confluent.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index 8701cc7e..62f406ed 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -1,6 +1,13 @@ #!/bin/sh [ -e /tmp/confluent.initq ] && return 0 . /lib/dracut-lib.sh +if [ -f /tmp/dd_disk ]; then + for dd in $(cat /tmp/dd_disk); do + if [ -e $dd ]; then + driver-updates --disk $dd $dd + fi + done +fi setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' & udevadm trigger udevadm trigger --type=devices --action=add @@ -20,13 +27,6 @@ function confluentpython() { /usr/bin/python2 $* fi } -if [ -f /tmp/dd_disk ]; then - for dd in $(cat /tmp/dd_disk); do - if [ -e $dd ]; then - driver-updates --disk $dd $dd - fi - done -fi vlaninfo=$(getarg vlan) if [ ! -z "$vlaninfo" ]; then vldev=${vlaninfo#*:} From 517242df476df7e55bb9edb0a824cec0e1c2c358 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 7 Jun 2024 11:36:39 -0400 Subject: [PATCH 182/319] Avoid double run of driver disk content --- .../usr/lib/dracut/hooks/pre-trigger/01-confluent.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index 62f406ed..84b75451 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -1,14 +1,16 @@ #!/bin/sh [ -e /tmp/confluent.initq ] && return 0 . /lib/dracut-lib.sh +setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' & if [ -f /tmp/dd_disk ]; then for dd in $(cat /tmp/dd_disk); do if [ -e $dd ]; then driver-updates --disk $dd $dd + rm $dd fi done + rm /tmp/dd_disk fi -setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' & udevadm trigger udevadm trigger --type=devices --action=add udevadm settle From 663b3208682297eccbcdd160f4e36c998ef4d40c Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 7 Jun 2024 12:43:15 -0400 Subject: [PATCH 183/319] Add more bounds checking in copernicus --- confluent_osdeploy/utils/copernicus.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/utils/copernicus.c b/confluent_osdeploy/utils/copernicus.c index 9c8ace60..526c06d2 100644 --- a/confluent_osdeploy/utils/copernicus.c +++ b/confluent_osdeploy/utils/copernicus.c @@ -31,6 +31,8 @@ int add_uuid(char* destination, int maxsize) { strncpy(destination, "/uuid=", maxsize); uuidsize = read(uuidf, destination + 6, maxsize - 6); close(uuidf); + if (uuidsize < 0) { return 0; } + if (uuidsize > 524288) { return 0; } if (destination[uuidsize + 5] == '\n') { destination[uuidsize + 5 ] = 0; } @@ -42,9 +44,11 @@ int add_confluent_uuid(char* destination, int maxsize) { int uuidsize; uuidf = open("/confluent_uuid", O_RDONLY); if (uuidf < 0) { return 0; } - strncpy(destination, "/confluentuuid=", maxsize); uuidsize = read(uuidf, destination + 15, maxsize - 15); close(uuidf); + if (uuidsize < 0) { return 0; } + if (uuidsize > 524288) { return 0; } + strncpy(destination, "/confluentuuid=", maxsize); if (destination[uuidsize + 14] == '\n') { destination[uuidsize + 14] = 0; } From 8db638262909a9f92cd0ea175bc3a68b4e3581b5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 11 Jun 2024 09:25:57 -0400 Subject: [PATCH 184/319] Change autoversion scheme for dev builds We need to be compliant with python versioning for the tools to keep working --- confluent_server/makesetup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/makesetup b/confluent_server/makesetup index f20e2d25..de39ea18 100755 --- a/confluent_server/makesetup +++ b/confluent_server/makesetup @@ -2,7 +2,7 @@ cd `dirname $0` VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + VERSION=$VERSION.dev$NUMCOMMITS+g`git describe|cut -d- -f 3` fi echo $VERSION > VERSION sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py From d06be555684139a7a00de2bdac62a4a656587740 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 11 Jun 2024 15:51:02 -0400 Subject: [PATCH 185/319] Add dependencies for Ubuntu Noble --- confluent_server/builddeb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/builddeb b/confluent_server/builddeb index f71bfce4..4071b5b1 100755 --- a/confluent_server/builddeb +++ b/confluent_server/builddeb @@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then if grep wheezy /etc/os-release; then sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control else - sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd/' debian/control + sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-pyasyncore/' debian/control fi if grep wheezy /etc/os-release; then echo 'confluent_client python-confluent-client' >> debian/pydist-overrides From 8e407cb96e4667164ad6b2b323d8a9cbe970525d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 14 Jun 2024 11:21:46 -0400 Subject: [PATCH 186/319] Correct mistake with confluent uuid copy-in in copernicus --- confluent_osdeploy/utils/copernicus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/utils/copernicus.c b/confluent_osdeploy/utils/copernicus.c index 526c06d2..3b92044c 100644 --- a/confluent_osdeploy/utils/copernicus.c +++ b/confluent_osdeploy/utils/copernicus.c @@ -44,11 +44,11 @@ int add_confluent_uuid(char* destination, int maxsize) { int uuidsize; uuidf = open("/confluent_uuid", O_RDONLY); if (uuidf < 0) { return 0; } + strncpy(destination, "/confluentuuid=", maxsize); uuidsize = read(uuidf, destination + 15, maxsize - 15); close(uuidf); if (uuidsize < 0) { return 0; } if (uuidsize > 524288) { return 0; } - strncpy(destination, "/confluentuuid=", maxsize); if (destination[uuidsize + 14] == '\n') { destination[uuidsize + 14] = 0; } From 162e4d1d1eb164118cc7e60fca861e4429d73481 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sat, 15 Jun 2024 15:43:13 -0400 Subject: [PATCH 187/319] Add retry logic for the el8 identity image support This allows slow linking interfaces to come up rather than being missed --- .../dracut/hooks/pre-trigger/01-confluent.sh | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh index 84b75451..f8b576a2 100644 --- a/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh +++ b/confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh @@ -63,43 +63,48 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null && continue ip link set $(basename $i) up done - for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do - if [ "$autoconfigmethod" = "dhcp" ]; then - /usr/libexec/nm-initrd-generator ip=$NICGUESS:dhcp - else - v4addr=$(grep ^ipv4_address: $tcfg) - v4addr=${v4addr#ipv4_address: } - v4plen=${v4addr#*/} - v4addr=${v4addr%/*} - v4gw=$(grep ^ipv4_gateway: $tcfg) - v4gw=${v4gw#ipv4_gateway: } - ip addr add dev $NICGUESS $v4addr/$v4plen - if [ "$v4gw" = "null" ]; then - v4gw="" - fi - if [ ! -z "$v4gw" ]; then - ip route add default via $v4gw - fi - v4nm=$(grep ipv4_netmask: $tcfg) - v4nm=${v4nm#ipv4_netmask: } - DETECTED=0 - for dsrv in $deploysrvs; do - if curl --capath /tls/ -s --connect-timeout 3 https://$dsrv/confluent-public/ > /dev/null; then - rm /run/NetworkManager/system-connections/* - /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname:$NICGUESS:none - DETECTED=1 - ifname=$NICGUESS + TRIES=30 + DETECTED=0 + while [ "$DETECTED" = 0 ] && [ $TRIES -gt 0 ]; do + TRIES=$((TRIES - 1)) + for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do + if [ "$autoconfigmethod" = "dhcp" ]; then + /usr/libexec/nm-initrd-generator ip=$NICGUESS:dhcp + else + v4addr=$(grep ^ipv4_address: $tcfg) + v4addr=${v4addr#ipv4_address: } + v4plen=${v4addr#*/} + v4addr=${v4addr%/*} + v4gw=$(grep ^ipv4_gateway: $tcfg) + v4gw=${v4gw#ipv4_gateway: } + ip addr add dev $NICGUESS $v4addr/$v4plen + if [ "$v4gw" = "null" ]; then + v4gw="" + fi + if [ ! -z "$v4gw" ]; then + ip route add default via $v4gw + fi + v4nm=$(grep ipv4_netmask: $tcfg) + v4nm=${v4nm#ipv4_netmask: } + DETECTED=0 + for dsrv in $deploysrvs; do + if curl --capath /tls/ -s --connect-timeout 3 https://$dsrv/confluent-public/ > /dev/null; then + rm /run/NetworkManager/system-connections/* + /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname:$NICGUESS:none + DETECTED=1 + ifname=$NICGUESS + break + fi + done + if [ ! -z "$v4gw" ]; then + ip route del default via $v4gw + fi + ip addr flush dev $NICGUESS + if [ $DETECTED = 1 ]; then break fi - done - if [ ! -z "$v4gw" ]; then - ip route del default via $v4gw fi - ip addr flush dev $NICGUESS - if [ $DETECTED = 1 ]; then - break - fi - fi + done done for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do ip addr flush dev $NICGUESS From d231326dfbe0cdd70b7d258f2172ccf5d98bfa17 Mon Sep 17 00:00:00 2001 From: Simon Thompson Date: Fri, 21 Jun 2024 18:34:47 +0200 Subject: [PATCH 188/319] add class to run cmd by ssh --- .../confluent/plugins/shell/ssh.py | 112 +++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/plugins/shell/ssh.py b/confluent_server/confluent/plugins/shell/ssh.py index 2a6b65ec..f802f842 100644 --- a/confluent_server/confluent/plugins/shell/ssh.py +++ b/confluent_server/confluent/plugins/shell/ssh.py @@ -43,7 +43,6 @@ if cryptography and cryptography.__version__.split('.') < ['1', '5']: paramiko.transport.Transport._preferred_keys) - class HostKeyHandler(paramiko.client.MissingHostKeyPolicy): def __init__(self, configmanager, node): @@ -112,7 +111,7 @@ class SshShell(conapi.Console): # that would rather not use the nodename as anything but an opaque # identifier self.datacallback = callback - if self.username is not b'': + if self.username != b'': self.logon() else: self.inputmode = 0 @@ -259,6 +258,115 @@ class SshShell(conapi.Console): self.ssh.close() self.datacallback = None + def create(nodes, element, configmanager, inputdata): if len(nodes) == 1: return SshShell(nodes[0], configmanager) + + +class SshConn(): + + def __init__(self, node, config, username=b'', password=b''): + self.node = node + self.ssh = None + self.datacallback = None + self.nodeconfig = config + self.username = username + self.password = password + self.connected = False + self.inputmode = 0 # 0 = username, 1 = password... + + def __del__(self): + if self.connected: + self.close() + + def do_logon(self): + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy( + HostKeyHandler(self.nodeconfig, self.node)) + log.log({'info': f"Connecting to {self.node} by ssh"}) + try: + if self.password: + self.ssh.connect(self.node, username=self.username, + password=self.password, allow_agent=False, + look_for_keys=False) + else: + self.ssh.connect(self.node, username=self.username) + except paramiko.AuthenticationException as e: + self.ssh.close() + self.inputmode = 0 + self.username = b'' + self.password = b'' + log.log({'warn': f"Error connecting to {self.node}: {str(e)}"}) + return + except paramiko.ssh_exception.NoValidConnectionsError as e: + self.ssh.close() + self.inputmode = 0 + self.username = b'' + self.password = b'' + log.log({'warn': f"Error connecting to {self.node}: {str(e)}"}) + return + except cexc.PubkeyInvalid as pi: + self.ssh.close() + self.keyaction = b'' + self.candidatefprint = pi.fingerprint + log.log({'warn': pi.message}) + self.keyattrname = pi.attrname + log.log({'info': f"New fingerprint: {pi.fingerprint}"}) + self.inputmode = -1 + return + except paramiko.SSHException as pi: + self.ssh.close() + self.inputmode = -2 + warn = str(pi) + if warnhostkey: + warn += ' (Older cryptography package on this host only ' \ + 'works with ed25519, check ssh startup on target ' \ + 'and permissions on /etc/ssh/*key)\r\n' + log.log({'warn': warn}) + return + except Exception as e: + self.ssh.close() + self.ssh.close() + self.inputmode = 0 + self.username = b'' + self.password = b'' + log.log({'warn': f"Error connecting to {self.node}: {str(e)}"}) + return + self.inputmode = 2 + self.connected = True + log.log({'info': f"Connected by ssh to {self.node}"}) + + def exec_command(self, cmd, cmdargs): + safecmd = cmd.translate(str.maketrans({"[": r"\]", + "]": r"\]", + "?": r"\?", + "!": r"\!", + "\\": r"\\", + "^": r"\^", + "$": r"\$", + " ": r"\ ", + "*": r"\*"})) + cmds = [safecmd] + for arg in cmdargs: + arg = arg.translate(str.maketrans({"[": r"\]", + "]": r"\]", + "?": r"\?", + "!": r"\!", + "\\": r"\\", + "^": r"\^", + "$": r"\$", + " ": r"\ ", + "*": r"\*"})) + arg = "%s" % (str(arg).replace(r"'", r"'\''"),) + cmds.append(arg) + + runcmd = " ".join(cmds) + stdin, stdout, stderr = self.ssh.exec_command(runcmd) + rcode = stdout.channel.recv_exit_status() + return stdout.readlines(), stderr.readlines() + + def close(self): + if self.ssh is not None: + self.ssh.close() + log.log({'info': f"Disconnected from {self.node}"}) From 166e4599b97fa47e9a6546b084c34f05b943674f Mon Sep 17 00:00:00 2001 From: Simon Thompson Date: Fri, 21 Jun 2024 18:35:10 +0200 Subject: [PATCH 189/319] add enos management plugin --- confluent_server/confluent/core.py | 2 +- .../plugins/hardwaremanagement/enos.py | 347 ++++++++++++++++++ 2 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 confluent_server/confluent/plugins/hardwaremanagement/enos.py diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index a8a4412b..ce792fcb 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -73,7 +73,7 @@ import sys import yaml pluginmap = {} -dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos') +dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol', b'geist', u'geist', b'deltapdu', u'deltapdu', b'eatonpdu', u'eatonpdu', b'affluent', u'affluent', b'cnos', u'cnos', b'enos', u'enos') PluginCollection = plugin.PluginCollection diff --git a/confluent_server/confluent/plugins/hardwaremanagement/enos.py b/confluent_server/confluent/plugins/hardwaremanagement/enos.py new file mode 100644 index 00000000..f568fae2 --- /dev/null +++ b/confluent_server/confluent/plugins/hardwaremanagement/enos.py @@ -0,0 +1,347 @@ + +# Copyright 2019 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +#Noncritical: +# - One or more temperature sensors is in the warning range; +#Critical: +# - One or more temperature sensors is in the failure range; +# - One or more fans are running < 100 RPM; +# - One power supply is off. + +import re +import eventlet +import eventlet.queue as queue +import confluent.exceptions as exc +webclient = eventlet.import_patched("pyghmi.util.webclient") +import confluent.messages as msg +import confluent.util as util +import confluent.plugins.shell.ssh as ssh + + +class SwitchSensor(object): + def __init__(self, name, states=None, units=None, value=None, health=None): + self.name = name + self.value = value + self.states = states + self.health = health + self.units = units + + +def _run_method(method, workers, results, configmanager, nodes, element): + creds = configmanager.get_node_attributes( + nodes, ["switchuser", "switchpass", "secret.hardwaremanagementpassword", + "secret.hardwaremanagementuser"], decrypt=True) + for node in nodes: + workers.add(eventlet.spawn(method, configmanager, creds, + node, results, element)) + + +def enos_login(node, configmanager, creds): + try: + ukey = "switchuser" + upass = "switchpass" + if ukey not in creds and "secret.hardwaremanagementuser" in creds[node]: + ukey = "secret.hardwaremanagementuser" + upass = "secret.hardwaremanagementpassword" + + if ukey not in creds[node]: + raise exc.TargetEndpointBadCredentials("Unable to authenticate - switchuser or secret.hardwaremanagementuser not set") + user = creds[node][ukey]["value"] + if upass not in creds[node]: + passwd = None + else: + passwd = creds[node][upass]["value"] + nssh = ssh.SshConn(node=node, config=configmanager, username=user, password=passwd) + nssh.do_logon() + return nssh + except Exception as e: + raise exc.TargetEndpointBadCredentials(f"Unable to authenticate {e}") + + +def enos_version(ssh): + sshStdout, sshStderr = ssh.exec_command(cmd="show", cmdargs=["version"]) + return sshStdout + + +def update(nodes, element, configmanager, inputdata): + for node in nodes: + yield msg.ConfluentNodeError(node, "Not Implemented") + + +def delete(nodes, element, configmanager, inputdata): + for node in nodes: + yield msg.ConfluentNodeError(node, "Not Implemented") + + +def create(nodes, element, configmanager, inputdata): + for node in nodes: + yield msg.ConfluentNodeError(node, "Not Implemented") + + +def retrieve(nodes, element, configmanager, inputdata): + results = queue.LightQueue() + workers = set([]) + if element == ["power", "state"]: + for node in nodes: + yield msg.PowerState(node=node, state="on") + return + elif element == ["health", "hardware"]: + _run_method(retrieve_health, workers, results, configmanager, nodes, element) + elif element[:3] == ["inventory", "hardware", "all"]: + _run_method(retrieve_inventory, workers, results, configmanager, nodes, element) + elif element[:3] == ["inventory", "firmware", "all"]: + _run_method(retrieve_firmware, workers, results, configmanager, nodes, element) + elif element[:3] == ["sensors", "hardware", "all"]: + _run_method(retrieve_sensors, workers, results, configmanager, nodes, element) + else: + for node in nodes: + yield msg.ConfluentNodeError(node, f"Not Implemented: {element}") + return + currtimeout = 10 + while workers: + try: + datum = results.get(10) + while datum: + if datum: + yield datum + datum = results.get_nowait() + except queue.Empty: + pass + eventlet.sleep(0.001) + for t in list(workers): + if t.dead: + workers.discard(t) + try: + while True: + datum = results.get_nowait() + if datum: + yield datum + except queue.Empty: + pass + + +def retrieve_inventory(configmanager, creds, node, results, element): + if len(element) == 3: + results.put(msg.ChildCollection("all")) + results.put(msg.ChildCollection("system")) + return + + switch = gather_data(configmanager, creds, node) + invinfo = switch["inventory"] + + for fan, data in switch["fans"].items(): + invinfo["inventory"][0]["information"][f"Fan #{fan}"] = data["state"] + + for psu, data in switch["psus"].items(): + invinfo["inventory"][0]["information"][f"PSU #{psu}"] = data["state"] + + results.put(msg.KeyValueData(invinfo, node)) + + +def gather_data(configmanager, creds, node): + nssh = enos_login(node=node, configmanager=configmanager, creds=creds) + switch_lines = enos_version(ssh=nssh) + switch_data = {} + sysinfo = {"Product name": {"regex": ".*RackSwitch (\w+)"}, + "Serial Number": {"regex": "ESN\s*\w*\s*: ([\w-]+)"}, + "Board Serial Number": {"regex": "Switch Serial No: (\w+)"}, + "Model": {"regex": "MTM\s*\w*\s*: ([\w-]+)"}, + "FRU Number": {"regex": "Hardware Part\s*\w*\s*: (\w+)"}, + "Airflow": {"regex": "System Fan Airflow\s*\w*\s*: ([\w-]+)"}, + } + + invinfo = { + "inventory": [{ + "name": "System", + "present": True, + "information": { + "Manufacturer": "Lenovo", + } + }] + } + + switch_data["sensors"] = [] + + switch_data["fans"] = gather_fans(switch_lines) + for fan, data in switch_data["fans"].items(): + if "rpm" in data: + health = "ok" + if int(data["rpm"]) < 100: + health = "critical" + switch_data["sensors"].append(SwitchSensor(name=f"Fan {fan}", value=data['rpm'], + units="RPM", health=health)) + + switch_data["psus"] = gather_psus(switch_lines) + + # Hunt for the temp limits + phylimit = {"warn": None, "shut": None} + templimit = {"warn": None, "shut": None} + for line in switch_lines: + match = re.match(r"([\w\s]+)Warning[\w\s]+\s(\d+)[\sA-Za-z\/]+\s(\d+)[\s\w\/]+\s(\d*)", line) + if match: + if "System" in match.group(1): + templimit["warn"] = int(match.group(2)) + templimit["shut"] = int(match.group(3)) + elif "PHYs" in match.group(1): + phylimit["warn"] = int(match.group(2)) + phylimit["shut"] = int(match.group(3)) + if not phylimit["warn"]: + phylimit = templimit + + for line in switch_lines: + # match the inventory data + for key in sysinfo.keys(): + match = re.match(re.compile(sysinfo[key]["regex"]), line) + if match: + invinfo["inventory"][0]["information"][key] = match.group(1).strip() + + # match temp sensors logging where failed + match = re.match(r"Temperature\s+([\d\s\w]+)\s*:\s*(\d+)+\s+([CF])+", line) + if match: + health = "ok" + temp = int(match.group(2)) + name = f"{match.group(1).strip()} Temp" + if "Phy" in name: + if temp > phylimit["warn"]: + health = "warning" + if temp > phylimit["shut"]: + health = "critical" + else: + if temp > templimit["warn"]: + health = "warning" + if temp > templimit["shut"]: + health = "critical" + switch_data["sensors"].append(SwitchSensor(name=name, + value=temp, units=f"°{match.group(3)}", health=health)) + match = re.match(r"\s*(\w+) Faults\s*:\s+(.+)", line) + if match and match.group(2) not in ["()", "None"]: + switch_data["sensors"].append(SwitchSensor(name=f"{match.group(1)} Fault", + value=match.group(2).strip(), units="", health="critical")) + + switch_data["inventory"] = invinfo + + sysfw = {"Software Version": "Unknown", "Boot kernel": "Unknown"} + for line in switch_lines: + for key in sysfw.keys(): + regex = f"{key}\s*\w*\s* ([0-9.]+)" + match = re.match(re.compile(regex), line) + if match: + sysfw[key] = match.group(1) + switch_data["firmware"] = sysfw + + return switch_data + + +def gather_psus(data): + psus = {} + for line in data: + # some switches are: + # Power Supply 1: Back-To-Front + # others are: + # Internal Power Supply: On + if "Power Supply" in line: + match = re.match(re.compile(f"Power Supply (\d)+.*"), line) + if match: + psu = match.group(1) + if psu not in psus: + psus[psu] = {} + m = re.match(r".+\s+(\w+\-\w+\-\w+)\s*\[*.*$", line) + if m: + psus[psu]["airflow"] = m.group(1) + psus[psu]["state"] = "Present" + else: + psus[psu]["state"] = "Not installed" + else: + for psu in range(1, 10): + if "Power Supply" in line and psu not in psus: + if psu not in psus: + psus[psu] = {} + if "Not Installed" in line: + psus[psu]["state"] = "Not installed" + break + else: + psus[psu]["state"] = "Present" + break + return psus + + +def gather_fans(data): + fans = {} + for line in data: + # look for presence of fans + if "Fan" in line: + match = re.match(re.compile(f"Fan (\d)+.*"), line) + if match: + fan = match.group(1) + if match: + if fan not in fans: + fans[fan] = {} + if "rpm" in line or "RPM" in line: + if "Module" in line: + m = re.search(r"Module\s+(\d)+:", line) + if m: + fans[fan]["Module"] = m.group(1) + fans[fan]["state"] = "Present" + m = re.search(r"(\d+)\s*:\s+(RPM=)*(\d+)(rpm)*", line) + if m: + fans[fan]["rpm"] = m.group(3) + + m = re.search(r"\s+(PWM=)*(\d+)(%|pwm)+", line) + if m: + fans[fan]["pwm"] = m.group(2) + + m = re.search(r"(.+)\s+(\w+\-\w+\-\w+)$", line) + if m: + fans[fan]["airflow"] = m.group(1) + else: + fans[fan]["state"] = "Not installed" + return fans + + +def retrieve_firmware(configmanager, creds, node, results, element): + if len(element) == 3: + results.put(msg.ChildCollection("all")) + return + sysinfo = gather_data(configmanager, creds, node)["firmware"] + items = [{ + "Software": {"version": sysinfo["Software Version"]}, + }, + { + "Boot kernel": {"version": sysinfo["Boot kernel"]}, + }] + results.put(msg.Firmware(items, node)) + + +def retrieve_health(configmanager, creds, node, results, element): + switch = gather_data(configmanager, creds, node) + badreadings = [] + summary = "ok" + sensors = gather_data(configmanager, creds, node)["sensors"] + + for sensor in sensors: + if sensor.health not in ["ok"]: + if sensor.health in ["critical"]: + summary = "critical" + elif summary in ["ok"] and sensor.health in ["warning"]: + summary = "warning" + badreadings.append(sensor) + results.put(msg.HealthSummary(summary, name=node)) + results.put(msg.SensorReadings(badreadings, name=node)) + + +def retrieve_sensors(configmanager, creds, node, results, element): + sensors = gather_data(configmanager, creds, node)["sensors"] + results.put(msg.SensorReadings(sensors, node)) From f2d9c3868ba6ad3287cccd5c9b433d675a9a98f3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 24 Jun 2024 15:56:26 -0400 Subject: [PATCH 190/319] Draft work on MegaRAC out of band discovery --- .../confluent/discovery/protocols/ssdp.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 3c1edc74..b5847965 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -86,6 +86,7 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha 'hwaddr': mac, 'addresses': [peer], } + targurl = None for headline in rsp[1:]: if not headline: continue @@ -105,13 +106,18 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha if not value.endswith('/redfish/v1/'): return elif header == 'LOCATION': - if not value.endswith('/DeviceDescription.json'): + if '/eth' in value and value.endswith('.xml'): + targurl = '/redfish/v1/' + continue # MegaRAC redfish + elif value.endswith('/DeviceDescription.json'): + targurl = '/DeviceDescription.json' + else: return - if handler: - eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer) + if handler and targurl: + eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl) -def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer): - retdata = check_fish(('/DeviceDescription.json', peerdata)) +def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl): + retdata = check_fish((targurl, peerdata)) if retdata: known_peers.add(peer) newmacs.add(mac) @@ -411,6 +417,10 @@ def _find_service(service, target): continue if '/DeviceDescription.json' in peerdata[nid]['urls']: pooltargs.append(('/DeviceDescription.json', peerdata[nid])) + else: + for targurl in peerdata[nid]['urls']: + if '/eth' in targurl and targurl.endswith('.xml'): + pooltargs.append(('/redfish/v1/', peerdata[nid])) # For now, don't interrogate generic redfish bmcs # This is due to a need to deduplicate from some supported SLP # targets (IMM, TSM, others) From 762a8ee73f3820f559124a6c237a05d30cdc8288 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 25 Jun 2024 16:25:19 -0400 Subject: [PATCH 191/319] Correct proxyDHCP buffer use It was possible for proxyDHCP to look past the network designated end of packet. Fix this by consistently using the memoryview that was trimmed to size. --- confluent_server/confluent/discovery/protocols/pxe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 4a39654f..133d8abd 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -315,9 +315,9 @@ def proxydhcp(handler, nodeguess): optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4 except ValueError: continue - hwlen = rq[2] - opts, disco = opts_to_dict(rq, optidx, 3) - disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]]) + hwlen = rqv[2] + opts, disco = opts_to_dict(rqv, optidx, 3) + disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rqv[28:28+hwlen]]) node = None if disco.get('hwaddr', None) in macmap: node = macmap[disco['hwaddr']] From 07005d83ca09784b47903fb44f34d02aca48ec6e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 27 Jun 2024 11:25:33 -0400 Subject: [PATCH 192/319] Add MegaRAC discovery support for recent MegaRAC Create a generic redfish discovery and a MegaRAC specific variant. This should open the door for more generic common base redfish discovery for vaguely compatible implementations. For now, MegaRAC only overrides the default username and password (which is undefined in the redfish spec). Also, have SSDP recognize the variant, and tolerate odd nonsense like SSDP replies coming from all manner of odd port numbers (no way to make a sane firewall rule to capture that odd behavior, but at application level we have a chance). --- confluent_server/confluent/discovery/core.py | 9 +- .../confluent/discovery/handlers/megarac.py | 51 ++++ .../discovery/handlers/redfishbmc.py | 269 ++++++++++++++++++ .../confluent/discovery/protocols/ssdp.py | 52 +++- 4 files changed, 366 insertions(+), 15 deletions(-) create mode 100644 confluent_server/confluent/discovery/handlers/megarac.py create mode 100644 confluent_server/confluent/discovery/handlers/redfishbmc.py diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index dfb50b9f..7b94154a 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -74,6 +74,7 @@ import confluent.discovery.handlers.tsm as tsm import confluent.discovery.handlers.pxe as pxeh import confluent.discovery.handlers.smm as smm import confluent.discovery.handlers.xcc as xcc +import confluent.discovery.handlers.megarac as megarac import confluent.exceptions as exc import confluent.log as log import confluent.messages as msg @@ -113,6 +114,7 @@ nodehandlers = { 'service:lenovo-smm': smm, 'service:lenovo-smm2': smm, 'lenovo-xcc': xcc, + 'megarac-bmc': megarac, 'service:management-hardware.IBM:integrated-management-module2': imm, 'pxe-client': pxeh, 'onie-switch': None, @@ -132,6 +134,7 @@ servicenames = { 'service:lenovo-smm2': 'lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', + 'megarac-bmc': 'megarac-bmc', #'openbmc': 'openbmc', 'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2', 'service:io-device.Lenovo:management-module': 'lenovo-switch', @@ -147,6 +150,7 @@ servicebyname = { 'lenovo-smm2': 'service:lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', + 'megarac-bmc': 'megarac-bmc', 'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2', 'lenovo-switch': 'service:io-device.Lenovo:management-module', 'thinkagile-storage': 'service:thinkagile-storagebmc', @@ -453,7 +457,7 @@ def iterate_addrs(addrs, countonly=False): yield 1 return yield addrs - + def _parameterize_path(pathcomponents): listrequested = False childcoll = True @@ -542,7 +546,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents): if len(pathcomponents) > 2: raise Exception('TODO') currsubs = get_subscriptions() - return [msg.ChildCollection(x) for x in currsubs] + return [msg.ChildCollection(x) for x in currsubs] elif operation == 'retrieve': return handle_read_api_request(pathcomponents) elif (operation in ('update', 'create') and @@ -1703,3 +1707,4 @@ if __name__ == '__main__': start_detection() while True: eventlet.sleep(30) + diff --git a/confluent_server/confluent/discovery/handlers/megarac.py b/confluent_server/confluent/discovery/handlers/megarac.py new file mode 100644 index 00000000..d7d8786a --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/megarac.py @@ -0,0 +1,51 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.redfishbmc as redfishbmc +import eventlet.support.greendns + + +getaddrinfo = eventlet.support.greendns.getaddrinfo + + +class NodeHandler(redfishbmc.NodeHandler): + + def get_firmware_default_account_info(self): + return ('admin', 'admin') + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]]} + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) + diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py new file mode 100644 index 00000000..eed401de --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/redfishbmc.py @@ -0,0 +1,269 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.generic as generic +import confluent.exceptions as exc +import confluent.netutil as netutil +import confluent.util as util +import eventlet +import eventlet.support.greendns +import json +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode + +getaddrinfo = eventlet.support.greendns.getaddrinfo + +webclient = eventlet.import_patched('pyghmi.util.webclient') + +def get_host_interface_urls(wc, mginfo): + returls = [] + hifurl = mginfo.get('HostInterfaces', {}).get('@odata.id', None) + if not hifurl: + return None + hifinfo = wc.grab_json_response(hifurl) + hifurls = hifinfo.get('Members', []) + for hifurl in hifurls: + hifurl = hifurl['@odata.id'] + hifinfo = wc.grab_json_response(hifurl) + acturl = hifinfo.get('ManagerEthernetInterface', {}).get('@odata.id', None) + if acturl: + returls.append(acturl) + return returls + + +class NodeHandler(generic.NodeHandler): + devname = 'BMC' + + def __init__(self, info, configmanager): + self.trieddefault = None + self.targuser = None + self.curruser = None + self.currpass = None + self.targpass = None + self.nodename = None + self.csrftok = None + self.channel = None + self.atdefault = True + super(NodeHandler, self).__init__(info, configmanager) + + def get_firmware_default_account_info(self): + raise Exception('This must be subclassed') + + def scan(self): + c = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) + i = c.grab_json_response('/redfish/v1/') + uuid = i.get('UUID', None) + if uuid: + self.info['uuid'] = uuid.lower() + + def validate_cert(self, certificate): + # broadly speaking, merely checks consistency moment to moment, + # but if https_cert gets stricter, this check means something + fprint = util.get_fingerprint(self.https_cert) + return util.cert_matches(fprint, certificate) + + def _get_wc(self): + defuser, defpass = self.get_firmware_default_account_info() + wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) + wc.set_basic_credentials(defuser, defpass) + wc.set_header('Content-Type', 'application/json') + authmode = 0 + if not self.trieddefault: + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status == 403: + self.trieddefault = True + chgurl = None + rsp = json.loads(rsp) + currerr = rsp.get('error', {}) + ecode = currerr.get('code', None) + if ecode.endswith('PasswordChangeRequired'): + for einfo in currerr.get('@Message.ExtendedInfo', []): + if einfo.get('MessageId', None).endswith('PasswordChangeRequired'): + for msgarg in einfo.get('MessageArgs'): + chgurl = msgarg + break + if chgurl: + if self.targpass == defpass: + raise Exception("Must specify a non-default password to onboard this BMC") + wc.set_header('If-Match', '*') + cpr = wc.grab_json_response_with_status(chgurl, {'Password': self.targpass}, method='PATCH') + if cpr[1] >= 200 and cpr[1] < 300: + self.curruser = defuser + self.currpass = self.targpass + wc.set_basic_credentials(self.curruser, self.currpass) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + tries = 10 + while status >= 300 and tries: + eventlet.sleep(1) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + return wc + + if status > 400: + self.trieddefault = True + if status == 401: + wc.set_basic_credentials(self.DEFAULT_USER, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status == 200: # Default user still, but targpass + self.currpass = self.targpass + self.curruser = defuser + return wc + elif self.targuser != defuser: + wc.set_basic_credentials(self.targuser, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + raise Exception("Target BMC does not recognize firmware default credentials nor the confluent stored credential") + else: + self.curruser = defuser + self.currpass = defpass + return wc + if self.curruser: + wc.set_basic_credentials(self.curruser, self.currpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + return None + return wc + wc.set_basic_credentials(self.targuser, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + return None + self.curruser = self.targuser + self.currpass = self.targpass + return wc + + def config(self, nodename): + self.nodename = nodename + creds = self.configmanager.get_node_attributes( + nodename, ['secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword', + 'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'], + True) + cd = creds.get(nodename, {}) + defuser, defpass = self.get_firmware_default_account_info() + user, passwd, _ = self.get_node_credentials( + nodename, creds, defuser, defpass) + user = util.stringify(user) + passwd = util.stringify(passwd) + self.targuser = user + self.targpass = passwd + wc = self._get_wc() + srvroot, status = wc.grab_json_response_with_status('/redfish/v1/') + curruserinfo = {} + authupdate = {} + wc.set_header('Content-Type', 'application/json') + if user != self.curruser: + authupdate['UserName'] = user + if passwd != self.currpass: + authupdate['Password'] = passwd + if authupdate: + targaccturl = None + asrv = srvroot.get('AccountService', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(asrv) + accts = rsp.get('Accounts', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(accts) + accts = rsp.get('Members', []) + for accturl in accts: + accturl = accturl.get('@odata.id', '') + if accturl: + rsp, status = wc.grab_json_response_with_status(accturl) + if rsp.get('UserName', None) == self.curruser: + targaccturl = accturl + break + else: + raise Exception("Unable to identify Account URL to modify on this BMC") + rsp, status = wc.grab_json_response_with_status(targaccturl, authupdate, method='PATCH') + if status >= 300: + raise Exception("Failed attempting to update credentials on BMC") + wc.set_basic_credentials(user, passwd) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + tries = 10 + while tries and status >= 300: + tries -= 1 + eventlet.sleep(1.0) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if ('hardwaremanagement.manager' in cd and + cd['hardwaremanagement.manager']['value'] and + not cd['hardwaremanagement.manager']['value'].startswith( + 'fe80::')): + newip = cd['hardwaremanagement.manager']['value'] + newip = newip.split('/', 1)[0] + newipinfo = getaddrinfo(newip, 0)[0] + newip = newipinfo[-1][0] + if ':' in newip: + raise exc.NotImplementedException('IPv6 remote config TODO') + mgrs = srvroot['Managers']['@odata.id'] + rsp = wc.grab_json_response(mgrs) + if len(rsp['Members']) != 1: + raise Exception("Can not handle multiple Managers") + mgrurl = rsp['Members'][0]['@odata.id'] + mginfo = wc.grab_json_response(mgrurl) + hifurls = get_host_interface_urls(wc, mginfo) + mgtnicinfo = mginfo['EthernetInterfaces']['@odata.id'] + mgtnicinfo = wc.grab_json_response(mgtnicinfo) + mgtnics = [x['@odata.id'] for x in mgtnicinfo.get('Members', [])] + actualnics = [] + for candnic in mgtnics: + if candnic in hifurls: + continue + actualnics.append(candnic) + if len(actualnics) != 1: + raise Exception("Multi-interface BMCs are not supported currently") + currnet = wc.grab_json_response(actualnics[0]) + netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip) + newconfig = { + "Address": newip, + "SubnetMask": netutil.cidr_to_mask(netconfig['prefix']), + } + newgw = netconfig['ipv4_gateway'] + if newgw: + newconfig['Gateway'] = newgw + else: + newconfig['Gateway'] = newip # required property, set to self just to have a value + for net in currnet.get("IPv4Addresses", []): + if net["Address"] == newip and net["SubnetMask"] == newconfig['SubnetMask'] and (not newgw or newconfig['Gateway'] == newgw): + break + else: + wc.set_header('If-Match', '*') + rsp, status = wc.grab_json_response_with_status(actualnics[0], {'IPv4StaticAddresses': [newconfig]}, method='PATCH') + elif self.ipaddr.startswith('fe80::'): + self.configmanager.set_node_attributes( + {nodename: {'hardwaremanagement.manager': self.ipaddr}}) + else: + raise exc.TargetEndpointUnreachable( + 'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)') + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]] } + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 3c1edc74..ec8275f1 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -60,6 +60,7 @@ def active_scan(handler, protocol=None): known_peers = set([]) for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::service:affluent']): for addr in scanned['addresses']: + addr = addr[0:1] + addr[2:] if addr in known_peers: break hwaddr = neighutil.get_hwaddr(addr[0]) @@ -79,13 +80,20 @@ def scan(services, target=None): def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler): - if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']: - peerbymacaddress[mac]['addresses'].append(peer) + if mac in peerbymacaddress: + normpeer = peer[0:1] + peer[2:] + for currpeer in peerbymacaddress[mac]['addresses']: + currnormpeer = currpeer[0:1] + peer[2:] + if currnormpeer == normpeer: + break + else: + peerbymacaddress[mac]['addresses'].append(peer) else: peerdata = { 'hwaddr': mac, 'addresses': [peer], } + targurl = None for headline in rsp[1:]: if not headline: continue @@ -105,13 +113,20 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha if not value.endswith('/redfish/v1/'): return elif header == 'LOCATION': - if not value.endswith('/DeviceDescription.json'): + if '/eth' in value and value.endswith('.xml'): + targurl = '/redfish/v1/' + targtype = 'megarac-bmc' + continue # MegaRAC redfish + elif value.endswith('/DeviceDescription.json'): + targurl = '/DeviceDescription.json' + targtype = 'megarac-bmc' + else: return - if handler: - eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer) + if handler and targurl: + eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype) -def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer): - retdata = check_fish(('/DeviceDescription.json', peerdata)) +def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype): + retdata = check_fish((targurl, peerdata, targtype)) if retdata: known_peers.add(peer) newmacs.add(mac) @@ -322,7 +337,7 @@ def _find_service(service, target): host = '[{0}]'.format(host) msg = smsg.format(host, service) if not isinstance(msg, bytes): - msg = msg.encode('utf8') + msg = msg.encode('utf8') net6.sendto(msg, addr[4]) else: net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) @@ -410,7 +425,11 @@ def _find_service(service, target): if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()): continue if '/DeviceDescription.json' in peerdata[nid]['urls']: - pooltargs.append(('/DeviceDescription.json', peerdata[nid])) + pooltargs.append(('/DeviceDescription.json', peerdata[nid], 'lenovo-xcc')) + else: + for targurl in peerdata[nid]['urls']: + if '/eth' in targurl and targurl.endswith('.xml'): + pooltargs.append(('/redfish/v1/', peerdata[nid], 'megarac-bmc')) # For now, don't interrogate generic redfish bmcs # This is due to a need to deduplicate from some supported SLP # targets (IMM, TSM, others) @@ -425,7 +444,7 @@ def _find_service(service, target): def check_fish(urldata, port=443, verifycallback=None): if not verifycallback: verifycallback = lambda x: True - url, data = urldata + url, data, targtype = urldata try: wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5) peerinfo = wc.grab_json_response(url) @@ -447,7 +466,7 @@ def check_fish(urldata, port=443, verifycallback=None): peerinfo = wc.grab_json_response('/redfish/v1/') if url == '/redfish/v1/': if 'UUID' in peerinfo: - data['services'] = ['service:redfish-bmc'] + data['services'] = [targtype] data['uuid'] = peerinfo['UUID'].lower() return data return None @@ -466,7 +485,12 @@ def _parse_ssdp(peer, rsp, peerdata): if code == b'200': if nid in peerdata: peerdatum = peerdata[nid] - if peer not in peerdatum['addresses']: + normpeer = peer[0:1] + peer[2:] + for currpeer in peerdatum['addresses']: + currnormpeer = currpeer[0:1] + peer[2:] + if currnormpeer == normpeer: + break + else: peerdatum['addresses'].append(peer) else: peerdatum = { @@ -501,5 +525,7 @@ def _parse_ssdp(peer, rsp, peerdata): if __name__ == '__main__': def printit(rsp): - print(repr(rsp)) + pass # print(repr(rsp)) active_scan(printit) + + From 9d979256eb2c8f96e6a2c334beb57a504eb30f02 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 27 Jun 2024 11:36:41 -0400 Subject: [PATCH 193/319] Revert "Add MegaRAC discovery support for recent MegaRAC" This reverts commit 07005d83ca09784b47903fb44f34d02aca48ec6e. Premature addition to master branch --- confluent_server/confluent/discovery/core.py | 9 +- .../confluent/discovery/handlers/megarac.py | 51 ---- .../discovery/handlers/redfishbmc.py | 269 ------------------ .../confluent/discovery/protocols/ssdp.py | 52 +--- 4 files changed, 15 insertions(+), 366 deletions(-) delete mode 100644 confluent_server/confluent/discovery/handlers/megarac.py delete mode 100644 confluent_server/confluent/discovery/handlers/redfishbmc.py diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index 7b94154a..dfb50b9f 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -74,7 +74,6 @@ import confluent.discovery.handlers.tsm as tsm import confluent.discovery.handlers.pxe as pxeh import confluent.discovery.handlers.smm as smm import confluent.discovery.handlers.xcc as xcc -import confluent.discovery.handlers.megarac as megarac import confluent.exceptions as exc import confluent.log as log import confluent.messages as msg @@ -114,7 +113,6 @@ nodehandlers = { 'service:lenovo-smm': smm, 'service:lenovo-smm2': smm, 'lenovo-xcc': xcc, - 'megarac-bmc': megarac, 'service:management-hardware.IBM:integrated-management-module2': imm, 'pxe-client': pxeh, 'onie-switch': None, @@ -134,7 +132,6 @@ servicenames = { 'service:lenovo-smm2': 'lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', - 'megarac-bmc': 'megarac-bmc', #'openbmc': 'openbmc', 'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2', 'service:io-device.Lenovo:management-module': 'lenovo-switch', @@ -150,7 +147,6 @@ servicebyname = { 'lenovo-smm2': 'service:lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', - 'megarac-bmc': 'megarac-bmc', 'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2', 'lenovo-switch': 'service:io-device.Lenovo:management-module', 'thinkagile-storage': 'service:thinkagile-storagebmc', @@ -457,7 +453,7 @@ def iterate_addrs(addrs, countonly=False): yield 1 return yield addrs - + def _parameterize_path(pathcomponents): listrequested = False childcoll = True @@ -546,7 +542,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents): if len(pathcomponents) > 2: raise Exception('TODO') currsubs = get_subscriptions() - return [msg.ChildCollection(x) for x in currsubs] + return [msg.ChildCollection(x) for x in currsubs] elif operation == 'retrieve': return handle_read_api_request(pathcomponents) elif (operation in ('update', 'create') and @@ -1707,4 +1703,3 @@ if __name__ == '__main__': start_detection() while True: eventlet.sleep(30) - diff --git a/confluent_server/confluent/discovery/handlers/megarac.py b/confluent_server/confluent/discovery/handlers/megarac.py deleted file mode 100644 index d7d8786a..00000000 --- a/confluent_server/confluent/discovery/handlers/megarac.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2024 Lenovo -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import confluent.discovery.handlers.redfishbmc as redfishbmc -import eventlet.support.greendns - - -getaddrinfo = eventlet.support.greendns.getaddrinfo - - -class NodeHandler(redfishbmc.NodeHandler): - - def get_firmware_default_account_info(self): - return ('admin', 'admin') - - -def remote_nodecfg(nodename, cfm): - cfg = cfm.get_node_attributes( - nodename, 'hardwaremanagement.manager') - ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( - 'value', None) - ipaddr = ipaddr.split('/', 1)[0] - ipaddr = getaddrinfo(ipaddr, 0)[0][-1] - if not ipaddr: - raise Exception('Cannot remote configure a system without known ' - 'address') - info = {'addresses': [ipaddr]} - nh = NodeHandler(info, cfm) - nh.config(nodename) - - -if __name__ == '__main__': - import confluent.config.configmanager as cfm - c = cfm.ConfigManager(None) - import sys - info = {'addresses': [[sys.argv[1]]]} - print(repr(info)) - testr = NodeHandler(info, c) - testr.config(sys.argv[2]) - diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py deleted file mode 100644 index eed401de..00000000 --- a/confluent_server/confluent/discovery/handlers/redfishbmc.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2024 Lenovo -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import confluent.discovery.handlers.generic as generic -import confluent.exceptions as exc -import confluent.netutil as netutil -import confluent.util as util -import eventlet -import eventlet.support.greendns -import json -try: - from urllib import urlencode -except ImportError: - from urllib.parse import urlencode - -getaddrinfo = eventlet.support.greendns.getaddrinfo - -webclient = eventlet.import_patched('pyghmi.util.webclient') - -def get_host_interface_urls(wc, mginfo): - returls = [] - hifurl = mginfo.get('HostInterfaces', {}).get('@odata.id', None) - if not hifurl: - return None - hifinfo = wc.grab_json_response(hifurl) - hifurls = hifinfo.get('Members', []) - for hifurl in hifurls: - hifurl = hifurl['@odata.id'] - hifinfo = wc.grab_json_response(hifurl) - acturl = hifinfo.get('ManagerEthernetInterface', {}).get('@odata.id', None) - if acturl: - returls.append(acturl) - return returls - - -class NodeHandler(generic.NodeHandler): - devname = 'BMC' - - def __init__(self, info, configmanager): - self.trieddefault = None - self.targuser = None - self.curruser = None - self.currpass = None - self.targpass = None - self.nodename = None - self.csrftok = None - self.channel = None - self.atdefault = True - super(NodeHandler, self).__init__(info, configmanager) - - def get_firmware_default_account_info(self): - raise Exception('This must be subclassed') - - def scan(self): - c = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) - i = c.grab_json_response('/redfish/v1/') - uuid = i.get('UUID', None) - if uuid: - self.info['uuid'] = uuid.lower() - - def validate_cert(self, certificate): - # broadly speaking, merely checks consistency moment to moment, - # but if https_cert gets stricter, this check means something - fprint = util.get_fingerprint(self.https_cert) - return util.cert_matches(fprint, certificate) - - def _get_wc(self): - defuser, defpass = self.get_firmware_default_account_info() - wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) - wc.set_basic_credentials(defuser, defpass) - wc.set_header('Content-Type', 'application/json') - authmode = 0 - if not self.trieddefault: - rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - if status == 403: - self.trieddefault = True - chgurl = None - rsp = json.loads(rsp) - currerr = rsp.get('error', {}) - ecode = currerr.get('code', None) - if ecode.endswith('PasswordChangeRequired'): - for einfo in currerr.get('@Message.ExtendedInfo', []): - if einfo.get('MessageId', None).endswith('PasswordChangeRequired'): - for msgarg in einfo.get('MessageArgs'): - chgurl = msgarg - break - if chgurl: - if self.targpass == defpass: - raise Exception("Must specify a non-default password to onboard this BMC") - wc.set_header('If-Match', '*') - cpr = wc.grab_json_response_with_status(chgurl, {'Password': self.targpass}, method='PATCH') - if cpr[1] >= 200 and cpr[1] < 300: - self.curruser = defuser - self.currpass = self.targpass - wc.set_basic_credentials(self.curruser, self.currpass) - _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - tries = 10 - while status >= 300 and tries: - eventlet.sleep(1) - _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - return wc - - if status > 400: - self.trieddefault = True - if status == 401: - wc.set_basic_credentials(self.DEFAULT_USER, self.targpass) - rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - if status == 200: # Default user still, but targpass - self.currpass = self.targpass - self.curruser = defuser - return wc - elif self.targuser != defuser: - wc.set_basic_credentials(self.targuser, self.targpass) - rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - if status != 200: - raise Exception("Target BMC does not recognize firmware default credentials nor the confluent stored credential") - else: - self.curruser = defuser - self.currpass = defpass - return wc - if self.curruser: - wc.set_basic_credentials(self.curruser, self.currpass) - rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - if status != 200: - return None - return wc - wc.set_basic_credentials(self.targuser, self.targpass) - rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - if status != 200: - return None - self.curruser = self.targuser - self.currpass = self.targpass - return wc - - def config(self, nodename): - self.nodename = nodename - creds = self.configmanager.get_node_attributes( - nodename, ['secret.hardwaremanagementuser', - 'secret.hardwaremanagementpassword', - 'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'], - True) - cd = creds.get(nodename, {}) - defuser, defpass = self.get_firmware_default_account_info() - user, passwd, _ = self.get_node_credentials( - nodename, creds, defuser, defpass) - user = util.stringify(user) - passwd = util.stringify(passwd) - self.targuser = user - self.targpass = passwd - wc = self._get_wc() - srvroot, status = wc.grab_json_response_with_status('/redfish/v1/') - curruserinfo = {} - authupdate = {} - wc.set_header('Content-Type', 'application/json') - if user != self.curruser: - authupdate['UserName'] = user - if passwd != self.currpass: - authupdate['Password'] = passwd - if authupdate: - targaccturl = None - asrv = srvroot.get('AccountService', {}).get('@odata.id') - rsp, status = wc.grab_json_response_with_status(asrv) - accts = rsp.get('Accounts', {}).get('@odata.id') - rsp, status = wc.grab_json_response_with_status(accts) - accts = rsp.get('Members', []) - for accturl in accts: - accturl = accturl.get('@odata.id', '') - if accturl: - rsp, status = wc.grab_json_response_with_status(accturl) - if rsp.get('UserName', None) == self.curruser: - targaccturl = accturl - break - else: - raise Exception("Unable to identify Account URL to modify on this BMC") - rsp, status = wc.grab_json_response_with_status(targaccturl, authupdate, method='PATCH') - if status >= 300: - raise Exception("Failed attempting to update credentials on BMC") - wc.set_basic_credentials(user, passwd) - _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - tries = 10 - while tries and status >= 300: - tries -= 1 - eventlet.sleep(1.0) - _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') - if ('hardwaremanagement.manager' in cd and - cd['hardwaremanagement.manager']['value'] and - not cd['hardwaremanagement.manager']['value'].startswith( - 'fe80::')): - newip = cd['hardwaremanagement.manager']['value'] - newip = newip.split('/', 1)[0] - newipinfo = getaddrinfo(newip, 0)[0] - newip = newipinfo[-1][0] - if ':' in newip: - raise exc.NotImplementedException('IPv6 remote config TODO') - mgrs = srvroot['Managers']['@odata.id'] - rsp = wc.grab_json_response(mgrs) - if len(rsp['Members']) != 1: - raise Exception("Can not handle multiple Managers") - mgrurl = rsp['Members'][0]['@odata.id'] - mginfo = wc.grab_json_response(mgrurl) - hifurls = get_host_interface_urls(wc, mginfo) - mgtnicinfo = mginfo['EthernetInterfaces']['@odata.id'] - mgtnicinfo = wc.grab_json_response(mgtnicinfo) - mgtnics = [x['@odata.id'] for x in mgtnicinfo.get('Members', [])] - actualnics = [] - for candnic in mgtnics: - if candnic in hifurls: - continue - actualnics.append(candnic) - if len(actualnics) != 1: - raise Exception("Multi-interface BMCs are not supported currently") - currnet = wc.grab_json_response(actualnics[0]) - netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip) - newconfig = { - "Address": newip, - "SubnetMask": netutil.cidr_to_mask(netconfig['prefix']), - } - newgw = netconfig['ipv4_gateway'] - if newgw: - newconfig['Gateway'] = newgw - else: - newconfig['Gateway'] = newip # required property, set to self just to have a value - for net in currnet.get("IPv4Addresses", []): - if net["Address"] == newip and net["SubnetMask"] == newconfig['SubnetMask'] and (not newgw or newconfig['Gateway'] == newgw): - break - else: - wc.set_header('If-Match', '*') - rsp, status = wc.grab_json_response_with_status(actualnics[0], {'IPv4StaticAddresses': [newconfig]}, method='PATCH') - elif self.ipaddr.startswith('fe80::'): - self.configmanager.set_node_attributes( - {nodename: {'hardwaremanagement.manager': self.ipaddr}}) - else: - raise exc.TargetEndpointUnreachable( - 'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)') - - -def remote_nodecfg(nodename, cfm): - cfg = cfm.get_node_attributes( - nodename, 'hardwaremanagement.manager') - ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( - 'value', None) - ipaddr = ipaddr.split('/', 1)[0] - ipaddr = getaddrinfo(ipaddr, 0)[0][-1] - if not ipaddr: - raise Exception('Cannot remote configure a system without known ' - 'address') - info = {'addresses': [ipaddr]} - nh = NodeHandler(info, cfm) - nh.config(nodename) - -if __name__ == '__main__': - import confluent.config.configmanager as cfm - c = cfm.ConfigManager(None) - import sys - info = {'addresses': [[sys.argv[1]]] } - print(repr(info)) - testr = NodeHandler(info, c) - testr.config(sys.argv[2]) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index ec8275f1..3c1edc74 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -60,7 +60,6 @@ def active_scan(handler, protocol=None): known_peers = set([]) for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::service:affluent']): for addr in scanned['addresses']: - addr = addr[0:1] + addr[2:] if addr in known_peers: break hwaddr = neighutil.get_hwaddr(addr[0]) @@ -80,20 +79,13 @@ def scan(services, target=None): def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler): - if mac in peerbymacaddress: - normpeer = peer[0:1] + peer[2:] - for currpeer in peerbymacaddress[mac]['addresses']: - currnormpeer = currpeer[0:1] + peer[2:] - if currnormpeer == normpeer: - break - else: - peerbymacaddress[mac]['addresses'].append(peer) + if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']: + peerbymacaddress[mac]['addresses'].append(peer) else: peerdata = { 'hwaddr': mac, 'addresses': [peer], } - targurl = None for headline in rsp[1:]: if not headline: continue @@ -113,20 +105,13 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha if not value.endswith('/redfish/v1/'): return elif header == 'LOCATION': - if '/eth' in value and value.endswith('.xml'): - targurl = '/redfish/v1/' - targtype = 'megarac-bmc' - continue # MegaRAC redfish - elif value.endswith('/DeviceDescription.json'): - targurl = '/DeviceDescription.json' - targtype = 'megarac-bmc' - else: + if not value.endswith('/DeviceDescription.json'): return - if handler and targurl: - eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype) + if handler: + eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer) -def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype): - retdata = check_fish((targurl, peerdata, targtype)) +def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer): + retdata = check_fish(('/DeviceDescription.json', peerdata)) if retdata: known_peers.add(peer) newmacs.add(mac) @@ -337,7 +322,7 @@ def _find_service(service, target): host = '[{0}]'.format(host) msg = smsg.format(host, service) if not isinstance(msg, bytes): - msg = msg.encode('utf8') + msg = msg.encode('utf8') net6.sendto(msg, addr[4]) else: net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) @@ -425,11 +410,7 @@ def _find_service(service, target): if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()): continue if '/DeviceDescription.json' in peerdata[nid]['urls']: - pooltargs.append(('/DeviceDescription.json', peerdata[nid], 'lenovo-xcc')) - else: - for targurl in peerdata[nid]['urls']: - if '/eth' in targurl and targurl.endswith('.xml'): - pooltargs.append(('/redfish/v1/', peerdata[nid], 'megarac-bmc')) + pooltargs.append(('/DeviceDescription.json', peerdata[nid])) # For now, don't interrogate generic redfish bmcs # This is due to a need to deduplicate from some supported SLP # targets (IMM, TSM, others) @@ -444,7 +425,7 @@ def _find_service(service, target): def check_fish(urldata, port=443, verifycallback=None): if not verifycallback: verifycallback = lambda x: True - url, data, targtype = urldata + url, data = urldata try: wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5) peerinfo = wc.grab_json_response(url) @@ -466,7 +447,7 @@ def check_fish(urldata, port=443, verifycallback=None): peerinfo = wc.grab_json_response('/redfish/v1/') if url == '/redfish/v1/': if 'UUID' in peerinfo: - data['services'] = [targtype] + data['services'] = ['service:redfish-bmc'] data['uuid'] = peerinfo['UUID'].lower() return data return None @@ -485,12 +466,7 @@ def _parse_ssdp(peer, rsp, peerdata): if code == b'200': if nid in peerdata: peerdatum = peerdata[nid] - normpeer = peer[0:1] + peer[2:] - for currpeer in peerdatum['addresses']: - currnormpeer = currpeer[0:1] + peer[2:] - if currnormpeer == normpeer: - break - else: + if peer not in peerdatum['addresses']: peerdatum['addresses'].append(peer) else: peerdatum = { @@ -525,7 +501,5 @@ def _parse_ssdp(peer, rsp, peerdata): if __name__ == '__main__': def printit(rsp): - pass # print(repr(rsp)) + print(repr(rsp)) active_scan(printit) - - From dde6ceadfbb3a62fe39981393ebbfb297e64f997 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 3 Jul 2024 14:36:54 -0400 Subject: [PATCH 194/319] Allow local ISO to proceed if detected with Ubuntu --- .../ubuntu22.04/initramfs/scripts/init-premount/confluent | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent index 03761f3a..8a7e3777 100755 --- a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent +++ b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent @@ -79,8 +79,12 @@ if [ ! -z "$cons" ]; then fi echo "Preparing to deploy $osprofile from $MGR" echo $osprofile > /custom-installation/confluent/osprofile -echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf -fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso" +. /etc/os-release +DIRECTISO=$(blkid -t TYPE=iso9660 |grep -Ei ' LABEL="Ubuntu-Server '$VERSION_ID) +if [ -z "$DIRECTISO" ]; then + echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf + fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso" +fi if [ ! -z "$cons" ]; then fcmdline="$fcmdline console=${cons#/dev/}" fi From 4f21e627054a0b09dcc2c0bdeea05f0897af04b9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 8 Jul 2024 10:02:03 -0400 Subject: [PATCH 195/319] Fix typo in the auth code --- confluent_server/confluent/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/auth.py b/confluent_server/confluent/auth.py index fd07a133..fb82af24 100644 --- a/confluent_server/confluent/auth.py +++ b/confluent_server/confluent/auth.py @@ -58,7 +58,7 @@ _allowedbyrole = { '/nodes/', '/node*/media/uploads/', '/node*/inventory/firmware/updates/*', - '/node*/suppport/servicedata*', + '/node*/support/servicedata*', '/node*/attributes/expression', '/nodes/*/console/session*', '/nodes/*/shell/sessions*', From 7bd41af2ccca18b82f6ffeb95d7b7c4500b20937 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Jul 2024 08:41:34 -0400 Subject: [PATCH 196/319] More properly error on bad requests Avoid incurring an error code 500 issue in reaction to certain bad request data. --- confluent_server/confluent/httpapi.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index e30df36d..10ab8b86 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -175,6 +175,8 @@ def _get_query_dict(env, reqbody, reqtype): qstring = None if qstring: for qpair in qstring.split('&'): + if '=' not in qpair: + continue qkey, qvalue = qpair.split('=') qdict[qkey] = qvalue if reqbody is not None: @@ -668,7 +670,11 @@ def resourcehandler_backend(env, start_response): if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0: reqbody = env['wsgi.input'].read(int(env['CONTENT_LENGTH'])) reqtype = env['CONTENT_TYPE'] - operation = opmap[env['REQUEST_METHOD']] + operation = opmap.get(env['REQUEST_METHOD'], None) + if not operation: + start_response('400 Bad Method', headers) + yield '' + return querydict = _get_query_dict(env, reqbody, reqtype) if operation != 'retrieve' and 'restexplorerop' in querydict: operation = querydict['restexplorerop'] From bc624d9360c06508abeff9f796e0a36224e2dde1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jul 2024 15:15:56 -0400 Subject: [PATCH 197/319] Fix Ubuntu 24.04 network bring up Ubuntu 24.04 does not check conf files in /run before assuming dhcp anymore. Influence its logic to skip dhcp if we have static for it --- .../ubuntu22.04/initramfs/scripts/casper-bottom/99confluent | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent index e066714e..d629cf32 100755 --- a/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent +++ b/confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent @@ -26,12 +26,14 @@ if [ -e /tmp/cnflnthmackeytmp ]; then chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile https://$MGR/confluent-api/self/registerapikey cp /root/$passfile /root/custom-installation/confluent/confluent.apikey DEVICE=$(cat /tmp/autodetectnic) + IP=done else chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey MGR=[$MGR] nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1) nic=$(ip link |grep ^$nic:|awk '{print $2}') DEVICE=${nic%:} + IP=done fi if [ -z "$MGTIFACE" ]; then chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg From 8c193fe33f29831b3625b217fb0fe55f386f61da Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jul 2024 15:30:47 -0400 Subject: [PATCH 198/319] Fix issues with firstboot on Ubuntu 22+ --- .../ubuntu22.04/profiles/default/scripts/firstboot.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh index c0ba44ab..996bfffe 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh @@ -3,11 +3,11 @@ echo "Confluent first boot is running" HOME=$(getent passwd $(whoami)|cut -d: -f 6) export HOME ( -exec >> /target/var/log/confluent/confluent-firstboot.log -exec 2>> /target/var/log/confluent/confluent-firstboot.log -chmod 600 /target/var/log/confluent/confluent-firstboot.log +exec >> /var/log/confluent/confluent-firstboot.log +exec 2>> /var/log/confluent/confluent-firstboot.log +chmod 600 /var/log/confluent/confluent-firstboot.log cp -a /etc/confluent/ssh/* /etc/ssh/ -systemctl restart sshd +systemctl restart ssh rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}') if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then echo root:$rootpw | chpasswd -e @@ -27,4 +27,4 @@ run_remote_parts firstboot.d run_remote_config firstboot.d curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus ) & -tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console +tail --pid $! -n 0 -F /var/log/confluent/confluent-post.log > /dev/console From 08a5bffa90486596b8fa392d9aba470d0e14e966 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jul 2024 15:52:49 -0400 Subject: [PATCH 199/319] Write multiple grub.cfg paths Some have different requirements on how to find grub.cfg --- confluent_server/confluent/osimage.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index e0c1a8cb..3d5f50de 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -158,7 +158,7 @@ def find_glob(loc, fileglob): for cdir, _, fs in os.walk(loc): for f in fs: if fnmatch(f, fileglob): - return os.path.join(cdir, f) + return [os.path.join(cdir, f)] return None @@ -182,9 +182,13 @@ def update_boot_linux(profiledir, profile, label): # well need to honor grubprefix path if different grubcfgpath = find_glob(profiledir + '/boot', 'grub.cfg') if not grubcfgpath: - grubcfgpath = profiledir + '/boot/efi/boot/grub.cfg' - with open(grubcfgpath, 'w') as grubout: - grubout.write(grubcfg) + grubcfgpath = [ + profiledir + '/boot/efi/boot/grub.cfg' + profiledir + '/boot/boot/grub.cfg' + ] + for grubcfgpth in grubcfgpath: + with open(grubcfgpth, 'w') as grubout: + grubout.write(grubcfg) ipxeargs = kernelargs for initramfs in initrds: ipxeargs += " initrd=" + initramfs From 3aea1ec7d67082ece19f9306594b53ecc8dc52e8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jul 2024 16:21:36 -0400 Subject: [PATCH 200/319] Fix list syntax in grub cfg --- confluent_server/confluent/osimage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 3d5f50de..5297424c 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -183,7 +183,7 @@ def update_boot_linux(profiledir, profile, label): grubcfgpath = find_glob(profiledir + '/boot', 'grub.cfg') if not grubcfgpath: grubcfgpath = [ - profiledir + '/boot/efi/boot/grub.cfg' + profiledir + '/boot/efi/boot/grub.cfg', profiledir + '/boot/boot/grub.cfg' ] for grubcfgpth in grubcfgpath: From c0cc673c63e551745f96380a453d29cf4065f408 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jul 2024 16:31:06 -0400 Subject: [PATCH 201/319] Make directory exist before creating file --- confluent_server/confluent/osimage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 5297424c..5beb08f5 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -187,6 +187,7 @@ def update_boot_linux(profiledir, profile, label): profiledir + '/boot/boot/grub.cfg' ] for grubcfgpth in grubcfgpath: + os.makedirs(os.path.dirname(grubcfgpth), 0o755, exist_ok=True) with open(grubcfgpth, 'w') as grubout: grubout.write(grubcfg) ipxeargs = kernelargs From 7a3e1dfde3ed3e3c486c636c76f60c6fd12c5614 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jul 2024 16:48:46 -0400 Subject: [PATCH 202/319] Fix grub fallback path for more grub --- confluent_server/confluent/osimage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 5beb08f5..557cc99d 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -184,7 +184,7 @@ def update_boot_linux(profiledir, profile, label): if not grubcfgpath: grubcfgpath = [ profiledir + '/boot/efi/boot/grub.cfg', - profiledir + '/boot/boot/grub.cfg' + profiledir + '/boot/boot/grub/grub.cfg' ] for grubcfgpth in grubcfgpath: os.makedirs(os.path.dirname(grubcfgpth), 0o755, exist_ok=True) From 945dff09f34744c560ad9fed8b4a1d5a8a99c178 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 15 Jul 2024 08:19:13 -0400 Subject: [PATCH 203/319] Change to generic linux/inird command in Grub Modern grub has removed these variants, and should only be required for very old non-EFI stub kernels --- confluent_server/confluent/osimage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 557cc99d..0e3d8a58 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -167,7 +167,7 @@ def update_boot_linux(profiledir, profile, label): kernelargs = profile.get('kernelargs', '') grubcfg = "set timeout=5\nmenuentry '" grubcfg += label - grubcfg += "' {\n linuxefi /kernel " + kernelargs + "\n" + grubcfg += "' {\n linux /kernel " + kernelargs + "\n" initrds = [] for initramfs in glob.glob(profiledir + '/boot/initramfs/*.cpio'): initramfs = os.path.basename(initramfs) @@ -175,7 +175,7 @@ def update_boot_linux(profiledir, profile, label): for initramfs in os.listdir(profiledir + '/boot/initramfs'): if initramfs not in initrds: initrds.append(initramfs) - grubcfg += " initrdefi " + grubcfg += " initrd " for initramfs in initrds: grubcfg += " /initramfs/{0}".format(initramfs) grubcfg += "\n}\n" From 8d726bced97a8f1bb45a3a433fdf22a03b515daf Mon Sep 17 00:00:00 2001 From: tkucherera Date: Mon, 15 Jul 2024 09:22:59 -0400 Subject: [PATCH 204/319] better error handling --- confluent_client/bin/nodebmcpassword | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/confluent_client/bin/nodebmcpassword b/confluent_client/bin/nodebmcpassword index f76b076c..135abb96 100755 --- a/confluent_client/bin/nodebmcpassword +++ b/confluent_client/bin/nodebmcpassword @@ -88,6 +88,7 @@ for rsp in session.read('/noderange/{0}/configuration/management_controller/user for node in databynode: if 'error' in rsp['databynode'][node]: print(node, ':', rsp['databynode'][node]['error']) + errorNodes.add(node) continue for user in rsp['databynode'][node]['users']: if user['username'] == username: @@ -97,6 +98,10 @@ for rsp in session.read('/noderange/{0}/configuration/management_controller/user uid_dict[user['uid']] = uid_dict[user['uid']] + ',{}'.format(node) break +if not uid_dict: + print("Error: Could not reach target node's bmc user") + sys.exit(1) + for uid in uid_dict: success = session.simple_noderange_command(uid_dict[uid], 'configuration/management_controller/users/{0}'.format(uid), new_password, key='password', errnodes=errorNodes) # = 0 if successful From abf12f2b962ca94550e321684228cd028e5f3618 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 15 Jul 2024 11:26:58 -0400 Subject: [PATCH 205/319] Reinstate linuxefi/initrdefi for older GRUB Technically, Grub never had 'linuxefi/initrdefi' commands officially, so this is a bit weird. However, if we see signs of GRUB older than 2.03, we will assume that is requires the linuxefi/initrdefi commands from the out of tree patch to support EFI the old way. This corresponds with EL7. Other variants seem ok with the more proper linux/initrd command names. --- confluent_server/confluent/osimage.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 0e3d8a58..5ff28d16 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -165,9 +165,27 @@ def find_glob(loc, fileglob): def update_boot_linux(profiledir, profile, label): profname = os.path.basename(profiledir) kernelargs = profile.get('kernelargs', '') + needefi = False + for grubexe in glob.glob(profiledir + '/boot/efi/boot/grubx64.efi'): + with open(grubexe, 'rb') as grubin: + grubcontent = grubin.read() + uaidx = grubcontent.find(b'User-Agent: GRUB 2.0') + if uaidx > 0: + grubcontent = grubcontent[uaidx:] + cridx = grubcontent.find(b'\r') + if cridx > 1: + grubcontent = grubcontent[:cridx] + grubver = grubcontent.split(b'~', 1)[0] + grubver = grubver.rsplit(b' ', 1)[-1] + grubver = grubver.split(b'.') + if len(grubver) > 1: + if int(grubver[0]) < 3 and int(grubver[1]) < 3: + needefi = True + lincmd = 'linuxefi' if needefi else 'linux' + initrdcmd = 'initrdefi' if needefi else 'initrd' grubcfg = "set timeout=5\nmenuentry '" grubcfg += label - grubcfg += "' {\n linux /kernel " + kernelargs + "\n" + grubcfg += "' {\n " + lincmd + " /kernel " + kernelargs + "\n" initrds = [] for initramfs in glob.glob(profiledir + '/boot/initramfs/*.cpio'): initramfs = os.path.basename(initramfs) @@ -175,7 +193,7 @@ def update_boot_linux(profiledir, profile, label): for initramfs in os.listdir(profiledir + '/boot/initramfs'): if initramfs not in initrds: initrds.append(initramfs) - grubcfg += " initrd " + grubcfg += " " + initrdcmd + " " for initramfs in initrds: grubcfg += " /initramfs/{0}".format(initramfs) grubcfg += "\n}\n" From 9d5432f8cd22d2bfa5a206b227b119d95f6a473e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 18 Jul 2024 08:40:40 -0400 Subject: [PATCH 206/319] Fix network configuration when middle name ends in 'net' --- confluent_server/confluent/netutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index 9e9fd597..9bac92c2 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -320,7 +320,7 @@ def get_full_net_config(configmanager, node, serverip=None): if val is None: continue if attrib.startswith('net.'): - attrib = attrib.replace('net.', '').rsplit('.', 1) + attrib = attrib.replace('net.', '', 1).rsplit('.', 1) if len(attrib) == 1: iface = None attrib = attrib[0] From b4a33b810230e2c37dce78ca6a1d659de576ec76 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 18 Jul 2024 17:26:49 +0200 Subject: [PATCH 207/319] Fix EL stateful install Sometimes stateful install can fail if vgchange -a n is run after dd. Use wipefs instead and fix order of both commands. Furthermore, use the $INSALLDISK variable. --- confluent_osdeploy/el8/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh index 4d76aaa3..cd831360 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh @@ -115,7 +115,7 @@ grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/insta if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then INSTALLDISK=$(cat /tmp/installdisk) sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e s/%%LUKSHOOK%%/$LUKSPARTY/ /tmp/partitioning.template > /tmp/partitioning - dd if=/dev/zero of=/dev/$(cat /tmp/installdisk) bs=1M count=1 >& /dev/null vgchange -a n >& /dev/null + wipefs -a -f /dev/$INSTALLDISK >& /dev/null fi kill $logshowpid From 294ef8e88c0cd52942bca848092a6529ba05a927 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 19 Jul 2024 09:28:29 -0400 Subject: [PATCH 208/319] Fix for IB diskless boot to install clone The infiniband section must be defined for the OS to use the IB link. If it is missing then networking does not come up during firstboot. Fix this by having an inifiniband section including explicitly declaring use of datagram mode. This should suffice for all install use cases, and may be changed after firstboot starts. --- .../usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh index a4f10ee2..9b885e82 100644 --- a/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh +++ b/confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh @@ -171,6 +171,13 @@ permissions= wait-device-timeout=60000 EOC +if [ "$linktype" = infiniband ]; then +cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC +[infiniband] +transport-mode=datagram + +EOC +fi autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}') auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}') if [ "$autoconfigmethod" = "dhcp" ]; then From ede941c0d91d5bd00229f1b28998097e9881530d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 13:46:27 -0400 Subject: [PATCH 209/319] Add deb packaging of imgutil --- imgutil/builddeb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 imgutil/builddeb diff --git a/imgutil/builddeb b/imgutil/builddeb new file mode 100755 index 00000000..4258fbc4 --- /dev/null +++ b/imgutil/builddeb @@ -0,0 +1,24 @@ +#!/bin/bash +VERSION=`git describe|cut -d- -f 1` +NUMCOMMITS=`git describe|cut -d- -f 2` +if [ "$NUMCOMMITS" != "$VERSION" ]; then + VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` +fi +mkdir -p /tmp/confluent-imgutil +cp -a * /tmp/confluent-imgutil +cp ../LICENSE /tmp/confluent-imgutil +cd /tmp/confluent-imgutil +rm -rf deb/confluent_imgutil_$VERSION/ +mkdir -p deb/confluent_imgutil_$VERSION/DEBIAN/ +mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/lib/imgutil +mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/bin +mv imgutil deb/confluent_imgutil_$VERSION/opt/confluent/bin/ +chmod a+x deb/confluent_imgutil_$VERSION/opt/confluent/bin/imgutil +mv ubuntu* suse15 el7 el9 el8 deb/confluent_imgutil_$VERSION/opt/confluent/lib/imgutil/ +mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/share/licenses/confluent_imgutil +cp LICENSE deb/confluent_imgutil_$VERSION/opt/confluent/share/licenses/confluent_imgutil +sed -e 's/#VERSION#/'$VERSION/ control.tmpl > deb/confluent_imgutil_$VERSION/DEBIAN/control +dpkg-deb --build deb/lenovo_confluent_$VERSION +if [ ! -z "$1" ]; then + mv deb/lenovo-confluent_$VERSION.deb $1 +fi From 36ca68f44dba1bb028fc32a0aa4f17e8c9154a0a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 13:46:45 -0400 Subject: [PATCH 210/319] Add control file for deb build of imgutil --- imgutil/control.tmpl | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 imgutil/control.tmpl diff --git a/imgutil/control.tmpl b/imgutil/control.tmpl new file mode 100644 index 00000000..a0fe21af --- /dev/null +++ b/imgutil/control.tmpl @@ -0,0 +1,8 @@ +Package: confluent-imgutil +Version: #VERSION# +Section: base +Priority: optional +Maintainer: Jarrod Johnson +Description: Web frontend for confluent server +Architecture: all + From 69fa3f10c0f38cf7a6fc52d13ed2c0fd5d2e41f9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 13:46:27 -0400 Subject: [PATCH 211/319] Add deb packaging of imgutil --- imgutil/builddeb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 imgutil/builddeb diff --git a/imgutil/builddeb b/imgutil/builddeb new file mode 100755 index 00000000..4258fbc4 --- /dev/null +++ b/imgutil/builddeb @@ -0,0 +1,24 @@ +#!/bin/bash +VERSION=`git describe|cut -d- -f 1` +NUMCOMMITS=`git describe|cut -d- -f 2` +if [ "$NUMCOMMITS" != "$VERSION" ]; then + VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` +fi +mkdir -p /tmp/confluent-imgutil +cp -a * /tmp/confluent-imgutil +cp ../LICENSE /tmp/confluent-imgutil +cd /tmp/confluent-imgutil +rm -rf deb/confluent_imgutil_$VERSION/ +mkdir -p deb/confluent_imgutil_$VERSION/DEBIAN/ +mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/lib/imgutil +mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/bin +mv imgutil deb/confluent_imgutil_$VERSION/opt/confluent/bin/ +chmod a+x deb/confluent_imgutil_$VERSION/opt/confluent/bin/imgutil +mv ubuntu* suse15 el7 el9 el8 deb/confluent_imgutil_$VERSION/opt/confluent/lib/imgutil/ +mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/share/licenses/confluent_imgutil +cp LICENSE deb/confluent_imgutil_$VERSION/opt/confluent/share/licenses/confluent_imgutil +sed -e 's/#VERSION#/'$VERSION/ control.tmpl > deb/confluent_imgutil_$VERSION/DEBIAN/control +dpkg-deb --build deb/lenovo_confluent_$VERSION +if [ ! -z "$1" ]; then + mv deb/lenovo-confluent_$VERSION.deb $1 +fi From 7154a1d60ca641d8ea44128ece8432d89ca4bfeb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 13:46:45 -0400 Subject: [PATCH 212/319] Add control file for deb build of imgutil --- imgutil/control.tmpl | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 imgutil/control.tmpl diff --git a/imgutil/control.tmpl b/imgutil/control.tmpl new file mode 100644 index 00000000..a0fe21af --- /dev/null +++ b/imgutil/control.tmpl @@ -0,0 +1,8 @@ +Package: confluent-imgutil +Version: #VERSION# +Section: base +Priority: optional +Maintainer: Jarrod Johnson +Description: Web frontend for confluent server +Architecture: all + From 4f18294d93f281af4eaa088a1a29317cf10c8574 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 13:57:38 -0400 Subject: [PATCH 213/319] Fix path in debian build for imgutil --- imgutil/builddeb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imgutil/builddeb b/imgutil/builddeb index 4258fbc4..7e12a6e6 100755 --- a/imgutil/builddeb +++ b/imgutil/builddeb @@ -18,7 +18,7 @@ mv ubuntu* suse15 el7 el9 el8 deb/confluent_imgutil_$VERSION/opt/confluent/lib/i mkdir -p deb/confluent_imgutil_$VERSION/opt/confluent/share/licenses/confluent_imgutil cp LICENSE deb/confluent_imgutil_$VERSION/opt/confluent/share/licenses/confluent_imgutil sed -e 's/#VERSION#/'$VERSION/ control.tmpl > deb/confluent_imgutil_$VERSION/DEBIAN/control -dpkg-deb --build deb/lenovo_confluent_$VERSION +dpkg-deb --build deb/confluent_imgutil_$VERSION if [ ! -z "$1" ]; then - mv deb/lenovo-confluent_$VERSION.deb $1 + mv deb/confluent_imgutil_$VERSION.deb $1 fi From 34b03da4941210cd8ad19ae7852cb7d390eafd82 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 16:33:07 -0400 Subject: [PATCH 214/319] Update for Ubuntu 24.04 diskless --- imgutil/imgutil | 1 + imgutil/ubuntu24.04 | 1 + 2 files changed, 2 insertions(+) create mode 120000 imgutil/ubuntu24.04 diff --git a/imgutil/imgutil b/imgutil/imgutil index 022279cc..d5714306 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -655,6 +655,7 @@ class DebHandler(OsHandler): def prep_root(self, args): shutil.copy('/etc/apt/sources.list', os.path.join(self.targpath, 'etc/apt/sources.list')) + shutil.copytree('/etc/apt/sources.list.d', os.path.join(self.targpath, 'etc/apt/sources.list.d')) args.cmd = ['apt-get', 'update'] run_constrainedx(fancy_chroot, (args, self.targpath)) args.cmd = ['apt-get', '-y', 'install'] + self.includepkgs diff --git a/imgutil/ubuntu24.04 b/imgutil/ubuntu24.04 new file mode 120000 index 00000000..7d13753d --- /dev/null +++ b/imgutil/ubuntu24.04 @@ -0,0 +1 @@ +ubuntu \ No newline at end of file From 1ade704daa4bddb98481ab502d815d9df33bae92 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 22 Jul 2024 16:40:44 -0400 Subject: [PATCH 215/319] Fix imgutil copy of ubuntu sources --- imgutil/imgutil | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index d5714306..1c6d3cc0 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -655,7 +655,8 @@ class DebHandler(OsHandler): def prep_root(self, args): shutil.copy('/etc/apt/sources.list', os.path.join(self.targpath, 'etc/apt/sources.list')) - shutil.copytree('/etc/apt/sources.list.d', os.path.join(self.targpath, 'etc/apt/sources.list.d')) + for listfile in glob.glob('/etc/apt/sources.list.d/*'): + shutil.copy(listfile, os.path.join(self.targpath, listfile[1:])) args.cmd = ['apt-get', 'update'] run_constrainedx(fancy_chroot, (args, self.targpath)) args.cmd = ['apt-get', '-y', 'install'] + self.includepkgs From 33ed1a5e640802deccfaefbab894770ac53b4667 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jul 2024 09:32:20 -0400 Subject: [PATCH 216/319] Add onboot for ubuntu diskless --- .../profiles/default/scripts/onboot.service | 11 +++++ .../profiles/default/scripts/onboot.sh | 40 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.service create mode 100644 confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.service b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.service new file mode 100644 index 00000000..f9235033 --- /dev/null +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.service @@ -0,0 +1,11 @@ +[Unit] +Description=Confluent onboot hook +Requires=network-online.target +After=network-online.target + +[Service] +ExecStart=/opt/confluent/bin/onboot.sh + +[Install] +WantedBy=multi-user.target + diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh new file mode 100644 index 00000000..60ccaa44 --- /dev/null +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# This script is executed on each boot as it is +# completed. It is best to edit the middle of the file as +# noted below so custom commands are executed before +# the script notifies confluent that install is fully complete. + +nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}') +confluent_apikey=$(cat /etc/confluent/confluent.apikey) +v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}') +if [ "$v4meth" = "null" -o -z "$v4meth" ]; then + confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg|awk '{print $2}') +fi +if [ -z "$confluent_mgr" ]; then + confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}') +fi +confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}') +timedatectl set-timezone $(grep ^timezone: /etc/confluent/confluent.deploycfg|awk '{print $2}') +hostnamectl set-hostname $nodename +export nodename confluent_mgr confluent_profile +. /etc/confluent/functions +mkdir -p /var/log/confluent +chmod 700 /var/log/confluent +exec >> /var/log/confluent/confluent-onboot.log +exec 2>> /var/log/confluent/confluent-onboot.log +chmod 600 /var/log/confluent/confluent-onboot.log +tail -f /var/log/confluent/confluent-onboot.log > /dev/console & +logshowpid=$! + +run_remote_python syncfileclient +run_remote_python confignet + +# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh +run_remote_parts onboot.d + +# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/ +run_remote_config onboot.d + +#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus +kill $logshowpid From bb04faed04c28c92e49723f592dbd0c2a5df278d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jul 2024 10:01:53 -0400 Subject: [PATCH 217/319] Explicitly request bash under ubuntu, which tends to use dash --- .../ubuntu20.04-diskless/profiles/default/scripts/onboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh index 60ccaa44..cc470d6f 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/onboot.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This script is executed on each boot as it is # completed. It is best to edit the middle of the file as From a94b9235e8ddf6941a58d9f576b7ec67e4a58613 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jul 2024 10:14:32 -0400 Subject: [PATCH 218/319] Tighten umask on confignet to avoid ubuntu warnings --- confluent_osdeploy/common/profile/scripts/confignet | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 8cda6c83..72462834 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -192,8 +192,10 @@ class NetplanManager(object): if needcfgwrite: needcfgapply = True newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}} + oumask = os.umask(0o77) with open('/etc/netplan/{0}-confluentcfg.yaml'.format(devname), 'w') as planout: planout.write(yaml.dump(newcfg)) + os.umask(oumask) if needcfgapply: subprocess.call(['netplan', 'apply']) From cf4475cfccbc84425fee513ec25c96c26a7bf0a0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jul 2024 10:23:05 -0400 Subject: [PATCH 219/319] Escape the '\W' to avoid stepping on python processing --- imgutil/imgutil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 1c6d3cc0..4ef06776 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -947,7 +947,7 @@ def fancy_chroot(args, installroot): os.chroot(installroot) os.chdir('/') _mount('/', '/', flags=MS_BIND) # Make / manifest as a mounted filesystem in exec - os.environ['PS1'] = '[\x1b[1m\x1b[4mIMGUTIL EXEC {0}\x1b[0m \W]$ '.format(imgname) + os.environ['PS1'] = '[\x1b[1m\x1b[4mIMGUTIL EXEC {0}\x1b[0m \\W]$ '.format(imgname) os.environ['CONFLUENT_IMGUTIL_MODE'] = 'exec' if oshandler: oshandler.set_source('/run/confluentdistro') From 8f58567a700e0283ee681acd7a790bc917a0c693 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 23 Jul 2024 11:05:51 -0400 Subject: [PATCH 220/319] Add ssh to default services of a built ubuntu image --- imgutil/imgutil | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 4ef06776..907a3b64 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -661,6 +661,12 @@ class DebHandler(OsHandler): run_constrainedx(fancy_chroot, (args, self.targpath)) args.cmd = ['apt-get', '-y', 'install'] + self.includepkgs run_constrainedx(fancy_chroot, (args, self.targpath)) + servicefile = os.path.join(self.targpath, 'usr/lib/systemd/system/ssh.service') + if os.path.exists(servicefile): + os.symlink('/usr/lib/systemd/system/ssh.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/ssh.service')) + else: + os.symlink('/usr/lib/systemd/system/sshd.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/sshd.service')) + class ElHandler(OsHandler): From 2235faa76dea31f1a455c1f2208bdd2e086b8be4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jul 2024 08:33:20 -0400 Subject: [PATCH 221/319] Stop using private interface of PyCA PyCA changes their minds about which bindings to include. So make the binding ourselves since PyCA removed it in certain versions. This is a backport of the implementation from the async port effort. --- confluent_server/confluent/sockapi.py | 31 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py index 2d4db15b..8aca0058 100644 --- a/confluent_server/confluent/sockapi.py +++ b/confluent_server/confluent/sockapi.py @@ -70,15 +70,17 @@ try: # so we need to ffi that in using a strategy compatible with PyOpenSSL import OpenSSL.SSL as libssln import OpenSSL.crypto as crypto - from OpenSSL._util import ffi except ImportError: libssl = None - ffi = None crypto = None plainsocket = None libc = ctypes.CDLL(ctypes.util.find_library('c')) +libsslc = ctypes.CDLL(ctypes.util.find_library('ssl')) +libsslc.SSL_CTX_set_cert_verify_callback.argtypes = [ + ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] + def _should_authlog(path, operation): if (operation == 'retrieve' and @@ -389,11 +391,24 @@ def _tlshandler(bind_host, bind_port): else: eventlet.spawn_n(_tlsstartup, cnn) +@ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p) +def verify_stub(store, misc): + return 1 + +class PyObject_HEAD(ctypes.Structure): + _fields_ = [ + ("ob_refcnt", ctypes.c_ssize_t), + ("ob_type", ctypes.c_void_p), + ] + + +# see main/Modules/_ssl.c, only caring about the SSL_CTX pointer +class PySSLContext(ctypes.Structure): + _fields_ = [ + ("ob_base", PyObject_HEAD), + ("ctx", ctypes.c_void_p), + ] -if ffi: - @ffi.callback("int(*)( X509_STORE_CTX *, void*)") - def verify_stub(store, misc): - return 1 def _tlsstartup(cnn): @@ -416,8 +431,8 @@ def _tlsstartup(cnn): ctx.use_certificate_file('/etc/confluent/srvcert.pem') ctx.use_privatekey_file('/etc/confluent/privkey.pem') ctx.set_verify(libssln.VERIFY_PEER, lambda *args: True) - libssln._lib.SSL_CTX_set_cert_verify_callback(ctx._context, - verify_stub, ffi.NULL) + ssl_ctx = PySSLContext.from_address(id(ctx)).ctx + libsslc.SSL_CTX_set_cert_verify_callback(ssl_ctx, verify_stub, 0) cnn = libssl.Connection(ctx, cnn) cnn.set_accept_state() cnn.do_handshake() From c91af840e510ccbed54a858a97623792398aa1ee Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jul 2024 11:12:31 -0400 Subject: [PATCH 222/319] Robust handling of relative link resolv.conf resolv.conf may be a relative link, normal file, or absolute link. Handle all cases. --- imgutil/imgutil | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imgutil/imgutil b/imgutil/imgutil index 907a3b64..bc34af01 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -942,6 +942,8 @@ def fancy_chroot(args, installroot): sourceresolv = '/etc/resolv.conf' if os.path.islink(sourceresolv): sourceresolv = os.readlink(sourceresolv) + # normalize and resolve relative and absolute paths + sourceresolv = os.path.normpath(os.path.join('/etc', sourceresolv)) dstresolv = os.path.join(installroot, 'etc/resolv.conf') if os.path.islink(dstresolv): dstresolv = os.path.join(installroot, os.readlink(dstresolv)[1:]) From 714fefe31bbff6d5e3fee989707554dc59daa2a7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jul 2024 14:41:39 -0400 Subject: [PATCH 223/319] Fix unethered boot for ubuntu --- .../ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh index f1b8e45a..ff8d253d 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh @@ -8,7 +8,7 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed fi done mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay -if grep confluennt_imagemethtod=untethered /proc/cmdline > /dev/null; then +if grep confluennt_imagemethod=untethered /proc/cmdline > /dev/null; then mount -t tmpfs untethered /mnt/remoteimg curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs else From a92edc7924fbc6f593bf05e07cb7d8d38d155050 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jul 2024 15:20:02 -0400 Subject: [PATCH 224/319] Apply ownership sanity check even for root User could accidently run 'confluent' in a way that makes no sense, block it the most accessible way. The pid file should have blocked it, but systemd purges the directory even on failure. --- confluent_server/confluent/main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py index b49d8f56..9fb27972 100644 --- a/confluent_server/confluent/main.py +++ b/confluent_server/confluent/main.py @@ -220,16 +220,20 @@ def setlimits(): def assure_ownership(path): try: if os.getuid() != os.stat(path).st_uid: - sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path)) + if os.getuid() == 0: + sys.stderr.write('Attempting to run as root, when non-root usage is detected\n') + else: + sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path)) sys.exit(1) except OSError as e: if e.errno == 13: - sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path)) + if os.getuid() == 0: + sys.stderr.write('Attempting to run as root, when non-root usage is detected\n') + else: + sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path)) sys.exit(1) def sanity_check(): - if os.getuid() == 0: - return True assure_ownership('/etc/confluent') assure_ownership('/etc/confluent/cfg') for filename in glob.glob('/etc/confluent/cfg/*'): From 6e8d8dabd11e5821d8c34a38803e631c58792c58 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jul 2024 15:28:03 -0400 Subject: [PATCH 225/319] Fix whitespace issue --- confluent_server/confluent/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py index 9fb27972..b0e3508a 100644 --- a/confluent_server/confluent/main.py +++ b/confluent_server/confluent/main.py @@ -227,7 +227,7 @@ def assure_ownership(path): sys.exit(1) except OSError as e: if e.errno == 13: - if os.getuid() == 0: + if os.getuid() == 0: sys.stderr.write('Attempting to run as root, when non-root usage is detected\n') else: sys.stderr.write('{} is not owned by confluent user, change ownership\n'.format(path)) From 8f1a1130a8b9348aa682925ec9cda1871238bb70 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 24 Jul 2024 15:55:04 -0400 Subject: [PATCH 226/319] Add a selfcheck to check misdone collective manager --- confluent_server/bin/confluent_selfcheck | 27 +++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/confluent_server/bin/confluent_selfcheck b/confluent_server/bin/confluent_selfcheck index b9651d17..64794ae4 100755 --- a/confluent_server/bin/confluent_selfcheck +++ b/confluent_server/bin/confluent_selfcheck @@ -24,6 +24,9 @@ import eventlet import greenlet import pwd import signal +import confluent.collective.manager as collective +import confluent.noderange as noderange + def fprint(txt): sys.stdout.write(txt) @@ -258,6 +261,9 @@ if __name__ == '__main__': uuid = rsp.get('id.uuid', {}).get('value', None) if uuid: uuidok = True + if 'collective.managercandidates' in rsp: + # Check if current node in candidates + pass if 'deployment.useinsecureprotocols' in rsp: insec = rsp.get('deployment.useinsecureprotocols', {}).get('value', None) if insec != 'firmware': @@ -276,8 +282,27 @@ if __name__ == '__main__': switch_value = rsp[key].get('value',None) if switch_value and switch_value not in valid_nodes: emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).') - print(f"Checking network configuration for {args.node}") cfg = configmanager.ConfigManager(None) + cfd = cfg.get_node_attributes( + args.node, ('deployment.*', 'collective.managercandidates')) + profile = cfd.get(args.node, {}).get( + 'deployment.pendingprofile', {}).get('value', None) + if not profile: + emprint( + f'{args.node} is not currently set to deploy any ' + 'profile, network boot attempts will be ignored') + candmgrs = cfd.get(args.node, {}).get( + 'collective.managercandidates', {}).get('value', None) + if candmgrs: + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: # fallback to unverified noderange + candmgrs = noderange.NodeRange(candmgrs).nodes + if collective.get_myname() not in candmgrs: + emprint(f'{args.node} has deployment restricted to ' + 'certain collective managers excluding the ' + 'system running the selfcheck') + print(f"Checking network configuration for {args.node}") bootablev4nics = [] bootablev6nics = [] targsships = [] From c3e918fc5fa3dbc7c9bf6d4b247b657b72279c05 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 09:42:24 -0400 Subject: [PATCH 227/319] Fix mistake in untethered support --- .../ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh index ff8d253d..0db99754 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh @@ -8,7 +8,7 @@ for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed fi done mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay -if grep confluennt_imagemethod=untethered /proc/cmdline > /dev/null; then +if grep confluent_imagemethod=untethered /proc/cmdline > /dev/null; then mount -t tmpfs untethered /mnt/remoteimg curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs else From 0f955cd068ca67182d668a693398bdace02d4b9b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 11:24:41 -0400 Subject: [PATCH 228/319] Begin work on a cryptboot support for ubuntu Start implementing a tpm2-initramfs-tool based approach. This requires a bit of an odd transition as the PCR 7 is likely to change between the install phase and the boot phase, so we have to select different PCRs, but that requires an argument to pass that crypttab does not support. --- .../profiles/default/autoinstall/user-data | 1 + .../profiles/default/scripts/post.sh | 26 ++++++++++++++++++- .../profiles/default/scripts/pre.sh | 12 +++++---- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data index 5b6c9894..7c4181d4 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data @@ -10,6 +10,7 @@ autoinstall: storage: layout: name: lvm +#CRYPTBOOT password: %%CRYPTPASS%% match: path: "%%INSTALLDISK%%" user-data: diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index d9730889..69e1593e 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -60,10 +60,12 @@ cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin mount -o bind /dev /target/dev mount -o bind /proc /target/proc mount -o bind /sys /target/sys +mount -o bind /run /target/run mount -o bind /sys/firmware/efi/efivars /target/sys/firmware/efi/efivars if [ 1 = $updategrub ]; then chroot /target update-grub fi + echo "Port 22" >> /etc/ssh/sshd_config echo "Port 2222" >> /etc/ssh/sshd_config echo "Match LocalPort 22" >> /etc/ssh/sshd_config @@ -88,8 +90,30 @@ chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d source /target/etc/confluent/functions run_remote_config post + +if [ -f /etc/confluent_lukspass ]; then + $lukspass=$(cat /etc/confluent_lukspass) + chroot /target apt install tpm2-initramfs-tool + chroot /target tpm2-initramfs-tool seal --data "$(lukspass)" > /dev/null + # The default PCR 7 mutates, and crypttab does not provide a way to pass args + cat > /target/usr/bin/tpm2-initramfs-tool.pcr0 << EOF +#!/bin/sh +tpm2-initramfs-tool -p 0 \$* +EOF + chmod 755 /target/usr/bin/tpm2-initramfs-tool.pcr0 + cat > /target/etc/initramfs-tools/hooks/tpm2-initramfs-tool < /dev/console diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 5db222a7..ee61ac26 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -13,11 +13,6 @@ exec 2>> /var/log/confluent/confluent-pre.log chmod 600 /var/log/confluent/confluent-pre.log cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //') -if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then - echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console - [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev)) - while :; do sleep 86400; done -fi cat /custom-installation/ssh/*pubkey > /root/.ssh/authorized_keys @@ -45,6 +40,13 @@ if [ ! -e /tmp/installdisk ]; then python3 /custom-installation/getinstalldisk fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml +if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then + lukspass=$(head -c 64 < /dev/urandom |base64) + sed -i s!%%CRYPTPASS%%!$lukspass! /autoinstall.yaml + sed -i s!'#CRYPTBOOT'!! /autoinstall.yaml + echo $lukspass > /etc/confluent_lukspass + +fi ) & tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console From 41b722c3f7d583381008fb86968390033225cba1 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 25 Jul 2024 18:38:23 +0200 Subject: [PATCH 229/319] Use natural sort for lists in json dumps Previously, items were randomly arranged in lists in the json dump. This meant that the JSON files were different after each export. Now they are naturally sorted and identical. This should make it easier to save and compare the JSON dumps in version control systems. --- confluent_server/confluent/config/configmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 528924e8..6cbf4604 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -2647,7 +2647,7 @@ class ConfigManager(object): dumpdata[confarea][element][attribute]['cryptvalue'] = '!'.join(cryptval) elif isinstance(dumpdata[confarea][element][attribute], set): dumpdata[confarea][element][attribute] = \ - list(dumpdata[confarea][element][attribute]) + confluent.util.natural_sort(list(dumpdata[confarea][element][attribute])) return json.dumps( dumpdata, sort_keys=True, indent=4, separators=(',', ': ')) From 80296b6cbc8c9c2b03b25f03078fce0e2d66d5c6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 14:05:10 -0400 Subject: [PATCH 230/319] Point to the C context object rather than python class The OpenSSL variant of Context is a python class, but it does have a C context in it. --- confluent_server/confluent/sockapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py index 8aca0058..86534767 100644 --- a/confluent_server/confluent/sockapi.py +++ b/confluent_server/confluent/sockapi.py @@ -431,7 +431,7 @@ def _tlsstartup(cnn): ctx.use_certificate_file('/etc/confluent/srvcert.pem') ctx.use_privatekey_file('/etc/confluent/privkey.pem') ctx.set_verify(libssln.VERIFY_PEER, lambda *args: True) - ssl_ctx = PySSLContext.from_address(id(ctx)).ctx + ssl_ctx = PySSLContext.from_address(id(ctx._context)).ctx libsslc.SSL_CTX_set_cert_verify_callback(ssl_ctx, verify_stub, 0) cnn = libssl.Connection(ctx, cnn) cnn.set_accept_state() From 298be3b30a385af3c2506ba2737dbb530ac38e1d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 14:05:10 -0400 Subject: [PATCH 231/319] Point to the C context object rather than python class The OpenSSL variant of Context is a python class, but it does have a C context in it. --- confluent_server/confluent/sockapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/sockapi.py b/confluent_server/confluent/sockapi.py index 8aca0058..86534767 100644 --- a/confluent_server/confluent/sockapi.py +++ b/confluent_server/confluent/sockapi.py @@ -431,7 +431,7 @@ def _tlsstartup(cnn): ctx.use_certificate_file('/etc/confluent/srvcert.pem') ctx.use_privatekey_file('/etc/confluent/privkey.pem') ctx.set_verify(libssln.VERIFY_PEER, lambda *args: True) - ssl_ctx = PySSLContext.from_address(id(ctx)).ctx + ssl_ctx = PySSLContext.from_address(id(ctx._context)).ctx libsslc.SSL_CTX_set_cert_verify_callback(ssl_ctx, verify_stub, 0) cnn = libssl.Connection(ctx, cnn) cnn.set_accept_state() From 30aa6f382c47c0a5b04269f96daa20690ae296b3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 14:54:15 -0400 Subject: [PATCH 232/319] Ignore duplicate specifications of same key Particularly if traversing a lot of linked configuration, the same key/cert path may come up multiple times, check for equality and if equal, just keep going. --- confluent_server/confluent/certutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/certutil.py b/confluent_server/confluent/certutil.py index 9a478787..4ac67165 100644 --- a/confluent_server/confluent/certutil.py +++ b/confluent_server/confluent/certutil.py @@ -76,7 +76,7 @@ def get_certificate_paths(): continue kploc = check_apache_config(os.path.join(currpath, fname)) - if keypath and kploc[0]: + if keypath and kploc[0] and keypath != kploc[0]: return None, None # Ambiguous... if kploc[0]: keypath, certpath = kploc From 626f16cb6fcac5a7c9531014766b287ac9ca2d72 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 14:54:15 -0400 Subject: [PATCH 233/319] Ignore duplicate specifications of same key Particularly if traversing a lot of linked configuration, the same key/cert path may come up multiple times, check for equality and if equal, just keep going. --- confluent_server/confluent/certutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/certutil.py b/confluent_server/confluent/certutil.py index 9a478787..4ac67165 100644 --- a/confluent_server/confluent/certutil.py +++ b/confluent_server/confluent/certutil.py @@ -76,7 +76,7 @@ def get_certificate_paths(): continue kploc = check_apache_config(os.path.join(currpath, fname)) - if keypath and kploc[0]: + if keypath and kploc[0] and keypath != kploc[0]: return None, None # Ambiguous... if kploc[0]: keypath, certpath = kploc From 956e473fa6ea6fe3af1c61d32f02090ab6b59857 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 15:25:09 -0400 Subject: [PATCH 234/319] Have SSDP fallback to unverified noderanges when looking at candidates --- confluent_server/confluent/discovery/protocols/ssdp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 3c1edc74..12ec4ba7 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -251,7 +251,10 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None): break candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) if candmgrs: - candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: + candmgrs = noderange.NodeRange(candmgrs).nodes if collective.get_myname() not in candmgrs: break currtime = time.time() From dc7c9f4a3d324c8881fd312d8132ed8207f64e15 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 25 Jul 2024 15:25:09 -0400 Subject: [PATCH 235/319] Have SSDP fallback to unverified noderanges when looking at candidates --- confluent_server/confluent/discovery/protocols/ssdp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 3c1edc74..12ec4ba7 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -251,7 +251,10 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None): break candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) if candmgrs: - candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: + candmgrs = noderange.NodeRange(candmgrs).nodes if collective.get_myname() not in candmgrs: break currtime = time.time() From 1d6009a2f2d58211e031e8290d3367a2937422bb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 10:33:38 -0400 Subject: [PATCH 236/319] Switch to using systemd-cryptenroll The design more cleanly uses luks slot, but requires providing initramfs hooks. Those hooks are provided now. --- .../profiles/default/scripts/post.sh | 73 +++++++++++++++---- .../profiles/default/scripts/pre.sh | 9 ++- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 69e1593e..2c8be0c0 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -92,23 +92,66 @@ source /target/etc/confluent/functions run_remote_config post if [ -f /etc/confluent_lukspass ]; then - $lukspass=$(cat /etc/confluent_lukspass) - chroot /target apt install tpm2-initramfs-tool - chroot /target tpm2-initramfs-tool seal --data "$(lukspass)" > /dev/null - # The default PCR 7 mutates, and crypttab does not provide a way to pass args - cat > /target/usr/bin/tpm2-initramfs-tool.pcr0 << EOF -#!/bin/sh -tpm2-initramfs-tool -p 0 \$* -EOF - chmod 755 /target/usr/bin/tpm2-initramfs-tool.pcr0 - cat > /target/etc/initramfs-tools/hooks/tpm2-initramfs-tool </target/etc/initramfs-tools/scripts/local-top/systemdecrypt << EOS +#!/bin/sh +case \$1 in +prereqs) + echo + exit 0 + ;; +esac + +systemdecryptnow() { +. /usr/lib/cryptsetup/functions +local CRYPTTAB_SOURCE=\$(awk '{print \$2}' /systemdecrypt/crypttab) +local CRYPTTAB_NAME=\$(awk '{print \$1}' /systemdecrypt/crypttab) +crypttab_resolve_source +/lib/systemd/systemd-cryptsetup attach "\${CRYPTTAB_NAME}" "\${CRYPTTAB_SOURCE}" none tpm2-device=auto +} + +systemdecryptnow +EOS + chmod 755 /target/etc/initramfs-tools/scripts/local-top/systemdecrypt + cat > /target/etc/initramfs-tools/hooks/systemdecrypt <> \$DESTDIR/scripts/local-top/ORDER + +if [ -f \$DESTDIR/cryptroot/crypttab ]; then + mv \$DESTDIR/cryptroot/crypttab \$DESTDIR/systemdecrypt/crypttab +fi EOF - chmod 755 /target/etc/initramfs-tools/hooks/tpm2-initramfs-tool chroot /target update-initramfs -u fi python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus -d 'status: staged' diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index ee61ac26..bfe1c7db 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -41,10 +41,15 @@ if [ ! -e /tmp/installdisk ]; then fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then - lukspass=$(head -c 64 < /dev/urandom |base64) + if ! grep '#CRYPTBOOT' /autoinstall.yaml > /dev/null; then + echo "****Encrypted boot requested, but the user-data does not have a hook to enable,halting install" > /dev/console + [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but the user-data does not have a hook to enable,halting install" >> $(cat /tmp/autoconsdev)) + while :; do sleep 86400; done + fi + lukspass=$(head -c 66 < /dev/urandom |base64 -w0) sed -i s!%%CRYPTPASS%%!$lukspass! /autoinstall.yaml sed -i s!'#CRYPTBOOT'!! /autoinstall.yaml - echo $lukspass > /etc/confluent_lukspass + echo -n $lukspass > /etc/confluent_lukspass fi ) & From 58ee85f39ebe558f11108942f18c8966e2cae896 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 11:33:01 -0400 Subject: [PATCH 237/319] Rework Ubuntu addcrypt support The comment based hook is destroyed during early install process. Use python to manipulate the autoinstall file in a more sophisticated way. Also refactor the initramfs hook material to be standalone files. --- .../profiles/default/autoinstall/user-data | 1 - .../profiles/default/scripts/addcrypt | 12 +++++ .../profiles/default/scripts/post.sh | 49 ++----------------- .../profiles/default/scripts/pre.sh | 3 +- .../profiles/default/scripts/systemdecrypt | 17 +++++++ .../default/scripts/systemdecrypt-hook | 22 +++++++++ 6 files changed, 58 insertions(+), 46 deletions(-) create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data index 7c4181d4..5b6c9894 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data @@ -10,7 +10,6 @@ autoinstall: storage: layout: name: lvm -#CRYPTBOOT password: %%CRYPTPASS%% match: path: "%%INSTALLDISK%%" user-data: diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt new file mode 100644 index 00000000..4f2ae905 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt @@ -0,0 +1,12 @@ +import yaml +import sys + +ainst = {} +with open('/autoinstall.yaml', 'r') as allin: + ainst = yaml.safe_load(allin) + +ainst['storage']['layout']['password'] = sys.argv[1] + +with open('/autoinstall.yaml', 'w') as allout: + yaml.safe_dump(ainst, allout) + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 2c8be0c0..998f7bda 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -108,50 +108,11 @@ if [ -f /etc/confluent_lukspass ]; then $lukspass=$(cat /etc/confluent_lukspass) chroot /target apt install libtss2-rc0 PASSWORD=$(lukspass) chroot /target systemd-cryptenroll --tpm2-device=auto $CRYPTTAB_SOURCE - cat >/target/etc/initramfs-tools/scripts/local-top/systemdecrypt << EOS -#!/bin/sh -case \$1 in -prereqs) - echo - exit 0 - ;; -esac - -systemdecryptnow() { -. /usr/lib/cryptsetup/functions -local CRYPTTAB_SOURCE=\$(awk '{print \$2}' /systemdecrypt/crypttab) -local CRYPTTAB_NAME=\$(awk '{print \$1}' /systemdecrypt/crypttab) -crypttab_resolve_source -/lib/systemd/systemd-cryptsetup attach "\${CRYPTTAB_NAME}" "\${CRYPTTAB_SOURCE}" none tpm2-device=auto -} - -systemdecryptnow -EOS - chmod 755 /target/etc/initramfs-tools/scripts/local-top/systemdecrypt - cat > /target/etc/initramfs-tools/hooks/systemdecrypt <> \$DESTDIR/scripts/local-top/ORDER - -if [ -f \$DESTDIR/cryptroot/crypttab ]; then - mv \$DESTDIR/cryptroot/crypttab \$DESTDIR/systemdecrypt/crypttab -fi -EOF + fetch_remote systemdecrypt + mv systemdecrypt /target/etc/initramfs-tools/scripts/local-top/systemdecrypt + fetch_remote systemdecrypt-hook + mv systemdecrypt-hook /target/etc/initramfs-tools/hooks/systemdecrypt + chmod 755 /target/etc/initramfs-tools/scripts/local-top/systemdecrypt /target/etc/initramfs-tools/hooks/systemdecrypt chroot /target update-initramfs -u fi python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus -d 'status: staged' diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index bfe1c7db..db0e967d 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -41,12 +41,13 @@ if [ ! -e /tmp/installdisk ]; then fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then + lukspass=$(head -c 66 < /dev/urandom |base64 -w0) + run_remote_python addcrypt if ! grep '#CRYPTBOOT' /autoinstall.yaml > /dev/null; then echo "****Encrypted boot requested, but the user-data does not have a hook to enable,halting install" > /dev/console [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but the user-data does not have a hook to enable,halting install" >> $(cat /tmp/autoconsdev)) while :; do sleep 86400; done fi - lukspass=$(head -c 66 < /dev/urandom |base64 -w0) sed -i s!%%CRYPTPASS%%!$lukspass! /autoinstall.yaml sed -i s!'#CRYPTBOOT'!! /autoinstall.yaml echo -n $lukspass > /etc/confluent_lukspass diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt new file mode 100644 index 00000000..6f0cbaed --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt @@ -0,0 +1,17 @@ +#!/bin/sh +case $1 in +prereqs) + echo + exit 0 + ;; +esac + +systemdecryptnow() { +. /usr/lib/cryptsetup/functions +local CRYPTTAB_SOURCE=$(awk '{print $2}' /systemdecrypt/crypttab) +local CRYPTTAB_NAME=$(awk '{print $1}' /systemdecrypt/crypttab) +crypttab_resolve_source +/lib/systemd/systemd-cryptsetup attach "${CRYPTTAB_NAME}" "${CRYPTTAB_SOURCE}" none tpm2-device=auto +} + +systemdecryptnow diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook new file mode 100644 index 00000000..48c9d16d --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook @@ -0,0 +1,22 @@ +#!/bin/sh +case "$1" in + prereqs) + echo + exit 0 + ;; +esac + +. /usr/share/initramfs-tools/hook-functions +mkdir -p $DESTDIR/systemdecrypt +copy_exec /lib/systemd/systemd-cryptsetup /lib/systemd +for i in /lib/x86_64-linux-gnu/libtss2* +do + copy_exec ${i} /lib/x86_64-linux-gnu +done +mkdir -p $DESTDIR/scripts/local-top + +echo /scripts/local-top/systemdecrypt >> $DESTDIR/scripts/local-top/ORDER + +if [ -f $DESTDIR/cryptroot/crypttab ]; then + mv $DESTDIR/cryptroot/crypttab $DESTDIR/systemdecrypt/crypttab +fi From f482d2ead993ae651f742e43b5977406e5d64cb4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 11:35:49 -0400 Subject: [PATCH 238/319] Amend crypt hook check The comment was changed, check for password instead. --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index db0e967d..02402a99 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -43,7 +43,7 @@ sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then lukspass=$(head -c 66 < /dev/urandom |base64 -w0) run_remote_python addcrypt - if ! grep '#CRYPTBOOT' /autoinstall.yaml > /dev/null; then + if ! grep 'pasword:' /autoinstall.yaml > /dev/null; then echo "****Encrypted boot requested, but the user-data does not have a hook to enable,halting install" > /dev/console [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but the user-data does not have a hook to enable,halting install" >> $(cat /tmp/autoconsdev)) while :; do sleep 86400; done From 1ddf735590bd6479dc7fa797a88869adaa963283 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 11:50:53 -0400 Subject: [PATCH 239/319] Fix omitted argument to addcrypt --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 02402a99..bd9d1d60 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -42,7 +42,7 @@ fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then lukspass=$(head -c 66 < /dev/urandom |base64 -w0) - run_remote_python addcrypt + run_remote_python addcrypt "$lukspass" if ! grep 'pasword:' /autoinstall.yaml > /dev/null; then echo "****Encrypted boot requested, but the user-data does not have a hook to enable,halting install" > /dev/console [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but the user-data does not have a hook to enable,halting install" >> $(cat /tmp/autoconsdev)) From c1747ad24ca961b10df0e94819b58861f7b2f0bb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 11:54:10 -0400 Subject: [PATCH 240/319] Correct spelling of key for luks check --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index bd9d1d60..77a16906 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -43,7 +43,7 @@ sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then lukspass=$(head -c 66 < /dev/urandom |base64 -w0) run_remote_python addcrypt "$lukspass" - if ! grep 'pasword:' /autoinstall.yaml > /dev/null; then + if ! grep 'password:' /autoinstall.yaml > /dev/null; then echo "****Encrypted boot requested, but the user-data does not have a hook to enable,halting install" > /dev/console [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but the user-data does not have a hook to enable,halting install" >> $(cat /tmp/autoconsdev)) while :; do sleep 86400; done From c563f48c71acfd9ffc44c1f88caef339b527de9d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 12:30:41 -0400 Subject: [PATCH 241/319] Fix assignment of lukspass variable. --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 998f7bda..28d45e41 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -105,7 +105,7 @@ if [ -f /etc/confluent_lukspass ]; then wall "Unable to find $CRYPTTAB_SOURCE, halting install" while :; do sleep 86400; done fi - $lukspass=$(cat /etc/confluent_lukspass) + lukspass=$(cat /etc/confluent_lukspass) chroot /target apt install libtss2-rc0 PASSWORD=$(lukspass) chroot /target systemd-cryptenroll --tpm2-device=auto $CRYPTTAB_SOURCE fetch_remote systemdecrypt From 7a602f58b2e1b62a0c18b4f5e9b72d20817bbf5d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 13:47:13 -0400 Subject: [PATCH 242/319] Fixes for ubuntu profile tpm support --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 28d45e41..4af3a01f 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -107,7 +107,7 @@ if [ -f /etc/confluent_lukspass ]; then fi lukspass=$(cat /etc/confluent_lukspass) chroot /target apt install libtss2-rc0 - PASSWORD=$(lukspass) chroot /target systemd-cryptenroll --tpm2-device=auto $CRYPTTAB_SOURCE + PASSWORD=$lukspass chroot /target systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs="" $CRYPTTAB_SOURCE fetch_remote systemdecrypt mv systemdecrypt /target/etc/initramfs-tools/scripts/local-top/systemdecrypt fetch_remote systemdecrypt-hook From 2df902e80e2df7782f6c594c3195862d1c04ea69 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 14:07:54 -0400 Subject: [PATCH 243/319] Remove luks password from argv Pass the luks password by environment variable instead. --- .../ubuntu22.04/profiles/default/scripts/addcrypt | 4 ++-- .../ubuntu22.04/profiles/default/scripts/pre.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt index 4f2ae905..750753c1 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/addcrypt @@ -1,11 +1,11 @@ import yaml -import sys +import os ainst = {} with open('/autoinstall.yaml', 'r') as allin: ainst = yaml.safe_load(allin) -ainst['storage']['layout']['password'] = sys.argv[1] +ainst['storage']['layout']['password'] = os.environ['lukspass'] with open('/autoinstall.yaml', 'w') as allout: yaml.safe_dump(ainst, allout) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 77a16906..4ec3f822 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -42,7 +42,8 @@ fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then lukspass=$(head -c 66 < /dev/urandom |base64 -w0) - run_remote_python addcrypt "$lukspass" + export lukspass + run_remote_python addcrypt if ! grep 'password:' /autoinstall.yaml > /dev/null; then echo "****Encrypted boot requested, but the user-data does not have a hook to enable,halting install" > /dev/console [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but the user-data does not have a hook to enable,halting install" >> $(cat /tmp/autoconsdev)) From 332068074d1b93d1b4b9a85e38643ba0d93f85ba Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 16:54:58 -0400 Subject: [PATCH 244/319] Extend systemdecrypt hook to support Ubuntu 24.04 Ubuntu 240.4 systemd-cryptsetup now has an external dependency. --- .../ubuntu22.04/profiles/default/scripts/systemdecrypt-hook | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook index 48c9d16d..ee602c7c 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/systemdecrypt-hook @@ -13,6 +13,10 @@ for i in /lib/x86_64-linux-gnu/libtss2* do copy_exec ${i} /lib/x86_64-linux-gnu done +if [ -f /lib/x86_64-linux-gnu/cryptsetup/libcryptsetup-token-systemd-tpm2.so ]; then + mkdir -p $DESTDIR/lib/x86_64-linux-gnu/cryptsetup + copy_exec /lib/x86_64-linux-gnu/cryptsetup/libcryptsetup-token-systemd-tpm2.so /lib/x86_64-linux-gnu/cryptsetup +fi mkdir -p $DESTDIR/scripts/local-top echo /scripts/local-top/systemdecrypt >> $DESTDIR/scripts/local-top/ORDER From 1af898dcb8a07ccf537bf41cb592066ce45fc0e2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 17:43:51 -0400 Subject: [PATCH 245/319] Fix encryptboot on EL8/EL9 --- confluent_osdeploy/el8/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh index cd831360..89d989b8 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh @@ -114,7 +114,7 @@ confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/installdisk if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then INSTALLDISK=$(cat /tmp/installdisk) - sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e s/%%LUKSHOOK%%/$LUKSPARTY/ /tmp/partitioning.template > /tmp/partitioning + sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e "s/%%LUKSHOOK%%/$LUKSPARTY/" /tmp/partitioning.template > /tmp/partitioning vgchange -a n >& /dev/null wipefs -a -f /dev/$INSTALLDISK >& /dev/null fi From bee9f1819717b2c5475671d9d77293f1bf11ca47 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 26 Jul 2024 17:59:42 -0400 Subject: [PATCH 246/319] Tolerate / in the apikey for LUKS setup The apikey is highly likely to have a /, and so we need to use something not in the base64 alphabet as a delimiter. --- confluent_osdeploy/el8/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh index 89d989b8..4deff814 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh @@ -114,7 +114,7 @@ confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/installdisk if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then INSTALLDISK=$(cat /tmp/installdisk) - sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e "s/%%LUKSHOOK%%/$LUKSPARTY/" /tmp/partitioning.template > /tmp/partitioning + sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e "s!%%LUKSHOOK%%!$LUKSPARTY!" /tmp/partitioning.template > /tmp/partitioning vgchange -a n >& /dev/null wipefs -a -f /dev/$INSTALLDISK >& /dev/null fi From 329f2b4485fc26005525bd11616cc95d1641e0d3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 29 Jul 2024 10:17:14 -0400 Subject: [PATCH 247/319] Amend cryptboot implementation for Ubuntu 22/24, EL8/EL9 Provide mechanism for administrator to place a custom key for potential interactive recovery into /var/lib/confluent/private/os//pending/luks.key If not provided, generate a unique one for each install. Either way, persist the key in /etc/confluent/luks.key, to facilitate later resealing if the user wants (clevis nor systemd prior to 256 supports unlock via TPM2, so keyfile is required for now). Migrating to otherwise escrowed passphrases and/or sealing to specific TPMs will be left to operators and/or third parties. --- confluent_osdeploy/el8/profiles/default/scripts/pre.sh | 10 ++++++++-- .../el8/profiles/default/scripts/tpm_luks.sh | 5 +++-- .../ubuntu22.04/profiles/default/scripts/post.sh | 2 ++ .../ubuntu22.04/profiles/default/scripts/pre.sh | 7 +++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh index 4deff814..880d22ac 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/pre.sh @@ -90,8 +90,14 @@ touch /tmp/cryptpkglist touch /tmp/pkglist touch /tmp/addonpackages if [ "$cryptboot" == "tpm2" ]; then - LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)" - echo $cryptboot >> /tmp/cryptboot + lukspass=$(python3 /opt/confluent/bin/apiclient /confluent-api/self/profileprivate/pending/luks.key 2> /dev/null) + if [ -z "$lukspass" ]; then + lukspass=$(python3 -c 'import os;import base64;print(base64.b64encode(os.urandom(66)).decode())') + fi + echo $lukspass > /etc/confluent/luks.key + chmod 000 /etc/confluent/luks.key + LUKSPARTY="--encrypted --passphrase=$lukspass" + echo $cryptboot >> /tmp/cryptboot echo clevis-dracut >> /tmp/cryptpkglist fi diff --git a/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh b/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh index df9c857f..c457ffd4 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh @@ -1,4 +1,5 @@ #!/bin/sh cryptdisk=$(blkid -t TYPE="crypto_LUKS"|sed -e s/:.*//) -clevis luks bind -f -d $cryptdisk -k - tpm2 '{}' < /etc/confluent/confluent.apikey -cryptsetup luksRemoveKey $cryptdisk < /etc/confluent/confluent.apikey +clevis luks bind -f -d $cryptdisk -k /etc/cofluent/luks.key tpm2 '{}' +chmod 000 /etc/confluent/luks.key +#cryptsetup luksRemoveKey $cryptdisk < /etc/confluent/confluent.apikey diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh index 4af3a01f..a86695ca 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh @@ -105,6 +105,8 @@ if [ -f /etc/confluent_lukspass ]; then wall "Unable to find $CRYPTTAB_SOURCE, halting install" while :; do sleep 86400; done fi + cp /etc/confluent_lukspass /target/etc/confluent/luks.key + chmod 000 /target/etc/confluent/luks.key lukspass=$(cat /etc/confluent_lukspass) chroot /target apt install libtss2-rc0 PASSWORD=$lukspass chroot /target systemd-cryptenroll --tpm2-device=auto --tpm2-pcrs="" $CRYPTTAB_SOURCE diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 4ec3f822..5b609565 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -41,7 +41,10 @@ if [ ! -e /tmp/installdisk ]; then fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then - lukspass=$(head -c 66 < /dev/urandom |base64 -w0) + lukspass=$(python3 /opt/confluent/bin/apiclient /confluent-api/self/profileprivate/pending/luks.key 2> /dev/null) + if [ -z "$lukspass" ]; then + lukspass=$(head -c 66 < /dev/urandom |base64 -w0) + fi export lukspass run_remote_python addcrypt if ! grep 'password:' /autoinstall.yaml > /dev/null; then @@ -52,7 +55,7 @@ if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "n sed -i s!%%CRYPTPASS%%!$lukspass! /autoinstall.yaml sed -i s!'#CRYPTBOOT'!! /autoinstall.yaml echo -n $lukspass > /etc/confluent_lukspass - + chmod 000 /etc/confluent_lukspass fi ) & tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console From e6dc383d2598b5c6a9d851b9ed7b5894a25e0532 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 29 Jul 2024 11:22:07 -0400 Subject: [PATCH 248/319] Fix mistake in EL8/EL9 LUKS --- confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh b/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh index c457ffd4..359c46f6 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh +++ b/confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh @@ -1,5 +1,5 @@ #!/bin/sh cryptdisk=$(blkid -t TYPE="crypto_LUKS"|sed -e s/:.*//) -clevis luks bind -f -d $cryptdisk -k /etc/cofluent/luks.key tpm2 '{}' +clevis luks bind -f -d $cryptdisk -k - tpm2 '{}' < /etc/confluent/luks.key chmod 000 /etc/confluent/luks.key #cryptsetup luksRemoveKey $cryptdisk < /etc/confluent/confluent.apikey From 1c4f1ae8175bcd03c8aa0e2bae6507ad23466ceb Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 29 Jul 2024 15:21:10 -0400 Subject: [PATCH 249/319] Try to add ntp and timezones to Ubuntu scripted install --- .../profiles/default/scripts/mergetime | 26 +++++++++++++++++++ .../profiles/default/scripts/pre.sh | 1 + 2 files changed, 27 insertions(+) create mode 100644 confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime new file mode 100644 index 00000000..0cacc1e8 --- /dev/null +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +import yaml +import os + +ainst = {} +with open('/autoinstall.yaml', 'r') as allin: + ainst = yaml.safe_load(allin) + +tz = None +ntps = [] +with open('/etc/confluent/confluent.deploycfg', 'r') as confluentdeploycfg: + dcfg = yaml.safe_load(confluentdeploycfg) + tz = dcfg['timezone'] + ntps = dcfg.get('ntpservers', []) + +if ntps and not ainst.get('ntp', None): + ainst['ntp'] = {} + ainst['ntp']['enabled'] = True + ainst['servers'] = ntps + +if tz and not ainst.get('timezone'): + ainst['timezone'] = tz + +with open('/autoinstall.yaml', 'w') as allout: + yaml.safe_dump(ainst, allout) + diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 5b609565..ad55120a 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -40,6 +40,7 @@ if [ ! -e /tmp/installdisk ]; then python3 /custom-installation/getinstalldisk fi sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml +run_remote_python mergetime if [ "$cryptboot" != "" ] && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then lukspass=$(python3 /opt/confluent/bin/apiclient /confluent-api/self/profileprivate/pending/luks.key 2> /dev/null) if [ -z "$lukspass" ]; then From 71ca9ef76c7abed0752931d1545b447288f3b7c0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 29 Jul 2024 15:57:34 -0400 Subject: [PATCH 250/319] Fix path to ntp servers in user-data mod for ubuntu --- .../ubuntu22.04/profiles/default/scripts/mergetime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime index 0cacc1e8..7edb2632 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/mergetime @@ -16,7 +16,7 @@ with open('/etc/confluent/confluent.deploycfg', 'r') as confluentdeploycfg: if ntps and not ainst.get('ntp', None): ainst['ntp'] = {} ainst['ntp']['enabled'] = True - ainst['servers'] = ntps + ainst['ntp']['servers'] = ntps if tz and not ainst.get('timezone'): ainst['timezone'] = tz From 89bd7c6053c0cd688e9b009b3cb5f07d58a5ecae Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 1 Aug 2024 09:40:39 -0400 Subject: [PATCH 251/319] Force load IB/OPA modules in case of IB boot Ubuntu diskless was not working with boot over IB --- .../initramfs/scripts/init-premount/confluent | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/initramfs/scripts/init-premount/confluent b/confluent_osdeploy/ubuntu20.04-diskless/initramfs/scripts/init-premount/confluent index 2f7094b9..a4ca41cf 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/initramfs/scripts/init-premount/confluent +++ b/confluent_osdeploy/ubuntu20.04-diskless/initramfs/scripts/init-premount/confluent @@ -58,6 +58,10 @@ if ! grep console= /proc/cmdline > /dev/null; then echo "Automatic console configured for $autocons" fi echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd +modprobe ib_ipoib +modprobe ib_umad +modprobe hfi1 +modprobe mlx5_ib cd /sys/class/net for nic in *; do ip link set $nic up From acce4de739c54bd1dc7c87bac85617274da0971f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 2 Aug 2024 11:57:04 -0400 Subject: [PATCH 252/319] Add support for an OpenBMC modification While stock OpenBmc does not care about subprotocols, some implementations use it as a carrier for the XSRF-TOKEN. Since base OpenBmc ignores it, we just offer it to any implementation just in case. --- confluent_server/confluent/plugins/console/openbmc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/console/openbmc.py b/confluent_server/confluent/plugins/console/openbmc.py index 17acae7c..8677ce17 100644 --- a/confluent_server/confluent/plugins/console/openbmc.py +++ b/confluent_server/confluent/plugins/console/openbmc.py @@ -141,7 +141,7 @@ class TsmConsole(conapi.Console): bmc = prefix + ']' self.ws = WrappedWebSocket(host=bmc) self.ws.set_verify_callback(kv) - self.ws.connect('wss://{0}/console0'.format(self.bmc), host=bmc, cookie='XSRF-TOKEN={0}; SESSION={1}'.format(wc.cookies['XSRF-TOKEN'], wc.cookies['SESSION'])) + self.ws.connect('wss://{0}/console0'.format(self.bmc), host=bmc, cookie='XSRF-TOKEN={0}; SESSION={1}'.format(wc.cookies['XSRF-TOKEN'], wc.cookies['SESSION']), subprotocols=[wc.cookies['XSRF-TOKEN']]) self.connected = True eventlet.spawn_n(self.recvdata) return From 4b6d41d2f82e1b0935542f9173a82705f8da1f6e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 2 Aug 2024 17:35:39 -0400 Subject: [PATCH 253/319] Begin work to support V4 Lenovo servers V4 Lenovo servers will have XCC3, and will have differences and mark an unambiguously redfish capable onboarding process. For now identify XCC3 variants and mark them, stubbing them to the xcc handler. An XCC3 handler will be made basing on the generic redfishbmc handler with accomodations for XCC specific data (e.g. DeviceDescription attributes and the Lenovo default user/password choice). --- confluent_server/confluent/discovery/core.py | 3 +++ .../confluent/discovery/protocols/ssdp.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index dfb50b9f..bb4c99df 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -113,6 +113,7 @@ nodehandlers = { 'service:lenovo-smm': smm, 'service:lenovo-smm2': smm, 'lenovo-xcc': xcc, + 'lenovo-xcc3': xcc, 'service:management-hardware.IBM:integrated-management-module2': imm, 'pxe-client': pxeh, 'onie-switch': None, @@ -132,6 +133,7 @@ servicenames = { 'service:lenovo-smm2': 'lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', + 'lenovo-xcc3': 'lenovo-xcc3', #'openbmc': 'openbmc', 'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2', 'service:io-device.Lenovo:management-module': 'lenovo-switch', @@ -147,6 +149,7 @@ servicebyname = { 'lenovo-smm2': 'service:lenovo-smm2', 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', + 'lenovo-xcc3': 'lenovo-xcc3', 'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2', 'lenovo-switch': 'service:io-device.Lenovo:management-module', 'thinkagile-storage': 'service:thinkagile-storagebmc', diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 12ec4ba7..34b4f6d0 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -431,18 +431,25 @@ def check_fish(urldata, port=443, verifycallback=None): url, data = urldata try: wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5) - peerinfo = wc.grab_json_response(url) + peerinfo = wc.grab_json_response(url, headers={'Accept': 'application/json'}) except socket.error: return None if url == '/DeviceDescription.json': + if not peerinfo: + return None try: peerinfo = peerinfo[0] + except KeyError: + peerinfo['xcc-variant'] = '3' + except IndexError: ++ return None + try: myuuid = peerinfo['node-uuid'].lower() if '-' not in myuuid: myuuid = '-'.join([myuuid[:8], myuuid[8:12], myuuid[12:16], myuuid[16:20], myuuid[20:]]) data['uuid'] = myuuid data['attributes'] = peerinfo - data['services'] = ['lenovo-xcc'] + data['services'] = ['lenovo-xcc'] if 'xcc-variant' not in peerinfo else ['lenovo-xcc' + peerinfo['xcc-variant']] return data except (IndexError, KeyError): return None From e9d4174ce5372e3538a539b3661d200ab04202d8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 08:35:10 -0400 Subject: [PATCH 254/319] Reapply "Add MegaRAC discovery support for recent MegaRAC" This reverts commit 9d979256eb2c8f96e6a2c334beb57a504eb30f02. --- confluent_server/confluent/discovery/core.py | 9 +- .../confluent/discovery/handlers/megarac.py | 51 ++++ .../discovery/handlers/redfishbmc.py | 269 ++++++++++++++++++ .../confluent/discovery/protocols/ssdp.py | 52 +++- 4 files changed, 366 insertions(+), 15 deletions(-) create mode 100644 confluent_server/confluent/discovery/handlers/megarac.py create mode 100644 confluent_server/confluent/discovery/handlers/redfishbmc.py diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index bb4c99df..b734cece 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -74,6 +74,7 @@ import confluent.discovery.handlers.tsm as tsm import confluent.discovery.handlers.pxe as pxeh import confluent.discovery.handlers.smm as smm import confluent.discovery.handlers.xcc as xcc +import confluent.discovery.handlers.megarac as megarac import confluent.exceptions as exc import confluent.log as log import confluent.messages as msg @@ -114,6 +115,7 @@ nodehandlers = { 'service:lenovo-smm2': smm, 'lenovo-xcc': xcc, 'lenovo-xcc3': xcc, + 'megarac-bmc': megarac, 'service:management-hardware.IBM:integrated-management-module2': imm, 'pxe-client': pxeh, 'onie-switch': None, @@ -134,6 +136,7 @@ servicenames = { 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', 'lenovo-xcc3': 'lenovo-xcc3', + 'megarac-bmc': 'megarac-bmc', #'openbmc': 'openbmc', 'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2', 'service:io-device.Lenovo:management-module': 'lenovo-switch', @@ -150,6 +153,7 @@ servicebyname = { 'affluent-switch': 'affluent-switch', 'lenovo-xcc': 'lenovo-xcc', 'lenovo-xcc3': 'lenovo-xcc3', + 'megarac-bmc': 'megarac-bmc', 'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2', 'lenovo-switch': 'service:io-device.Lenovo:management-module', 'thinkagile-storage': 'service:thinkagile-storagebmc', @@ -456,7 +460,7 @@ def iterate_addrs(addrs, countonly=False): yield 1 return yield addrs - + def _parameterize_path(pathcomponents): listrequested = False childcoll = True @@ -545,7 +549,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents): if len(pathcomponents) > 2: raise Exception('TODO') currsubs = get_subscriptions() - return [msg.ChildCollection(x) for x in currsubs] + return [msg.ChildCollection(x) for x in currsubs] elif operation == 'retrieve': return handle_read_api_request(pathcomponents) elif (operation in ('update', 'create') and @@ -1706,3 +1710,4 @@ if __name__ == '__main__': start_detection() while True: eventlet.sleep(30) + diff --git a/confluent_server/confluent/discovery/handlers/megarac.py b/confluent_server/confluent/discovery/handlers/megarac.py new file mode 100644 index 00000000..d7d8786a --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/megarac.py @@ -0,0 +1,51 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.redfishbmc as redfishbmc +import eventlet.support.greendns + + +getaddrinfo = eventlet.support.greendns.getaddrinfo + + +class NodeHandler(redfishbmc.NodeHandler): + + def get_firmware_default_account_info(self): + return ('admin', 'admin') + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]]} + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) + diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py new file mode 100644 index 00000000..eed401de --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/redfishbmc.py @@ -0,0 +1,269 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.generic as generic +import confluent.exceptions as exc +import confluent.netutil as netutil +import confluent.util as util +import eventlet +import eventlet.support.greendns +import json +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode + +getaddrinfo = eventlet.support.greendns.getaddrinfo + +webclient = eventlet.import_patched('pyghmi.util.webclient') + +def get_host_interface_urls(wc, mginfo): + returls = [] + hifurl = mginfo.get('HostInterfaces', {}).get('@odata.id', None) + if not hifurl: + return None + hifinfo = wc.grab_json_response(hifurl) + hifurls = hifinfo.get('Members', []) + for hifurl in hifurls: + hifurl = hifurl['@odata.id'] + hifinfo = wc.grab_json_response(hifurl) + acturl = hifinfo.get('ManagerEthernetInterface', {}).get('@odata.id', None) + if acturl: + returls.append(acturl) + return returls + + +class NodeHandler(generic.NodeHandler): + devname = 'BMC' + + def __init__(self, info, configmanager): + self.trieddefault = None + self.targuser = None + self.curruser = None + self.currpass = None + self.targpass = None + self.nodename = None + self.csrftok = None + self.channel = None + self.atdefault = True + super(NodeHandler, self).__init__(info, configmanager) + + def get_firmware_default_account_info(self): + raise Exception('This must be subclassed') + + def scan(self): + c = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) + i = c.grab_json_response('/redfish/v1/') + uuid = i.get('UUID', None) + if uuid: + self.info['uuid'] = uuid.lower() + + def validate_cert(self, certificate): + # broadly speaking, merely checks consistency moment to moment, + # but if https_cert gets stricter, this check means something + fprint = util.get_fingerprint(self.https_cert) + return util.cert_matches(fprint, certificate) + + def _get_wc(self): + defuser, defpass = self.get_firmware_default_account_info() + wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) + wc.set_basic_credentials(defuser, defpass) + wc.set_header('Content-Type', 'application/json') + authmode = 0 + if not self.trieddefault: + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status == 403: + self.trieddefault = True + chgurl = None + rsp = json.loads(rsp) + currerr = rsp.get('error', {}) + ecode = currerr.get('code', None) + if ecode.endswith('PasswordChangeRequired'): + for einfo in currerr.get('@Message.ExtendedInfo', []): + if einfo.get('MessageId', None).endswith('PasswordChangeRequired'): + for msgarg in einfo.get('MessageArgs'): + chgurl = msgarg + break + if chgurl: + if self.targpass == defpass: + raise Exception("Must specify a non-default password to onboard this BMC") + wc.set_header('If-Match', '*') + cpr = wc.grab_json_response_with_status(chgurl, {'Password': self.targpass}, method='PATCH') + if cpr[1] >= 200 and cpr[1] < 300: + self.curruser = defuser + self.currpass = self.targpass + wc.set_basic_credentials(self.curruser, self.currpass) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + tries = 10 + while status >= 300 and tries: + eventlet.sleep(1) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + return wc + + if status > 400: + self.trieddefault = True + if status == 401: + wc.set_basic_credentials(self.DEFAULT_USER, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status == 200: # Default user still, but targpass + self.currpass = self.targpass + self.curruser = defuser + return wc + elif self.targuser != defuser: + wc.set_basic_credentials(self.targuser, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + raise Exception("Target BMC does not recognize firmware default credentials nor the confluent stored credential") + else: + self.curruser = defuser + self.currpass = defpass + return wc + if self.curruser: + wc.set_basic_credentials(self.curruser, self.currpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + return None + return wc + wc.set_basic_credentials(self.targuser, self.targpass) + rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if status != 200: + return None + self.curruser = self.targuser + self.currpass = self.targpass + return wc + + def config(self, nodename): + self.nodename = nodename + creds = self.configmanager.get_node_attributes( + nodename, ['secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword', + 'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'], + True) + cd = creds.get(nodename, {}) + defuser, defpass = self.get_firmware_default_account_info() + user, passwd, _ = self.get_node_credentials( + nodename, creds, defuser, defpass) + user = util.stringify(user) + passwd = util.stringify(passwd) + self.targuser = user + self.targpass = passwd + wc = self._get_wc() + srvroot, status = wc.grab_json_response_with_status('/redfish/v1/') + curruserinfo = {} + authupdate = {} + wc.set_header('Content-Type', 'application/json') + if user != self.curruser: + authupdate['UserName'] = user + if passwd != self.currpass: + authupdate['Password'] = passwd + if authupdate: + targaccturl = None + asrv = srvroot.get('AccountService', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(asrv) + accts = rsp.get('Accounts', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(accts) + accts = rsp.get('Members', []) + for accturl in accts: + accturl = accturl.get('@odata.id', '') + if accturl: + rsp, status = wc.grab_json_response_with_status(accturl) + if rsp.get('UserName', None) == self.curruser: + targaccturl = accturl + break + else: + raise Exception("Unable to identify Account URL to modify on this BMC") + rsp, status = wc.grab_json_response_with_status(targaccturl, authupdate, method='PATCH') + if status >= 300: + raise Exception("Failed attempting to update credentials on BMC") + wc.set_basic_credentials(user, passwd) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + tries = 10 + while tries and status >= 300: + tries -= 1 + eventlet.sleep(1.0) + _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + if ('hardwaremanagement.manager' in cd and + cd['hardwaremanagement.manager']['value'] and + not cd['hardwaremanagement.manager']['value'].startswith( + 'fe80::')): + newip = cd['hardwaremanagement.manager']['value'] + newip = newip.split('/', 1)[0] + newipinfo = getaddrinfo(newip, 0)[0] + newip = newipinfo[-1][0] + if ':' in newip: + raise exc.NotImplementedException('IPv6 remote config TODO') + mgrs = srvroot['Managers']['@odata.id'] + rsp = wc.grab_json_response(mgrs) + if len(rsp['Members']) != 1: + raise Exception("Can not handle multiple Managers") + mgrurl = rsp['Members'][0]['@odata.id'] + mginfo = wc.grab_json_response(mgrurl) + hifurls = get_host_interface_urls(wc, mginfo) + mgtnicinfo = mginfo['EthernetInterfaces']['@odata.id'] + mgtnicinfo = wc.grab_json_response(mgtnicinfo) + mgtnics = [x['@odata.id'] for x in mgtnicinfo.get('Members', [])] + actualnics = [] + for candnic in mgtnics: + if candnic in hifurls: + continue + actualnics.append(candnic) + if len(actualnics) != 1: + raise Exception("Multi-interface BMCs are not supported currently") + currnet = wc.grab_json_response(actualnics[0]) + netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip) + newconfig = { + "Address": newip, + "SubnetMask": netutil.cidr_to_mask(netconfig['prefix']), + } + newgw = netconfig['ipv4_gateway'] + if newgw: + newconfig['Gateway'] = newgw + else: + newconfig['Gateway'] = newip # required property, set to self just to have a value + for net in currnet.get("IPv4Addresses", []): + if net["Address"] == newip and net["SubnetMask"] == newconfig['SubnetMask'] and (not newgw or newconfig['Gateway'] == newgw): + break + else: + wc.set_header('If-Match', '*') + rsp, status = wc.grab_json_response_with_status(actualnics[0], {'IPv4StaticAddresses': [newconfig]}, method='PATCH') + elif self.ipaddr.startswith('fe80::'): + self.configmanager.set_node_attributes( + {nodename: {'hardwaremanagement.manager': self.ipaddr}}) + else: + raise exc.TargetEndpointUnreachable( + 'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)') + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]] } + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 34b4f6d0..45d1e1f3 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -60,6 +60,7 @@ def active_scan(handler, protocol=None): known_peers = set([]) for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::service:affluent']): for addr in scanned['addresses']: + addr = addr[0:1] + addr[2:] if addr in known_peers: break hwaddr = neighutil.get_hwaddr(addr[0]) @@ -79,13 +80,20 @@ def scan(services, target=None): def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler): - if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']: - peerbymacaddress[mac]['addresses'].append(peer) + if mac in peerbymacaddress: + normpeer = peer[0:1] + peer[2:] + for currpeer in peerbymacaddress[mac]['addresses']: + currnormpeer = currpeer[0:1] + peer[2:] + if currnormpeer == normpeer: + break + else: + peerbymacaddress[mac]['addresses'].append(peer) else: peerdata = { 'hwaddr': mac, 'addresses': [peer], } + targurl = None for headline in rsp[1:]: if not headline: continue @@ -105,13 +113,20 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha if not value.endswith('/redfish/v1/'): return elif header == 'LOCATION': - if not value.endswith('/DeviceDescription.json'): + if '/eth' in value and value.endswith('.xml'): + targurl = '/redfish/v1/' + targtype = 'megarac-bmc' + continue # MegaRAC redfish + elif value.endswith('/DeviceDescription.json'): + targurl = '/DeviceDescription.json' + targtype = 'megarac-bmc' + else: return - if handler: - eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer) + if handler and targurl: + eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype) -def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer): - retdata = check_fish(('/DeviceDescription.json', peerdata)) +def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer, targurl, targtype): + retdata = check_fish((targurl, peerdata, targtype)) if retdata: known_peers.add(peer) newmacs.add(mac) @@ -325,7 +340,7 @@ def _find_service(service, target): host = '[{0}]'.format(host) msg = smsg.format(host, service) if not isinstance(msg, bytes): - msg = msg.encode('utf8') + msg = msg.encode('utf8') net6.sendto(msg, addr[4]) else: net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) @@ -413,7 +428,11 @@ def _find_service(service, target): if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()): continue if '/DeviceDescription.json' in peerdata[nid]['urls']: - pooltargs.append(('/DeviceDescription.json', peerdata[nid])) + pooltargs.append(('/DeviceDescription.json', peerdata[nid], 'lenovo-xcc')) + else: + for targurl in peerdata[nid]['urls']: + if '/eth' in targurl and targurl.endswith('.xml'): + pooltargs.append(('/redfish/v1/', peerdata[nid], 'megarac-bmc')) # For now, don't interrogate generic redfish bmcs # This is due to a need to deduplicate from some supported SLP # targets (IMM, TSM, others) @@ -428,7 +447,7 @@ def _find_service(service, target): def check_fish(urldata, port=443, verifycallback=None): if not verifycallback: verifycallback = lambda x: True - url, data = urldata + url, data, targtype = urldata try: wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5) peerinfo = wc.grab_json_response(url, headers={'Accept': 'application/json'}) @@ -457,7 +476,7 @@ def check_fish(urldata, port=443, verifycallback=None): peerinfo = wc.grab_json_response('/redfish/v1/') if url == '/redfish/v1/': if 'UUID' in peerinfo: - data['services'] = ['service:redfish-bmc'] + data['services'] = [targtype] data['uuid'] = peerinfo['UUID'].lower() return data return None @@ -476,7 +495,12 @@ def _parse_ssdp(peer, rsp, peerdata): if code == b'200': if nid in peerdata: peerdatum = peerdata[nid] - if peer not in peerdatum['addresses']: + normpeer = peer[0:1] + peer[2:] + for currpeer in peerdatum['addresses']: + currnormpeer = currpeer[0:1] + peer[2:] + if currnormpeer == normpeer: + break + else: peerdatum['addresses'].append(peer) else: peerdatum = { @@ -511,5 +535,7 @@ def _parse_ssdp(peer, rsp, peerdata): if __name__ == '__main__': def printit(rsp): - print(repr(rsp)) + pass # print(repr(rsp)) active_scan(printit) + + From cfb31a0d8dbc94df1e481c0c673d94e563eb1a03 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 10:00:22 -0400 Subject: [PATCH 255/319] Implement XCC3 discovery For XCC3, change to generic redfish onboarding mechanism. Extend the generic mechanism to be more specific in some ways that the XCC3 is pickier about. However, it's just reiteration of what should have already have been the case. --- confluent_server/confluent/discovery/core.py | 3 +- .../discovery/handlers/redfishbmc.py | 8 +- .../confluent/discovery/handlers/xcc3.py | 102 ++++++++++++++++++ 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 confluent_server/confluent/discovery/handlers/xcc3.py diff --git a/confluent_server/confluent/discovery/core.py b/confluent_server/confluent/discovery/core.py index b734cece..fd302f8b 100644 --- a/confluent_server/confluent/discovery/core.py +++ b/confluent_server/confluent/discovery/core.py @@ -74,6 +74,7 @@ import confluent.discovery.handlers.tsm as tsm import confluent.discovery.handlers.pxe as pxeh import confluent.discovery.handlers.smm as smm import confluent.discovery.handlers.xcc as xcc +import confluent.discovery.handlers.xcc3 as xcc3 import confluent.discovery.handlers.megarac as megarac import confluent.exceptions as exc import confluent.log as log @@ -114,7 +115,7 @@ nodehandlers = { 'service:lenovo-smm': smm, 'service:lenovo-smm2': smm, 'lenovo-xcc': xcc, - 'lenovo-xcc3': xcc, + 'lenovo-xcc3': xcc3, 'megarac-bmc': megarac, 'service:management-hardware.IBM:integrated-management-module2': imm, 'pxe-client': pxeh, diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py index eed401de..97629f36 100644 --- a/confluent_server/confluent/discovery/handlers/redfishbmc.py +++ b/confluent_server/confluent/discovery/handlers/redfishbmc.py @@ -80,6 +80,7 @@ class NodeHandler(generic.NodeHandler): wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) wc.set_basic_credentials(defuser, defpass) wc.set_header('Content-Type', 'application/json') + wc.set_header('Accept', 'application/json') authmode = 0 if not self.trieddefault: rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') @@ -114,7 +115,7 @@ class NodeHandler(generic.NodeHandler): if status > 400: self.trieddefault = True if status == 401: - wc.set_basic_credentials(self.DEFAULT_USER, self.targpass) + wc.set_basic_credentials(defuser, self.targpass) rsp, status = wc.grab_json_response_with_status('/redfish/v1/Managers') if status == 200: # Default user still, but targpass self.currpass = self.targpass @@ -236,7 +237,10 @@ class NodeHandler(generic.NodeHandler): break else: wc.set_header('If-Match', '*') - rsp, status = wc.grab_json_response_with_status(actualnics[0], {'IPv4StaticAddresses': [newconfig]}, method='PATCH') + rsp, status = wc.grab_json_response_with_status(actualnics[0], { + 'DHCPv4': {'DHCPEnabled': False}, + 'IPv4StaticAddresses': [newconfig]}, method='PATCH') + elif self.ipaddr.startswith('fe80::'): self.configmanager.set_node_attributes( {nodename: {'hardwaremanagement.manager': self.ipaddr}}) diff --git a/confluent_server/confluent/discovery/handlers/xcc3.py b/confluent_server/confluent/discovery/handlers/xcc3.py new file mode 100644 index 00000000..780de4fc --- /dev/null +++ b/confluent_server/confluent/discovery/handlers/xcc3.py @@ -0,0 +1,102 @@ +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.discovery.handlers.redfishbmc as redfishbmc +import eventlet.support.greendns +import confluent.util as util + +webclient = eventlet.import_patched('pyghmi.util.webclient') + + + +getaddrinfo = eventlet.support.greendns.getaddrinfo + + +class NodeHandler(redfishbmc.NodeHandler): + + def get_firmware_default_account_info(self): + return ('USERID', 'PASSW0RD') + + def scan(self): + ip, port = self.get_web_port_and_ip() + c = webclient.SecureHTTPConnection(ip, port, + verifycallback=self.validate_cert) + i = c.grab_json_response('/api/providers/logoninfo') + modelname = i.get('items', [{}])[0].get('machine_name', None) + if modelname: + self.info['modelname'] = modelname + for attrname in list(self.info.get('attributes', {})): + val = self.info['attributes'][attrname] + if '-uuid' == attrname[-5:] and len(val) == 32: + val = val.lower() + self.info['attributes'][attrname] = '-'.join([val[:8], val[8:12], val[12:16], val[16:20], val[20:]]) + attrs = self.info.get('attributes', {}) + room = attrs.get('room-id', None) + if room: + self.info['room'] = room + rack = attrs.get('rack-id', None) + if rack: + self.info['rack'] = rack + name = attrs.get('name', None) + if name: + self.info['hostname'] = name + unumber = attrs.get('lowest-u', None) + if unumber: + self.info['u'] = unumber + location = attrs.get('location', None) + if location: + self.info['location'] = location + mtm = attrs.get('enclosure-machinetype-model', None) + if mtm: + self.info['modelnumber'] = mtm.strip() + sn = attrs.get('enclosure-serial-number', None) + if sn: + self.info['serialnumber'] = sn.strip() + if attrs.get('enclosure-form-factor', None) == 'dense-computing': + encuuid = attrs.get('chassis-uuid', None) + if encuuid: + self.info['enclosure.uuid'] = fixuuid(encuuid) + slot = int(attrs.get('slot', 0)) + if slot != 0: + self.info['enclosure.bay'] = slot + + def validate_cert(self, certificate): + fprint = util.get_fingerprint(self.https_cert) + return util.cert_matches(fprint, certificate) + + +def remote_nodecfg(nodename, cfm): + cfg = cfm.get_node_attributes( + nodename, 'hardwaremanagement.manager') + ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get( + 'value', None) + ipaddr = ipaddr.split('/', 1)[0] + ipaddr = getaddrinfo(ipaddr, 0)[0][-1] + if not ipaddr: + raise Exception('Cannot remote configure a system without known ' + 'address') + info = {'addresses': [ipaddr]} + nh = NodeHandler(info, cfm) + nh.config(nodename) + + +if __name__ == '__main__': + import confluent.config.configmanager as cfm + c = cfm.ConfigManager(None) + import sys + info = {'addresses': [[sys.argv[1]]]} + print(repr(info)) + testr = NodeHandler(info, c) + testr.config(sys.argv[2]) + From 30c4d6b863e74cd42d974867c68a45c5873f3ecc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 11:07:50 -0400 Subject: [PATCH 256/319] Add IPMI enablement to generic Redfish handler If attributes indicate desire for IPMI, try to accomodate. --- .../discovery/handlers/redfishbmc.py | 102 +++++++++++++----- .../confluent/discovery/handlers/xcc.py | 2 +- .../confluent/discovery/handlers/xcc3.py | 1 + 3 files changed, 76 insertions(+), 29 deletions(-) diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py index 97629f36..d4e164d6 100644 --- a/confluent_server/confluent/discovery/handlers/redfishbmc.py +++ b/confluent_server/confluent/discovery/handlers/redfishbmc.py @@ -57,8 +57,28 @@ class NodeHandler(generic.NodeHandler): self.csrftok = None self.channel = None self.atdefault = True + self._srvroot = None + self._mgrinfo = None super(NodeHandler, self).__init__(info, configmanager) + def srvroot(self, wc): + if not self._srvroot: + srvroot, status = wc.grab_json_response_with_status('/redfish/v1/') + if status == 200: + self._srvroot = srvroot + return self._srvroot + + def mgrinfo(self, wc): + if not self._mgrinfo: + mgrs = self.srvroot(wc)['Managers']['@odata.id'] + rsp = wc.grab_json_response(mgrs) + if len(rsp['Members']) != 1: + raise Exception("Can not handle multiple Managers") + mgrurl = rsp['Members'][0]['@odata.id'] + self._mgrinfo = wc.grab_json_response(mgrurl) + return self._mgrinfo + + def get_firmware_default_account_info(self): raise Exception('This must be subclassed') @@ -75,6 +95,30 @@ class NodeHandler(generic.NodeHandler): fprint = util.get_fingerprint(self.https_cert) return util.cert_matches(fprint, certificate) + def enable_ipmi(self, wc): + npu = self.mgrinfo(wc).get( + 'NetworkProtocol', {}).get('@odata.id', None) + if not npu: + raise Exception('Cannot enable IPMI, no NetworkProtocol on BMC') + npi = wc.grab_json_response(npu) + if not npi.get('IPMI', {}).get('ProtocolEnabled'): + wc.set_header('If-Match', '*') + wc.grab_json_response_with_status( + npu, {'IPMI': {'ProtocolEnabled': True}}, method='PATCH') + acctinfo = wc.grab_json_response_with_status( + self.target_account_url(wc)) + acctinfo = acctinfo[0] + actypes = acctinfo['AccountTypes'] + candidates = acctinfo['AccountTypes@Redfish.AllowableValues'] + if 'IPMI' not in actypes and 'IPMI' in candidates: + actypes.append('IPMI') + acctupd = { + 'AccountTypes': actypes, + 'Password': self.currpass, + } + rsp = wc.grab_json_response_with_status( + self.target_account_url(wc), acctupd, method='PATCH') + def _get_wc(self): defuser, defpass = self.get_firmware_default_account_info() wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert) @@ -144,13 +188,33 @@ class NodeHandler(generic.NodeHandler): self.currpass = self.targpass return wc + def target_account_url(self, wc): + asrv = self.srvroot(wc).get('AccountService', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(asrv) + accts = rsp.get('Accounts', {}).get('@odata.id') + rsp, status = wc.grab_json_response_with_status(accts) + accts = rsp.get('Members', []) + for accturl in accts: + accturl = accturl.get('@odata.id', '') + if accturl: + rsp, status = wc.grab_json_response_with_status(accturl) + if rsp.get('UserName', None) == self.curruser: + targaccturl = accturl + break + else: + raise Exception("Unable to identify Account URL to modify on this BMC") + return targaccturl + def config(self, nodename): + mgrs = None self.nodename = nodename creds = self.configmanager.get_node_attributes( nodename, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword', - 'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'], - True) + 'hardwaremanagement.manager', + 'hardwaremanagement.method', + 'console.method'], + True) cd = creds.get(nodename, {}) defuser, defpass = self.get_firmware_default_account_info() user, passwd, _ = self.get_node_credentials( @@ -160,7 +224,6 @@ class NodeHandler(generic.NodeHandler): self.targuser = user self.targpass = passwd wc = self._get_wc() - srvroot, status = wc.grab_json_response_with_status('/redfish/v1/') curruserinfo = {} authupdate = {} wc.set_header('Content-Type', 'application/json') @@ -169,21 +232,7 @@ class NodeHandler(generic.NodeHandler): if passwd != self.currpass: authupdate['Password'] = passwd if authupdate: - targaccturl = None - asrv = srvroot.get('AccountService', {}).get('@odata.id') - rsp, status = wc.grab_json_response_with_status(asrv) - accts = rsp.get('Accounts', {}).get('@odata.id') - rsp, status = wc.grab_json_response_with_status(accts) - accts = rsp.get('Members', []) - for accturl in accts: - accturl = accturl.get('@odata.id', '') - if accturl: - rsp, status = wc.grab_json_response_with_status(accturl) - if rsp.get('UserName', None) == self.curruser: - targaccturl = accturl - break - else: - raise Exception("Unable to identify Account URL to modify on this BMC") + targaccturl = self.target_account_url(wc) rsp, status = wc.grab_json_response_with_status(targaccturl, authupdate, method='PATCH') if status >= 300: raise Exception("Failed attempting to update credentials on BMC") @@ -193,7 +242,11 @@ class NodeHandler(generic.NodeHandler): while tries and status >= 300: tries -= 1 eventlet.sleep(1.0) - _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') + _, status = wc.grab_json_response_with_status( + '/redfish/v1/Managers') + if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish' + or cd.get('console.method', {}).get('value', None) == 'ipmi'): + self.enable_ipmi(wc) if ('hardwaremanagement.manager' in cd and cd['hardwaremanagement.manager']['value'] and not cd['hardwaremanagement.manager']['value'].startswith( @@ -204,14 +257,8 @@ class NodeHandler(generic.NodeHandler): newip = newipinfo[-1][0] if ':' in newip: raise exc.NotImplementedException('IPv6 remote config TODO') - mgrs = srvroot['Managers']['@odata.id'] - rsp = wc.grab_json_response(mgrs) - if len(rsp['Members']) != 1: - raise Exception("Can not handle multiple Managers") - mgrurl = rsp['Members'][0]['@odata.id'] - mginfo = wc.grab_json_response(mgrurl) - hifurls = get_host_interface_urls(wc, mginfo) - mgtnicinfo = mginfo['EthernetInterfaces']['@odata.id'] + hifurls = get_host_interface_urls(wc, self.mgrinfo(wc)) + mgtnicinfo = self.mgrinfo(wc)['EthernetInterfaces']['@odata.id'] mgtnicinfo = wc.grab_json_response(mgtnicinfo) mgtnics = [x['@odata.id'] for x in mgtnicinfo.get('Members', [])] actualnics = [] @@ -240,7 +287,6 @@ class NodeHandler(generic.NodeHandler): rsp, status = wc.grab_json_response_with_status(actualnics[0], { 'DHCPv4': {'DHCPEnabled': False}, 'IPv4StaticAddresses': [newconfig]}, method='PATCH') - elif self.ipaddr.startswith('fe80::'): self.configmanager.set_node_attributes( {nodename: {'hardwaremanagement.manager': self.ipaddr}}) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index 49fe1e87..d4d67590 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -639,7 +639,7 @@ def remote_nodecfg(nodename, cfm): ipaddr = ipaddr.split('/', 1)[0] ipaddr = getaddrinfo(ipaddr, 0)[0][-1] if not ipaddr: - raise Excecption('Cannot remote configure a system without known ' + raise Exception('Cannot remote configure a system without known ' 'address') info = {'addresses': [ipaddr]} nh = NodeHandler(info, cfm) diff --git a/confluent_server/confluent/discovery/handlers/xcc3.py b/confluent_server/confluent/discovery/handlers/xcc3.py index 780de4fc..24974172 100644 --- a/confluent_server/confluent/discovery/handlers/xcc3.py +++ b/confluent_server/confluent/discovery/handlers/xcc3.py @@ -24,6 +24,7 @@ getaddrinfo = eventlet.support.greendns.getaddrinfo class NodeHandler(redfishbmc.NodeHandler): + devname = 'XCC' def get_firmware_default_account_info(self): return ('USERID', 'PASSW0RD') From fc5c1aa90f4551e634db7b063f1db5d945683eb2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 11:32:57 -0400 Subject: [PATCH 257/319] Fix SSDP error during merge --- confluent_server/confluent/discovery/protocols/ssdp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index 45d1e1f3..e2688d66 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -461,7 +461,7 @@ def check_fish(urldata, port=443, verifycallback=None): except KeyError: peerinfo['xcc-variant'] = '3' except IndexError: -+ return None + return None try: myuuid = peerinfo['node-uuid'].lower() if '-' not in myuuid: From 0fd07e842748d60cc67d125bdbf9540adae569bf Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 13:09:50 -0400 Subject: [PATCH 258/319] Fix race condition in SSDP snoop If an asynchronous handler is slow to enroll a target while another target causes an iteration of the snoop loop, the various modified structures had been discarded in the interim. Now persist the data structures iteration to iteration, using 'clear()' to empty them rather than getting brand new data structures each loop. --- .../confluent/discovery/protocols/ssdp.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index e2688d66..c7063838 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -116,10 +116,11 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha if '/eth' in value and value.endswith('.xml'): targurl = '/redfish/v1/' targtype = 'megarac-bmc' - continue # MegaRAC redfish + continue # MegaRAC redfish elif value.endswith('/DeviceDescription.json'): targurl = '/DeviceDescription.json' - targtype = 'megarac-bmc' + targtype = 'lenovo-xcc' + continue else: return if handler and targurl: @@ -179,11 +180,14 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None): net4.bind(('', 1900)) net6.bind(('', 1900)) peerbymacaddress = {} + newmacs = set([]) + deferrednotifies = [] + machandlers = {} while True: try: - newmacs = set([]) - deferrednotifies = [] - machandlers = {} + newmacs.clear() + deferrednotifies.clear() + machandlers.clear() r = select.select((net4, net6), (), (), 60) if r: r = r[0] From 0afc3eb03a89b16511a142351c1654cea6ada8a4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 13:12:54 -0400 Subject: [PATCH 259/319] Port SSDP improvements to SLP It may not apply, but better to be consistent. --- .../confluent/discovery/protocols/slp.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/slp.py b/confluent_server/confluent/discovery/protocols/slp.py index 30acb475..f1e334f3 100644 --- a/confluent_server/confluent/discovery/protocols/slp.py +++ b/confluent_server/confluent/discovery/protocols/slp.py @@ -471,10 +471,13 @@ def snoop(handler, protocol=None): # socket in use can occur when aliased ipv4 are encountered net.bind(('', 427)) net4.bind(('', 427)) - + newmacs = set([]) + known_peers = set([]) + peerbymacaddress = {} + deferpeers = [] while True: try: - newmacs = set([]) + newmacs.clear() r, _, _ = select.select((net, net4), (), (), 60) # clear known_peers and peerbymacaddress # to avoid stale info getting in... @@ -482,9 +485,9 @@ def snoop(handler, protocol=None): # addresses that come close together # calling code needs to understand deeper context, as snoop # will now yield dupe info over time - known_peers = set([]) - peerbymacaddress = {} - deferpeers = [] + known_peers.clear() + peerbymacaddress.clear() + deferpeers.clear() while r and len(deferpeers) < 256: for s in r: (rsp, peer) = s.recvfrom(9000) From e07e6ed152ea1396902199c2cc72993b3ac88706 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 14:56:23 -0400 Subject: [PATCH 260/319] Improve error handling in OpenBMC console --- confluent_server/confluent/plugins/console/openbmc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/console/openbmc.py b/confluent_server/confluent/plugins/console/openbmc.py index 8677ce17..519ca2d4 100644 --- a/confluent_server/confluent/plugins/console/openbmc.py +++ b/confluent_server/confluent/plugins/console/openbmc.py @@ -134,7 +134,12 @@ class TsmConsole(conapi.Console): kv = util.TLSCertVerifier( self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert wc = webclient.SecureHTTPConnection(self.origbmc, 443, verifycallback=kv) - rsp = wc.grab_json_response_with_status('/login', {'data': [self.username.decode('utf8'), self.password.decode("utf8")]}, headers={'Content-Type': 'application/json'}) + try: + rsp = wc.grab_json_response_with_status('/login', {'data': [self.username.decode('utf8'), self.password.decode("utf8")]}, headers={'Content-Type': 'application/json', 'Accept': 'application/json'}) + except Exception as e: + raise cexc.TargetEndpointUnreachable(str(e)) + if rsp[1] > 400: + raise cexc.TargetEndpointBadCredentials bmc = self.bmc if '%' in self.bmc: prefix = self.bmc.split('%')[0] From 8c1381633116dc389554aa1518fe5adbda33d53a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 5 Aug 2024 15:03:00 -0400 Subject: [PATCH 261/319] Fix fetch of model name for XCC3 systems --- confluent_server/confluent/discovery/handlers/xcc3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/confluent/discovery/handlers/xcc3.py b/confluent_server/confluent/discovery/handlers/xcc3.py index 24974172..050186e9 100644 --- a/confluent_server/confluent/discovery/handlers/xcc3.py +++ b/confluent_server/confluent/discovery/handlers/xcc3.py @@ -33,6 +33,7 @@ class NodeHandler(redfishbmc.NodeHandler): ip, port = self.get_web_port_and_ip() c = webclient.SecureHTTPConnection(ip, port, verifycallback=self.validate_cert) + c.set_header('Accept', 'application/json') i = c.grab_json_response('/api/providers/logoninfo') modelname = i.get('items', [{}])[0].get('machine_name', None) if modelname: From feaef79060850b9cc3c0d682c3c4737f227f4c28 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 6 Aug 2024 09:30:13 -0400 Subject: [PATCH 262/319] Successfully track credential currency across change --- confluent_server/confluent/discovery/handlers/redfishbmc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_server/confluent/discovery/handlers/redfishbmc.py b/confluent_server/confluent/discovery/handlers/redfishbmc.py index d4e164d6..7cf3f3d1 100644 --- a/confluent_server/confluent/discovery/handlers/redfishbmc.py +++ b/confluent_server/confluent/discovery/handlers/redfishbmc.py @@ -236,6 +236,8 @@ class NodeHandler(generic.NodeHandler): rsp, status = wc.grab_json_response_with_status(targaccturl, authupdate, method='PATCH') if status >= 300: raise Exception("Failed attempting to update credentials on BMC") + self.curruser = user + self.currpass = passwd wc.set_basic_credentials(user, passwd) _, status = wc.grab_json_response_with_status('/redfish/v1/Managers') tries = 10 From 21b1ac7690f301c9ef533868c8aae5e4a53dcf50 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 6 Aug 2024 09:34:46 -0400 Subject: [PATCH 263/319] Remove asyncore for jammy asyncore isn't needed before noble --- confluent_server/builddeb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_server/builddeb b/confluent_server/builddeb index 4071b5b1..a63d9d4f 100755 --- a/confluent_server/builddeb +++ b/confluent_server/builddeb @@ -35,6 +35,8 @@ cd deb_dist/!(*.orig)/ if [ "$OPKGNAME" = "confluent-server" ]; then if grep wheezy /etc/os-release; then sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control + elif grep jammy /etc/os-release; then + sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil/' debian/control else sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-pyasyncore/' debian/control fi From ef1f51ef988ac296b06ccf20c6ea3078a5c13bfc Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 6 Aug 2024 10:05:39 -0400 Subject: [PATCH 264/319] Wire in bmc config clear to redfish --- .../confluent/plugins/hardwaremanagement/redfish.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/redfish.py b/confluent_server/confluent/plugins/hardwaremanagement/redfish.py index f53cc393..2c2857de 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/redfish.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/redfish.py @@ -522,6 +522,8 @@ class IpmiHandler(object): return self.handle_sysconfig(True) elif self.element[1:3] == ['system', 'clear']: return self.handle_sysconfigclear() + elif self.element[1:3] == ['management_controller', 'clear']: + return self.handle_bmcconfigclear() elif self.element[1:3] == ['management_controller', 'licenses']: return self.handle_licenses() elif self.element[1:3] == ['management_controller', 'save_licenses']: @@ -1323,6 +1325,12 @@ class IpmiHandler(object): self.ipmicmd.set_bmc_configuration( self.inputdata.get_attributes(self.node)) + def handle_bmcconfigclear(self): + if 'read' == self.op: + raise exc.InvalidArgumentException( + 'Cannot read the "clear" resource') + self.ipmicmd.clear_bmc_configuration() + def handle_sysconfigclear(self): if 'read' == self.op: raise exc.InvalidArgumentException( From f2b9a4fa5d2bb5c5820c352b7fce641e79323c3d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 6 Aug 2024 12:25:21 -0400 Subject: [PATCH 265/319] Improve handling of ssh service being pre-hooked --- imgutil/imgutil | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index bc34af01..5b5de0b2 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -661,11 +661,20 @@ class DebHandler(OsHandler): run_constrainedx(fancy_chroot, (args, self.targpath)) args.cmd = ['apt-get', '-y', 'install'] + self.includepkgs run_constrainedx(fancy_chroot, (args, self.targpath)) - servicefile = os.path.join(self.targpath, 'usr/lib/systemd/system/ssh.service') + servicefile = os.path.join( + self.targpath, 'usr/lib/systemd/system/ssh.service') if os.path.exists(servicefile): - os.symlink('/usr/lib/systemd/system/ssh.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/ssh.service')) + targfile = os.path.join( + self.targpath, + 'etc/systemd/system/multi-user.target.wants/ssh.service') + if not os.path.exists(targfile): + os.symlink('/usr/lib/systemd/system/ssh.service', targfile) else: - os.symlink('/usr/lib/systemd/system/sshd.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/sshd.service')) + targfile = os.path.join( + self.targpath, + 'etc/systemd/system/multi-user.target.wants/sshd.service') + if not os.path.exists(targfile): + os.symlink('/usr/lib/systemd/system/sshd.service', targfile) From 7ab76004925ff82cceafaaaac158d71f9ba0f04b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 7 Aug 2024 07:56:11 -0400 Subject: [PATCH 266/319] Add cpio dependency for imgutil --- imgutil/confluent_imgutil.spec.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imgutil/confluent_imgutil.spec.tmpl b/imgutil/confluent_imgutil.spec.tmpl index 35ed4070..f7dea7a7 100644 --- a/imgutil/confluent_imgutil.spec.tmpl +++ b/imgutil/confluent_imgutil.spec.tmpl @@ -8,13 +8,13 @@ Source: confluent_imgutil.tar.xz BuildArch: noarch BuildRoot: /tmp/ %if "%{dist}" == ".el8" -Requires: squashfs-tools +Requires: squashfs-tools cpio %else %if "%{dist}" == ".el9" -Requires: squashfs-tools +Requires: squashfs-tools cpio %else %if "%{dist}" == ".el7" -Requires: squashfs-tools +Requires: squashfs-tools cpio %else Requires: squashfs %endif From 187fda4bb865b0f11c6333f6145e4a02f043527e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 7 Aug 2024 07:58:08 -0400 Subject: [PATCH 267/319] Add debootstrap dependency for imgutil --- imgutil/control.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/imgutil/control.tmpl b/imgutil/control.tmpl index a0fe21af..3bc8644c 100644 --- a/imgutil/control.tmpl +++ b/imgutil/control.tmpl @@ -5,4 +5,5 @@ Priority: optional Maintainer: Jarrod Johnson Description: Web frontend for confluent server Architecture: all +Depends: debootstrap From ca4955101d3ca912bf2c030ba77285212a2e149d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 7 Aug 2024 08:40:10 -0400 Subject: [PATCH 268/319] Improve "realness" of imgutil exec context Utilities that expected /dev/pts will now be satisfied, as a new /dev/pts is mounted. Further, systemd added a check in various utilities that was fouled by the previous method of appearing to have a root filesystem. Before, after chroot, we would bind mount / to itself, and this made things using /proc/mounts, /proc/self/mountinfo, df, mount, etc happy that there is a real looking root filesystem. However, by doing it after the chroot, systemd could statx on '..' and get a different mnt id than /. So it had to be done prior to the chroot. However it also had to be done before other mounts as bind mounting over it would block the submounts. This more closely imitates the initramfs behavior, where '/' starts life as a 'real' filesystem before being mounted up and switched into. This behavior was made to imitate the 'start_root.c' behavior as that seems to be more broadly successful. --- imgutil/imgutil | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/imgutil/imgutil b/imgutil/imgutil index 5b5de0b2..c5446069 100644 --- a/imgutil/imgutil +++ b/imgutil/imgutil @@ -963,7 +963,6 @@ def fancy_chroot(args, installroot): _mount('none', dstresolv, flags=MS_RDONLY|MS_REMOUNT|MS_BIND) os.chroot(installroot) os.chdir('/') - _mount('/', '/', flags=MS_BIND) # Make / manifest as a mounted filesystem in exec os.environ['PS1'] = '[\x1b[1m\x1b[4mIMGUTIL EXEC {0}\x1b[0m \\W]$ '.format(imgname) os.environ['CONFLUENT_IMGUTIL_MODE'] = 'exec' if oshandler: @@ -1004,7 +1003,13 @@ def build_root_backend(optargs): def _mount_constrained_fs(args, installroot): + # This is prepping for a chroot. + # For the target environment to be content with having a root + # filesystem, installroot must be a 'mount' entry of it's own, + # so bind mount to itself to satisfy + _mount(installroot, installroot, flags=MS_BIND) _mount('/dev', os.path.join(installroot, 'dev'), flags=MS_BIND|MS_RDONLY) + _mount('/dev/pts', os.path.join(installroot, 'dev/pts'), flags=MS_BIND|MS_RDONLY) _mount('proc', os.path.join(installroot, 'proc'), fstype='proc') _mount('sys', os.path.join(installroot, 'sys'), fstype='sysfs') _mount('runfs', os.path.join(installroot, 'run'), fstype='tmpfs') From 4453ba3b64bb41e6e37ae204115fdfdf6d4bc296 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 7 Aug 2024 09:20:34 -0400 Subject: [PATCH 269/319] Add cpio to confluent_server In order to do osdeploy processing, we must have cpio --- confluent_server/confluent_server.spec.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index bf81c969..04e63b21 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -14,13 +14,13 @@ Prefix: %{_prefix} BuildArch: noarch Requires: confluent_vtbufferd %if "%{dist}" == ".el7" -Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python-websocket-client python2-msgpack python-libarchive-c python-yaml python-monotonic +Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python-websocket-client python2-msgpack python-libarchive-c python-yaml python-monotonic cpio %else %if "%{dist}" == ".el8" -Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute +Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio %else %if "%{dist}" == ".el9" -Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute +Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio %else Requires: python3-dbm,python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodome >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dnspython, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-PyYAML openssl iproute %endif From 2fc4483bba0316c09931268b4daa19ba5e8e2042 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 8 Aug 2024 17:09:33 -0400 Subject: [PATCH 270/319] Backport SLP performance enhancement from async branch Same concept that could bog down async variant could be a slowdown for normal confluent. --- confluent_server/confluent/discovery/protocols/slp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_server/confluent/discovery/protocols/slp.py b/confluent_server/confluent/discovery/protocols/slp.py index f1e334f3..3ca7cd01 100644 --- a/confluent_server/confluent/discovery/protocols/slp.py +++ b/confluent_server/confluent/discovery/protocols/slp.py @@ -493,6 +493,8 @@ def snoop(handler, protocol=None): (rsp, peer) = s.recvfrom(9000) if peer in known_peers: continue + if peer in deferpeers: + continue mac = neighutil.get_hwaddr(peer[0]) if not mac: probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:] From 8fd39c36bb55b94f6fed48fb1163d4afdf713661 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 9 Aug 2024 07:55:42 -0400 Subject: [PATCH 271/319] Fix some mistakes in confignet --- confluent_osdeploy/common/profile/scripts/confignet | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 72462834..cb5684a8 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -86,7 +86,7 @@ def map_idx_to_name(): for line in subprocess.check_output(['ip', 'l']).decode('utf8').splitlines(): if line.startswith(' ') and 'link/' in line: typ = line.split()[0].split('/')[1] - devtype[prevdev] = typ if type != 'ether' else 'ethernet' + devtype[prevdev] = typ if typ != 'ether' else 'ethernet' if line.startswith(' '): continue idx, iface, rst = line.split(':', 2) @@ -413,7 +413,8 @@ class NetworkManager(object): cargs.append(arg) cargs.append(cmdargs[arg]) if u: - cmdargs['connection.interface-name'] = iname + cargs.append('connection.interface-name') + cargs.append(iname) subprocess.check_call(['nmcli', 'c', 'm', u] + cargs) subprocess.check_call(['nmcli', 'c', 'u', u]) else: From 6833cd9c5301ebfca28c3c68ee2f912b2ee0d643 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 9 Aug 2024 17:58:48 +0200 Subject: [PATCH 272/319] Add VLAN/PKEY support to confignet Introduce new node attribute net.vlan_id to support VLAN/PKEY configuration using confignet. --- .../common/profile/scripts/confignet | 27 ++++++++++++++----- .../confluent/config/attributes.py | 8 ++++-- confluent_server/confluent/netutil.py | 3 +++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index cb5684a8..7b0eddf9 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -405,20 +405,35 @@ class NetworkManager(object): else: cname = stgs.get('connection_name', None) iname = list(cfg['interfaces'])[0] - if not cname: - cname = iname + ctype = self.devtypes[iname] + if stgs.get('vlan_id', None): + vlan = stgs['vlan_id'] + if ctype == 'infiniband': + vlan = '0x{0}'.format(vlan) if not vlan.startswith('0x') else vlan + cmdargs['infiniband.parent'] = iname + cmdargs['infiniband.p-key'] = vlan + iname = '{0}.{1}'.format(iname, vlan[2:]) + cname = iname if not cname else cname + elif ctype == 'ethernet': + ctype = 'vlan' + cmdargs['vlan.parent'] = iname + cmdargs['vlan.id'] = vlan + iname = '{0}.{1}'.format(iname, vlan) + cname = iname if not cname else cname + else: + sys.stderr.write("Warning, unknown interface_name ({0}) device type ({1}) for VLAN/PKEY, skipping setup\n".format(iname, ctype)) + return + cname = iname if not cname else cname u = self.uuidbyname.get(cname, None) cargs = [] for arg in cmdargs: cargs.append(arg) cargs.append(cmdargs[arg]) if u: - cargs.append('connection.interface-name') - cargs.append(iname) - subprocess.check_call(['nmcli', 'c', 'm', u] + cargs) + subprocess.check_call(['nmcli', 'c', 'm', u, 'connection.interface-name', iname] + cargs) subprocess.check_call(['nmcli', 'c', 'u', u]) else: - subprocess.check_call(['nmcli', 'c', 'add', 'type', self.devtypes[iname], 'con-name', cname, 'connection.interface-name', iname] + cargs) + subprocess.check_call(['nmcli', 'c', 'add', 'type', ctype, 'con-name', cname, 'connection.interface-name', iname] + cargs) self.read_connections() u = self.uuidbyname.get(cname, None) if u: diff --git a/confluent_server/confluent/config/attributes.py b/confluent_server/confluent/config/attributes.py index 101ee03d..f926c962 100644 --- a/confluent_server/confluent/config/attributes.py +++ b/confluent_server/confluent/config/attributes.py @@ -469,9 +469,13 @@ node = { 'net.interface_names': { 'description': 'Interface name or comma delimited list of names to match for this interface. It is generally recommended ' 'to leave this blank unless needing to set up interfaces that are not on a common subnet with a confluent server, ' - 'as confluent servers provide autodetection for matching the correct network definition to an interface.' + 'as confluent servers provide autodetection for matching the correct network definition to an interface. ' 'This would be the default name per the deployed OS and can be a comma delimited list to denote members of ' - 'a team' + 'a team or a single interface for VLAN/PKEY connections.' + }, + 'net.vlan_id': { + 'description': 'Ethernet VLAN or InfiniBand PKEY to use for this connection. ' + 'Specify the parent device using net.interface_names.' }, 'net.ipv4_address': { 'description': 'When configuring static, use this address. If ' diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index 9bac92c2..c1a9210a 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -193,6 +193,9 @@ class NetManager(object): iname = attribs.get('interface_names', None) if iname: myattribs['interface_names'] = iname + vlanid = attribs.get('vlan_id', None) + if vlanid: + myattribs['vlan_id'] = vlanid teammod = attribs.get('team_mode', None) if teammod: myattribs['team_mode'] = teammod From 6943c2dc0f1f0cfa7b530d6d09ccbdd199764efb Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 9 Aug 2024 19:38:45 +0200 Subject: [PATCH 273/319] Make sure VLAN/PKEY connections are created last Needed for VLANs on bond connections etc. --- confluent_osdeploy/common/profile/scripts/confignet | 2 ++ 1 file changed, 2 insertions(+) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 7b0eddf9..20fcc8b8 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -516,6 +516,8 @@ if __name__ == '__main__': netname_to_interfaces['default']['interfaces'] -= netname_to_interfaces[netn]['interfaces'] if not netname_to_interfaces['default']['interfaces']: del netname_to_interfaces['default'] + # Make sure VLAN/PKEY connections are created last + netname_to_interfaces = dict(sorted(netname_to_interfaces.items(), key=lambda item: 'vlan_id' in item[1]['settings'])) rm_tmp_llas(tmpllas) if os.path.exists('/usr/sbin/netplan'): nm = NetplanManager(dc) From 005adec437dc631d5b3f9f7b38cc640336bdc636 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 9 Aug 2024 19:45:19 +0200 Subject: [PATCH 274/319] Add error handling for interface_names --- confluent_osdeploy/common/profile/scripts/confignet | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 20fcc8b8..562a8ca1 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -405,7 +405,10 @@ class NetworkManager(object): else: cname = stgs.get('connection_name', None) iname = list(cfg['interfaces'])[0] - ctype = self.devtypes[iname] + ctype = self.devtypes.get(iname, None) + if not ctype: + sys.stderr.write("Warning, no device found for interface_name ({0}), skipping setup\n".format(iname)) + return if stgs.get('vlan_id', None): vlan = stgs['vlan_id'] if ctype == 'infiniband': From 09611744258586eba22cb5fa19e16d5b6ca2759b Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Fri, 9 Aug 2024 19:55:42 +0200 Subject: [PATCH 275/319] Remove redundant code --- confluent_osdeploy/common/profile/scripts/confignet | 2 -- 1 file changed, 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 562a8ca1..650f4eb6 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -416,13 +416,11 @@ class NetworkManager(object): cmdargs['infiniband.parent'] = iname cmdargs['infiniband.p-key'] = vlan iname = '{0}.{1}'.format(iname, vlan[2:]) - cname = iname if not cname else cname elif ctype == 'ethernet': ctype = 'vlan' cmdargs['vlan.parent'] = iname cmdargs['vlan.id'] = vlan iname = '{0}.{1}'.format(iname, vlan) - cname = iname if not cname else cname else: sys.stderr.write("Warning, unknown interface_name ({0}) device type ({1}) for VLAN/PKEY, skipping setup\n".format(iname, ctype)) return From a6a1907611411f19092a7581fed1d5d15415cff9 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Tue, 13 Aug 2024 17:30:43 +0200 Subject: [PATCH 276/319] Do not overwrite the node SSH key with the last found public key Instead of overwriting the SSH public code for the node concatenate all found SSH keys together in one file. Signed-off-by: Adrian Reber --- confluent_server/confluent/sshutil.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/confluent_server/confluent/sshutil.py b/confluent_server/confluent/sshutil.py index cf17f37a..1f8960d8 100644 --- a/confluent_server/confluent/sshutil.py +++ b/confluent_server/confluent/sshutil.py @@ -213,15 +213,17 @@ def initialize_root_key(generate, automation=False): suffix = 'automationpubkey' else: suffix = 'rootpubkey' + keyname = '/var/lib/confluent/public/site/ssh/{0}.{1}'.format( + myname, suffix) for auth in authorized: - shutil.copy( - auth, - '/var/lib/confluent/public/site/ssh/{0}.{1}'.format( - myname, suffix)) - os.chmod('/var/lib/confluent/public/site/ssh/{0}.{1}'.format( - myname, suffix), 0o644) - os.chown('/var/lib/confluent/public/site/ssh/{0}.{1}'.format( - myname, suffix), neededuid, -1) + local_key = open(auth, 'r') + dest = open(keyname, 'a') + dest.write(local_key.read()) + local_key.close() + dest.close() + if os.path.exists(keyname): + os.chmod(keyname, 0o644) + os.chown(keyname, neededuid, -1) if alreadyexist: raise AlreadyExists() From 29d0e904876a249a6a3afdddc789b931b589e66e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 14 Aug 2024 11:26:51 -0400 Subject: [PATCH 277/319] Implement confluentdbutil 'merge' For now, implement 'skip', where conflicting nodes/groups are ignored in new input. --- confluent_server/bin/confluentdbutil | 15 ++- .../confluent/config/configmanager.py | 106 ++++++++++++------ 2 files changed, 79 insertions(+), 42 deletions(-) diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil index 25a5acf8..b7c1e5c7 100755 --- a/confluent_server/bin/confluentdbutil +++ b/confluent_server/bin/confluentdbutil @@ -30,7 +30,7 @@ import confluent.config.conf as conf import confluent.main as main argparser = optparse.OptionParser( - usage="Usage: %prog [options] [dump|restore] [path]") + usage="Usage: %prog [options] [dump|restore|merge] [path]") argparser.add_option('-p', '--password', help='Password to use to protect/unlock a protected dump') argparser.add_option('-i', '--interactivepassword', help='Prompt for password', @@ -51,13 +51,13 @@ argparser.add_option('-s', '--skipkeys', action='store_true', 'data is needed. keys do not change and as such ' 'they do not require incremental backup') (options, args) = argparser.parse_args() -if len(args) != 2 or args[0] not in ('dump', 'restore'): +if len(args) != 2 or args[0] not in ('dump', 'restore', 'merge'): argparser.print_help() sys.exit(1) dumpdir = args[1] -if args[0] == 'restore': +if args[0] in ('restore', 'merge'): pid = main.is_running() if pid is not None: print("Confluent is running, must shut down to restore db") @@ -69,9 +69,12 @@ if args[0] == 'restore': if options.interactivepassword: password = getpass.getpass('Enter password to restore backup: ') try: - cfm.init(True) - cfm.statelessmode = True - cfm.restore_db_from_directory(dumpdir, password) + stateless = args[0] == 'restore' + cfm.init(stateless) + cfm.statelessmode = stateless + cfm.restore_db_from_directory( + dumpdir, password, + merge="skip" if args[0] == 'merge' else False) cfm.statelessmode = False cfm.ConfigManager.wait_for_sync(True) if owner != 0: diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 6cbf4604..788c2d60 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -1903,7 +1903,7 @@ class ConfigManager(object): def add_group_attributes(self, attribmap): self.set_group_attributes(attribmap, autocreate=True) - def set_group_attributes(self, attribmap, autocreate=False): + def set_group_attributes(self, attribmap, autocreate=False, merge="replace", keydata=None, skipped=None): for group in attribmap: curr = attribmap[group] for attrib in curr: @@ -1924,11 +1924,11 @@ class ConfigManager(object): if cfgstreams: exec_on_followers('_rpc_set_group_attributes', self.tenant, attribmap, autocreate) - self._true_set_group_attributes(attribmap, autocreate) + self._true_set_group_attributes(attribmap, autocreate, merge=merge, keydata=keydata, skipped=skipped) - def _true_set_group_attributes(self, attribmap, autocreate=False): + def _true_set_group_attributes(self, attribmap, autocreate=False, merge="replace", keydata=None, skipped=None): changeset = {} - for group in attribmap: + for group in list(attribmap): if group == '': raise ValueError('"{0}" is not a valid group name'.format( group)) @@ -1941,6 +1941,11 @@ class ConfigManager(object): group)) if not autocreate and group not in self._cfgstore['nodegroups']: raise ValueError("{0} group does not exist".format(group)) + if merge == 'skip' and group in self._cfgstore['nodegroups']: + if skipped is not None: + skipped.append(group) + del attribmap[group] + continue for attr in list(attribmap[group]): # first do a pass to normalize out any aliased attribute names if attr in _attraliases: @@ -2015,6 +2020,9 @@ class ConfigManager(object): newdict = {'value': attribmap[group][attr]} else: newdict = attribmap[group][attr] + if keydata and attr.startswith('secret.') and 'cryptvalue' in newdict: + newdict['value'] = decrypt_value(newdict['cryptvalue'], keydata['cryptkey'], keydata['integritykey']) + del newdict['cryptvalue'] if 'value' in newdict and attr.startswith("secret."): newdict['cryptvalue'] = crypt_value(newdict['value']) del newdict['value'] @@ -2349,7 +2357,7 @@ class ConfigManager(object): - def set_node_attributes(self, attribmap, autocreate=False): + def set_node_attributes(self, attribmap, autocreate=False, merge="replace", keydata=None, skipped=None): for node in attribmap: curr = attribmap[node] for attrib in curr: @@ -2370,9 +2378,9 @@ class ConfigManager(object): if cfgstreams: exec_on_followers('_rpc_set_node_attributes', self.tenant, attribmap, autocreate) - self._true_set_node_attributes(attribmap, autocreate) + self._true_set_node_attributes(attribmap, autocreate, merge, keydata, skipped) - def _true_set_node_attributes(self, attribmap, autocreate): + def _true_set_node_attributes(self, attribmap, autocreate, merge="replace", keydata=None, skipped=None): # TODO(jbjohnso): multi mgr support, here if we have peers, # pickle the arguments and fire them off in eventlet # flows to peers, all should have the same result @@ -2380,7 +2388,7 @@ class ConfigManager(object): changeset = {} # first do a sanity check of the input upfront # this mitigates risk of arguments being partially applied - for node in attribmap: + for node in list(attribmap): node = confluent.util.stringify(node) if node == '': raise ValueError('"{0}" is not a valid node name'.format(node)) @@ -2393,6 +2401,11 @@ class ConfigManager(object): '"{0}" is not a valid node name'.format(node)) if autocreate is False and node not in self._cfgstore['nodes']: raise ValueError("node {0} does not exist".format(node)) + if merge == "skip" and node in self._cfgstore['nodes']: + del attribmap[node] + if skipped is not None: + skipped.append(node) + continue if 'groups' not in attribmap[node] and node not in self._cfgstore['nodes']: attribmap[node]['groups'] = [] for attrname in list(attribmap[node]): @@ -2463,6 +2476,9 @@ class ConfigManager(object): # add check here, skip None attributes if newdict is None: continue + if keydata and attrname.startswith('secret.') and 'cryptvalue' in newdict: + newdict['value'] = decrypt_value(newdict['cryptvalue'], keydata['cryptkey'], keydata['integritykey']) + del newdict['cryptvalue'] if 'value' in newdict and attrname.startswith("secret."): newdict['cryptvalue'] = crypt_value(newdict['value']) del newdict['value'] @@ -2503,14 +2519,14 @@ class ConfigManager(object): self._bg_sync_to_file() #TODO: wait for synchronization to suceed/fail??) - def _load_from_json(self, jsondata, sync=True): + def _load_from_json(self, jsondata, sync=True, merge=False, keydata=None): self.inrestore = True try: - self._load_from_json_backend(jsondata, sync=True) + self._load_from_json_backend(jsondata, sync=True, merge=merge, keydata=keydata) finally: self.inrestore = False - def _load_from_json_backend(self, jsondata, sync=True): + def _load_from_json_backend(self, jsondata, sync=True, merge=False, keydata=None): """Load fresh configuration data from jsondata :param jsondata: String of jsondata @@ -2563,20 +2579,27 @@ class ConfigManager(object): pass # Now we have to iterate through each fixed up element, using the # set attribute to flesh out inheritence and expressions - _cfgstore['main']['idmap'] = {} + if (not merge) or _cfgstore.get('main', {}).get('idmap', None) is None: + _cfgstore['main']['idmap'] = {} + attribmerge = merge if merge else "replace" for confarea in _config_areas: - self._cfgstore[confarea] = {} + if not merge or confarea not in self._cfgstore: + self._cfgstore[confarea] = {} if confarea not in tmpconfig: continue if confarea == 'nodes': - self.set_node_attributes(tmpconfig[confarea], True) + self.set_node_attributes(tmpconfig[confarea], True, merge=attribmerge, keydata=keydata) elif confarea == 'nodegroups': - self.set_group_attributes(tmpconfig[confarea], True) + self.set_group_attributes(tmpconfig[confarea], True, merge=attribmerge, keydata=keydata) elif confarea == 'usergroups': + if merge: + continue for usergroup in tmpconfig[confarea]: role = tmpconfig[confarea][usergroup].get('role', 'Administrator') self.create_usergroup(usergroup, role=role) elif confarea == 'users': + if merge: + continue for user in tmpconfig[confarea]: ucfg = tmpconfig[confarea][user] uid = ucfg.get('id', None) @@ -2876,7 +2899,7 @@ def _restore_keys(jsond, password, newpassword=None, sync=True): newpassword = keyfile.read() set_global('master_privacy_key', _format_key(cryptkey, password=newpassword), sync) - if integritykey: + if integritykey: set_global('master_integrity_key', _format_key(integritykey, password=newpassword), sync) _masterkey = cryptkey @@ -2911,35 +2934,46 @@ def _dump_keys(password, dojson=True): return keydata -def restore_db_from_directory(location, password): +def restore_db_from_directory(location, password, merge=False): + kdd = None try: with open(os.path.join(location, 'keys.json'), 'r') as cfgfile: keydata = cfgfile.read() - json.loads(keydata) - _restore_keys(keydata, password) + kdd = json.loads(keydata) + if merge: + if 'cryptkey' in kdd: + kdd['cryptkey'] = _parse_key(kdd['cryptkey'], password) + if 'integritykey' in kdd: + kdd['integritykey'] = _parse_key(kdd['integritykey'], password) + else: + kdd['integritykey'] = None # GCM + else: + kdd = None + _restore_keys(keydata, password) except IOError as e: if e.errno == 2: raise Exception("Cannot restore without keys, this may be a " "redacted dump") - try: - moreglobals = json.load(open(os.path.join(location, 'globals.json'))) - for globvar in moreglobals: - set_global(globvar, moreglobals[globvar]) - except IOError as e: - if e.errno != 2: - raise - try: - collective = json.load(open(os.path.join(location, 'collective.json'))) - _cfgstore['collective'] = {} - for coll in collective: - add_collective_member(coll, collective[coll]['address'], - collective[coll]['fingerprint']) - except IOError as e: - if e.errno != 2: - raise + if not merge: + try: + moreglobals = json.load(open(os.path.join(location, 'globals.json'))) + for globvar in moreglobals: + set_global(globvar, moreglobals[globvar]) + except IOError as e: + if e.errno != 2: + raise + try: + collective = json.load(open(os.path.join(location, 'collective.json'))) + _cfgstore['collective'] = {} + for coll in collective: + add_collective_member(coll, collective[coll]['address'], + collective[coll]['fingerprint']) + except IOError as e: + if e.errno != 2: + raise with open(os.path.join(location, 'main.json'), 'r') as cfgfile: cfgdata = cfgfile.read() - ConfigManager(tenant=None)._load_from_json(cfgdata) + ConfigManager(tenant=None)._load_from_json(cfgdata, merge=merge, keydata=kdd) ConfigManager.wait_for_sync(True) From 28b88bdb12d78f16a29549a3ab4f2d914252c434 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 14 Aug 2024 11:40:11 -0400 Subject: [PATCH 278/319] Add reporting of skipped nodes in a 'skip' merge --- confluent_server/bin/confluentdbutil | 16 +++++++++++++--- .../confluent/config/configmanager.py | 16 +++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil index b7c1e5c7..e74c2ab4 100755 --- a/confluent_server/bin/confluentdbutil +++ b/confluent_server/bin/confluentdbutil @@ -1,7 +1,7 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2017 Lenovo +# Copyright 2017,2024 Lenovo # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -72,9 +72,19 @@ if args[0] in ('restore', 'merge'): stateless = args[0] == 'restore' cfm.init(stateless) cfm.statelessmode = stateless + skipped = {'nodes': [], 'nodegroups': []} cfm.restore_db_from_directory( dumpdir, password, - merge="skip" if args[0] == 'merge' else False) + merge="skip" if args[0] == 'merge' else False, skipped=skipped) + if skipped['nodes']: + skippedn = ','.join(skipped['nodes']) + print('The following nodes were skipped during merge: ' + '{}'.format(skippedn)) + if skipped['nodegroups']: + skippedn = ','.join(skipped['nodegroups']) + print('The following node groups were skipped during merge: ' + '{}'.format(skippedn)) + cfm.statelessmode = False cfm.ConfigManager.wait_for_sync(True) if owner != 0: diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 788c2d60..7702b97d 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -2519,19 +2519,21 @@ class ConfigManager(object): self._bg_sync_to_file() #TODO: wait for synchronization to suceed/fail??) - def _load_from_json(self, jsondata, sync=True, merge=False, keydata=None): + def _load_from_json(self, jsondata, sync=True, merge=False, keydata=None, skipped=None): self.inrestore = True try: - self._load_from_json_backend(jsondata, sync=True, merge=merge, keydata=keydata) + self._load_from_json_backend(jsondata, sync=True, merge=merge, keydata=keydata, skipped=skipped) finally: self.inrestore = False - def _load_from_json_backend(self, jsondata, sync=True, merge=False, keydata=None): + def _load_from_json_backend(self, jsondata, sync=True, merge=False, keydata=None, skipped=None): """Load fresh configuration data from jsondata :param jsondata: String of jsondata :return: """ + if not skipped: + skipped = {'nodes': None, 'nodegroups': None} dumpdata = json.loads(jsondata) tmpconfig = {} for confarea in _config_areas: @@ -2588,9 +2590,9 @@ class ConfigManager(object): if confarea not in tmpconfig: continue if confarea == 'nodes': - self.set_node_attributes(tmpconfig[confarea], True, merge=attribmerge, keydata=keydata) + self.set_node_attributes(tmpconfig[confarea], True, merge=attribmerge, keydata=keydata, skipped=skipped['nodes']) elif confarea == 'nodegroups': - self.set_group_attributes(tmpconfig[confarea], True, merge=attribmerge, keydata=keydata) + self.set_group_attributes(tmpconfig[confarea], True, merge=attribmerge, keydata=keydata, skipped=skipped['nodegroups']) elif confarea == 'usergroups': if merge: continue @@ -2934,7 +2936,7 @@ def _dump_keys(password, dojson=True): return keydata -def restore_db_from_directory(location, password, merge=False): +def restore_db_from_directory(location, password, merge=False, skipped=None): kdd = None try: with open(os.path.join(location, 'keys.json'), 'r') as cfgfile: @@ -2973,7 +2975,7 @@ def restore_db_from_directory(location, password, merge=False): raise with open(os.path.join(location, 'main.json'), 'r') as cfgfile: cfgdata = cfgfile.read() - ConfigManager(tenant=None)._load_from_json(cfgdata, merge=merge, keydata=kdd) + ConfigManager(tenant=None)._load_from_json(cfgdata, merge=merge, keydata=kdd, skipped=skipped) ConfigManager.wait_for_sync(True) From 82e0d9c434482688b9278178cf741db69ced07bd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 14 Aug 2024 16:08:02 -0400 Subject: [PATCH 279/319] Rework ssh key init to reset key and use context management --- confluent_server/confluent/sshutil.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/sshutil.py b/confluent_server/confluent/sshutil.py index 1f8960d8..40512648 100644 --- a/confluent_server/confluent/sshutil.py +++ b/confluent_server/confluent/sshutil.py @@ -215,12 +215,13 @@ def initialize_root_key(generate, automation=False): suffix = 'rootpubkey' keyname = '/var/lib/confluent/public/site/ssh/{0}.{1}'.format( myname, suffix) + if authorized: + with open(keyname, 'w'): + pass for auth in authorized: - local_key = open(auth, 'r') - dest = open(keyname, 'a') - dest.write(local_key.read()) - local_key.close() - dest.close() + with open(auth, 'r') as local_key: + with open(keyname, 'a') as dest: + dest.write(local_key.read()) if os.path.exists(keyname): os.chmod(keyname, 0o644) os.chown(keyname, neededuid, -1) From 1a40842f0611a5890893c0278836295be3043790 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 15 Aug 2024 10:54:52 -0400 Subject: [PATCH 280/319] Fix osdeploy updateboot to find multiple grub.cfg --- confluent_server/confluent/osimage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/osimage.py b/confluent_server/confluent/osimage.py index 5ff28d16..387922fa 100644 --- a/confluent_server/confluent/osimage.py +++ b/confluent_server/confluent/osimage.py @@ -155,11 +155,12 @@ def update_boot_esxi(profiledir, profile, label): def find_glob(loc, fileglob): + grubcfgs = [] for cdir, _, fs in os.walk(loc): for f in fs: if fnmatch(f, fileglob): - return [os.path.join(cdir, f)] - return None + grubcfgs.append(os.path.join(cdir, f)) + return grubcfgs def update_boot_linux(profiledir, profile, label): From d82a98285751ddb0550e782d41f9ef944ce58239 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 16 Aug 2024 10:52:00 -0400 Subject: [PATCH 281/319] Have affluent 'power' status actually at least reaching the service --- .../plugins/hardwaremanagement/affluent.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/affluent.py b/confluent_server/confluent/plugins/hardwaremanagement/affluent.py index ea169b3b..8946ae0e 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/affluent.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/affluent.py @@ -45,6 +45,13 @@ class WebClient(object): 'target certificate fingerprint and ' 'pubkeys.tls_hardwaremanager attribute')) return {} + except (socket.gaierror, socket.herror, TimeoutError) as e: + results.put(msg.ConfluentTargetTimeout(e.strerror)) + return {} + except Exception as e: + results.put(msg.ConfluentNodeError(self.node, + repr(e))) + return {} if status == 401: results.put(msg.ConfluentTargetInvalidCredentials(self.node, 'Unable to authenticate')) return {} @@ -115,9 +122,7 @@ def retrieve(nodes, element, configmanager, inputdata): results = queue.LightQueue() workers = set([]) if element == ['power', 'state']: - for node in nodes: - yield msg.PowerState(node=node, state='on') - return + _run_method(retrieve_power, workers, results, configmanager, nodes, element) elif element == ['health', 'hardware']: _run_method(retrieve_health, workers, results, configmanager, nodes, element) elif element[:3] == ['inventory', 'hardware', 'all']: @@ -188,9 +193,15 @@ def retrieve_sensors(configmanager, creds, node, results, element): +def retrieve_power(configmanager, creds, node, results, element): + wc = WebClient(node, configmanager, creds) + hinfo = wc.fetch('/affluent/health', results) + if hinfo: + results.put(msg.PowerState(node=node, state='on')) + def retrieve_health(configmanager, creds, node, results, element): wc = WebClient(node, configmanager, creds) - hinfo = wc.fetch('/affluent/health', results) + hinfo = wc.fetch('/affluent/health', results) if hinfo: results.put(msg.HealthSummary(hinfo.get('health', 'unknown'), name=node)) results.put(msg.SensorReadings(hinfo.get('sensors', []), name=node)) From fb10221e1bc51ffad695598b28bf558145d0c199 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 16 Aug 2024 11:26:52 -0400 Subject: [PATCH 282/319] Amend affluent error handling Be more consistent and informative --- .../confluent/plugins/hardwaremanagement/affluent.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/affluent.py b/confluent_server/confluent/plugins/hardwaremanagement/affluent.py index 8946ae0e..ee10bd7d 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/affluent.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/affluent.py @@ -46,7 +46,13 @@ class WebClient(object): 'pubkeys.tls_hardwaremanager attribute')) return {} except (socket.gaierror, socket.herror, TimeoutError) as e: - results.put(msg.ConfluentTargetTimeout(e.strerror)) + results.put(msg.ConfluentTargetTimeout(self.node, str(e))) + return {} + except OSError as e: + if e.errno == 113: + results.put(msg.ConfluentTargetTimeout(self.node)) + else: + results.put(msg.ConfluentTargetTimeout(self.node), str(e)) return {} except Exception as e: results.put(msg.ConfluentNodeError(self.node, From 4640cb194fa5ea43dd037edd54988f2de4c45082 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 19 Aug 2024 13:28:49 -0400 Subject: [PATCH 283/319] Provide workaround for XCC refusal to rename the initial account --- .../confluent/discovery/handlers/xcc.py | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index d4d67590..ef009b6d 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -408,6 +408,34 @@ class NodeHandler(immhandler.NodeHandler): if user['users_user_name'] == '': return user['users_user_id'] + def create_tmp_account(self, wc): + rsp, status = wc.grab_json_response_with_status('/redfish/v1/AccountService/Accounts') + if status != 200: + raise Exception("Unable to list current accounts") + usednames = set([]) + tmpnam = '6pmu0ezczzcp' + tpass = base64.b64encode(os.urandom(9)).decode() + 'Iw47$' + ntpass = base64.b64encode(os.urandom(9)).decode() + 'Iw47$' + for acct in rsp.get("Members", []): + url = acct.get("@odata.id", None) + if url: + uinfo = wc.grab_json_response(url) + usednames.add(uinfo.get('UserName', None)) + if tmpnam in usednames: + raise Exception("Tmp account already exists") + rsp, status = wc.grab_json_response_with_status( + '/redfish/v1/AccountService/Accounts', + {'UserName': tmpnam, 'Password': tpass, 'RoleId': 'Administrator'}) + if status >= 300: + raise Exception("Failure creating tmp account: " + repr(rsp)) + tmpurl = rsp['@odata.id'] + wc.set_basic_credentials(tmpnam, tpass) + rsp, status = wc.grab_json_response_with_status( + tmpurl, {'Password': ntpass}, method='PATCH') + wc.set_basic_credentials(tmpnam, ntpass) + return tmpurl + + def _setup_xcc_account(self, username, passwd, wc): userinfo = wc.grab_json_response('/api/dataset/imm_users') uid = None @@ -442,16 +470,29 @@ class NodeHandler(immhandler.NodeHandler): wc.grab_json_response('/api/providers/logout') wc.set_basic_credentials(self._currcreds[0], self._currcreds[1]) status = 503 + tries = 2 + tmpaccount = None while status != 200: + tries -= 1 rsp, status = wc.grab_json_response_with_status( '/redfish/v1/AccountService/Accounts/{0}'.format(uid), {'UserName': username}, method='PATCH') if status != 200: rsp = json.loads(rsp) if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'): - eventlet.sleep(4) + if tries: + eventlet.sleep(4) + elif tmpaccount: + wc.grab_json_response_with_status(tmpaccount, method='DELETE') + raise Exception('Failed renaming main account') + else: + tmpaccount = self.create_tmp_account(wc) + tries = 8 else: break + if tmpaccount: + wc.set_basic_credentials(username, passwd) + wc.grab_json_response_with_status(tmpaccount, method='DELETE') self.tmppasswd = None self._currcreds = (username, passwd) return From dd2119c6d950e2d2528c3e2521bfb397abe165f9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 19 Aug 2024 16:26:48 -0400 Subject: [PATCH 284/319] Ignore very old ssh key file --- confluent_osdeploy/common/profile/scripts/setupssh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/confluent_osdeploy/common/profile/scripts/setupssh b/confluent_osdeploy/common/profile/scripts/setupssh index 3fdf0ef5..eb989bb7 100644 --- a/confluent_osdeploy/common/profile/scripts/setupssh +++ b/confluent_osdeploy/common/profile/scripts/setupssh @@ -3,6 +3,9 @@ [ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient [ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient for pubkey in /etc/ssh/ssh_host*key.pub; do + if [ "$pubkey" = /etc/ssh/ssh_host_key.pub ]; then + continue + fi certfile=${pubkey/.pub/-cert.pub} rm $certfile confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile From cbd457b464538b3b8015d26ccf94a10188494c0f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 21 Aug 2024 09:12:18 -0400 Subject: [PATCH 285/319] Cancel the recvr task on close This avoids stail recvr from sending duplicate data. --- confluent_server/confluent/plugins/console/openbmc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/console/openbmc.py b/confluent_server/confluent/plugins/console/openbmc.py index 519ca2d4..3ff08e5a 100644 --- a/confluent_server/confluent/plugins/console/openbmc.py +++ b/confluent_server/confluent/plugins/console/openbmc.py @@ -119,6 +119,7 @@ class TsmConsole(conapi.Console): self.datacallback = None self.nodeconfig = config self.connected = False + self.recvr = None def recvdata(self): @@ -148,13 +149,16 @@ class TsmConsole(conapi.Console): self.ws.set_verify_callback(kv) self.ws.connect('wss://{0}/console0'.format(self.bmc), host=bmc, cookie='XSRF-TOKEN={0}; SESSION={1}'.format(wc.cookies['XSRF-TOKEN'], wc.cookies['SESSION']), subprotocols=[wc.cookies['XSRF-TOKEN']]) self.connected = True - eventlet.spawn_n(self.recvdata) + self.recvr = eventlet.spawn(self.recvdata) return def write(self, data): self.ws.send(data) def close(self): + if self.recvr: + self.recvr.kill() + self.recvr = None if self.ws: self.ws.close() self.connected = False From e735a12b3ae73dd267291aca6cb7fd5474392af8 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Thu, 22 Aug 2024 12:38:52 +0200 Subject: [PATCH 286/319] Fix small typo --- confluent_server/bin/confluent_selfcheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/bin/confluent_selfcheck b/confluent_server/bin/confluent_selfcheck index 64794ae4..f1de6c71 100755 --- a/confluent_server/bin/confluent_selfcheck +++ b/confluent_server/bin/confluent_selfcheck @@ -323,7 +323,7 @@ if __name__ == '__main__': print('{} appears to have networking configuration suitable for IPv6 deployment via: {}'.format(args.node, ",".join(bootablev6nics))) else: emprint(f"{args.node} may not have any viable IP network configuration (check name resolution (DNS or hosts file) " - "and/or net.*ipv4_address, and verify that the deployment serer addresses and subnet mask/prefix length are accurate)") + "and/or net.*ipv4_address, and verify that the deployment server addresses and subnet mask/prefix length are accurate)") if not uuidok and not macok: allok = False emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work (Example resolution: nodeinventory {args.node} -s)') From 82be3c91d5ed996cbf22a4e1c0705085dfa4964a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Aug 2024 10:20:04 -0400 Subject: [PATCH 287/319] Add support for relay DHCP Use the relay DHCP agent information as basis for subnet comparison instead of self, when present. Use the candidate prefix length verbatim since the relay will offer no hint as to the 'proper' prefix length for the target segment. --- .../confluent/discovery/protocols/pxe.py | 3 ++- confluent_server/confluent/netutil.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 133d8abd..9e88318b 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -728,9 +728,10 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): repview[1:10] = reqview[1:10] # duplicate txid, hwlen, and others repview[10:11] = b'\x80' # always set broadcast repview[28:44] = reqview[28:44] # copy chaddr field + relayip = reqview[24:28].tobytes() gateway = None netmask = None - niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx']) + niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx'], relayip=relayip) nicerr = niccfg.get('error_msg', False) if nicerr: log.log({'error': nicerr}) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index c1a9210a..a852d96c 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -408,7 +408,7 @@ def noneify(cfgdata): # the ip as reported by recvmsg to match the subnet of that net.* interface # if switch and port available, that should match. def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, - serverip=None): + serverip=None, relayipn=b'\x00\x00\x00\x00'): """Fetch network configuration parameters for a nic For a given node and interface, find and retrieve the pertinent network @@ -489,6 +489,10 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, bestsrvbyfam = {} for myaddr in myaddrs: fam, svrip, prefix = myaddr[:3] + if fam == socket.AF_INET and relayipn != b'\x00\x00\x00\x00': + bootsvrip = relayipn + else: + bootsvrip = svrip candsrvs.append((fam, svrip, prefix)) if fam == socket.AF_INET: nver = '4' @@ -508,6 +512,8 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, candip = cfgbyname[candidate].get('ipv{}_address'.format(nver), None) if candip and '/' in candip: candip, candprefix = candip.split('/') + if fam == socket.AF_INET and relayipn != b'\x00\x00\x00\x00': + prefix = int(candprefix) if int(candprefix) != prefix: continue candgw = cfgbyname[candidate].get('ipv{}_gateway'.format(nver), None) @@ -515,7 +521,7 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, try: for inf in socket.getaddrinfo(candip, 0, fam, socket.SOCK_STREAM): candipn = socket.inet_pton(fam, inf[-1][0]) - if ipn_on_same_subnet(fam, svrip, candipn, prefix): + if ipn_on_same_subnet(fam, bootsvrip, candipn, prefix): bestsrvbyfam[fam] = svrip cfgdata['ipv{}_address'.format(nver)] = candip cfgdata['ipv{}_method'.format(nver)] = ipmethod @@ -533,7 +539,7 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, elif candgw: for inf in socket.getaddrinfo(candgw, 0, fam, socket.SOCK_STREAM): candgwn = socket.inet_pton(fam, inf[-1][0]) - if ipn_on_same_subnet(fam, svrip, candgwn, prefix): + if ipn_on_same_subnet(fam, bootsvrip, candgwn, prefix): candgws.append((fam, candgwn, prefix)) if foundaddr: return noneify(cfgdata) From 683a160c20577fcb4d839283391a8f188263e373 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Aug 2024 10:25:15 -0400 Subject: [PATCH 288/319] Amend mistake in parameter name --- confluent_server/confluent/discovery/protocols/pxe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 9e88318b..dac2d30a 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -731,7 +731,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): relayip = reqview[24:28].tobytes() gateway = None netmask = None - niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx'], relayip=relayip) + niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx'], relayipn=relayip) nicerr = niccfg.get('error_msg', False) if nicerr: log.log({'error': nicerr}) From 3d53b7631752b19aa6d40aa2af643b3f549df425 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Aug 2024 14:54:48 -0400 Subject: [PATCH 289/319] Fix relay replies being broadcast instead of unicast --- .../confluent/discovery/protocols/pxe.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index dac2d30a..ccddbe9c 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -614,7 +614,7 @@ def check_reply(node, info, packet, sock, cfg, reqview, addr): return return reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock) else: - return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile) + return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock) def reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock): myaddrs = netutil.get_my_addresses(addr[-1], socket.AF_INET6) @@ -698,7 +698,7 @@ def get_my_duid(): return _myuuid -def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): +def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=None): replen = 275 # default is going to be 286 # while myipn is describing presumed destination, it's really # vague in the face of aliases, need to convert to ifidx and evaluate @@ -787,6 +787,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): myipn = socket.inet_aton(myipn) orepview[12:16] = myipn repview[20:24] = myipn + repview[24:28] = relayip repview[236:240] = b'\x63\x82\x53\x63' repview[240:242] = b'\x35\x01' if rqtype == 1: # if discover, then offer @@ -856,7 +857,10 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile): ipinfo = 'without address, served from {0}'.format(myip) log.log({ 'info': 'Offering {0} boot {1} to {2}'.format(boottype, ipinfo, node)}) - send_raw_packet(repview, replen + 28, reqview, info) + if relayip != b'\x00\x00\x00\x00': + sock.sendto(repview[28:28 + replen], (socket.inet_ntoa(relayip), 67)) + else: + send_raw_packet(repview, replen + 28, reqview, info) def send_raw_packet(repview, replen, reqview, info): ifidx = info['netinfo']['ifidx'] @@ -881,9 +885,10 @@ def send_raw_packet(repview, replen, reqview, info): sendto(tsock.fileno(), pkt, replen, 0, ctypes.byref(targ), ctypes.sizeof(targ)) -def ack_request(pkt, rq, info): +def ack_request(pkt, rq, info, sock=None): hwlen = bytearray(rq[2:3].tobytes())[0] hwaddr = rq[28:28+hwlen].tobytes() + relayip = rq[24:28].tobytes() myipn = myipbypeer.get(hwaddr, None) if not myipn or pkt.get(54, None) != myipn: return @@ -902,7 +907,10 @@ def ack_request(pkt, rq, info): repview[12:len(rply)].tobytes()) datasum = ~datasum & 0xffff repview[26:28] = struct.pack('!H', datasum) - send_raw_packet(repview, len(rply), rq, info) + if relayip != b'\x00\x00\x00\x00': + sock.sendto(repview[28:], (socket.inet_ntoa(relayip), 67)) + else: + send_raw_packet(repview, len(rply), rq, info) def consider_discover(info, packet, sock, cfg, reqview, nodeguess, addr=None): if info.get('hwaddr', None) in macmap and info.get('uuid', None): @@ -910,7 +918,7 @@ def consider_discover(info, packet, sock, cfg, reqview, nodeguess, addr=None): elif info.get('uuid', None) in uuidmap: check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview, addr) elif packet.get(53, None) == b'\x03': - ack_request(packet, reqview, info) + ack_request(packet, reqview, info, sock) elif info.get('uuid', None) and info.get('hwaddr', None): if time.time() > ignoremacs.get(info['hwaddr'], 0) + 90: ignoremacs[info['hwaddr']] = time.time() From edc3a3e9f337491849ff73a7cf20ada0b334c826 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Aug 2024 16:39:52 -0400 Subject: [PATCH 290/319] Have confignet fallback to unicast per deploycfg In routed deployments, the scan mechanism will not be available. Fall back to routed access to the deploy server as indicated by deploycfg from install time. --- confluent_osdeploy/common/profile/scripts/confignet | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index 650f4eb6..9092f631 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -455,6 +455,12 @@ if __name__ == '__main__': srvs, _ = apiclient.scan_confluents() doneidxs = set([]) dc = None + if not srvs: # the multicast scan failed, fallback to deploycfg cfg file + with open('/etc/confluent/confluent.deploycfg', 'r') as dci: + for cfgline in dci.read().split('\n'): + if cfgline.startswith('deploy_server:'): + srvs = [cfgline.split()[1]] + break for srv in srvs: try: s = socket.create_connection((srv, 443)) From 5d4f0662d1abd30b6faa3725ca6ec0c0c9323571 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 23 Aug 2024 07:06:37 -0400 Subject: [PATCH 291/319] Fix add_local_repositories for routed IPv4 Routed IPv4 deployment is not guaranteed to have an IPv6 server. In this case the safer bet is to try to just accept the IPv4 anyway. --- .../el8/profiles/default/scripts/add_local_repositories | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories b/confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories index fb26d5ef..79b0b6c5 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories +++ b/confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories @@ -27,7 +27,7 @@ with open('/etc/confluent/confluent.deploycfg') as dplcfgfile: _, profile = line.split(' ', 1) if line.startswith('ipv4_method: '): _, v4cfg = line.split(' ', 1) -if v4cfg == 'static' or v4cfg =='dhcp': +if v4cfg == 'static' or v4cfg =='dhcp' or not server6: server = server4 if not server: server = '[{}]'.format(server6) From f0c5ac557f9785fea93176997612149b52558ce5 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 23 Aug 2024 13:54:27 -0400 Subject: [PATCH 292/319] Fix and extend Relay DHCP Support Relay DHCP support needed better logging and behavior. It had also broken non-relay clients. --- .../confluent/discovery/protocols/pxe.py | 184 +++++++++++------- 1 file changed, 110 insertions(+), 74 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index ccddbe9c..f8f1254f 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -346,7 +346,7 @@ def proxydhcp(handler, nodeguess): profile = None if not myipn: myipn = socket.inet_aton(recv) - profile = get_deployment_profile(node, cfg) + profile, stgprofile = get_deployment_profile(node, cfg) if profile: log.log({ 'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])}) @@ -356,7 +356,7 @@ def proxydhcp(handler, nodeguess): continue if opts.get(77, None) == b'iPXE': if not profile: - profile = get_deployment_profile(node, cfg) + profile, stgprofile = get_deployment_profile(node, cfg) if not profile: log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)}) continue @@ -385,8 +385,9 @@ def proxydhcp(handler, nodeguess): rpv[268:280] = b'\x3c\x09PXEClient\xff' net4011.sendto(rpv[:281], client) except Exception as e: - tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, - event=log.Events.stacktrace) + log.logtrace() + # tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, + # event=log.Events.stacktrace) def start_proxydhcp(handler, nodeguess=None): @@ -453,13 +454,14 @@ def snoop(handler, protocol=None, nodeguess=None): # with try/except if i < 64: continue - _, level, typ = struct.unpack('QII', cmsgarr[:16]) - if level == socket.IPPROTO_IP and typ == IP_PKTINFO: - idx, recv = struct.unpack('II', cmsgarr[16:24]) - recv = ipfromint(recv) - rqv = memoryview(rawbuffer)[:i] if rawbuffer[0] == 1: # Boot request - process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv) + _, level, typ = struct.unpack('QII', cmsgarr[:16]) + if level == socket.IPPROTO_IP and typ == IP_PKTINFO: + idx, recv = struct.unpack('II', cmsgarr[16:24]) + recv = ipfromint(recv) + rqv = memoryview(rawbuffer)[:i] + client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port)) + process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv, client) elif netc == net6: recv = 'ff02::1:2' pkt, addr = netc.recvfrom(2048) @@ -476,6 +478,10 @@ def snoop(handler, protocol=None, nodeguess=None): tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace) + +_mac_to_uuidmap = {} + + def process_dhcp6req(handler, rqv, addr, net, cfg, nodeguess): ip = addr[0] req, disco = v6opts_to_dict(bytearray(rqv[4:])) @@ -501,7 +507,7 @@ def process_dhcp6req(handler, rqv, addr, net, cfg, nodeguess): handler(info) consider_discover(info, req, net, cfg, None, nodeguess, addr) -def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv): +def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv, client): rq = bytearray(rqv) addrlen = rq[2] if addrlen > 16 or addrlen == 0: @@ -531,7 +537,12 @@ def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv): # We will fill out service to have something to byte into, # but the nature of the beast is that we do not have peers, # so that will not be present for a pxe snoop - info = {'hwaddr': netaddr, 'uuid': disco['uuid'], + theuuid = disco['uuid'] + if theuuid: + _mac_to_uuidmap[netaddr] = theuuid + elif netaddr in _mac_to_uuidmap: + theuuid = _mac_to_uuidmap[netaddr] + info = {'hwaddr': netaddr, 'uuid': theuuid, 'architecture': disco['arch'], 'netinfo': {'ifidx': idx, 'recvip': recv, 'txid': txid}, 'services': ('pxe-client',)} @@ -539,7 +550,7 @@ def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv): and time.time() > ignoredisco.get(netaddr, 0) + 90): ignoredisco[netaddr] = time.time() handler(info) - consider_discover(info, rqinfo, net4, cfg, rqv, nodeguess) + consider_discover(info, rqinfo, net4, cfg, rqv, nodeguess, requestor=client) @@ -583,29 +594,34 @@ def get_deployment_profile(node, cfg, cfd=None): if not cfd: cfd = cfg.get_node_attributes(node, ('deployment.*', 'collective.managercandidates')) profile = cfd.get(node, {}).get('deployment.pendingprofile', {}).get('value', None) - if not profile: - return None - candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) - if candmgrs: - try: - candmgrs = noderange.NodeRange(candmgrs, cfg).nodes - except Exception: # fallback to unverified noderange - candmgrs = noderange.NodeRange(candmgrs).nodes - if collective.get_myname() not in candmgrs: - return None - return profile + stgprofile = cfd.get(node, {}).get('deployment.stagedprofile', {}).get('value', None) + if profile or stgprofile: + candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) + if candmgrs: + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: # fallback to unverified noderange + candmgrs = noderange.NodeRange(candmgrs).nodes + if collective.get_myname() not in candmgrs: + return None, None + return profile, stgprofile staticassigns = {} myipbypeer = {} -def check_reply(node, info, packet, sock, cfg, reqview, addr): - httpboot = info['architecture'] == 'uefi-httpboot' +def check_reply(node, info, packet, sock, cfg, reqview, addr, requestor): + if not requestor: + requestor = ('0.0.0.0', None) + if requestor[0] == '0.0.0.0' and not info.get('uuid', None): + return # ignore DHCP from local non-PXE segment + httpboot = info.get('architecture', None) == 'uefi-httpboot' cfd = cfg.get_node_attributes(node, ('deployment.*', 'collective.managercandidates')) - profile = get_deployment_profile(node, cfg, cfd) - if not profile: + profile, stgprofile = get_deployment_profile(node, cfg, cfd) + if ((not profile) + and (requestor[0] == '0.0.0.0' or not stgprofile)): if time.time() > ignoremacs.get(info['hwaddr'], 0) + 90: ignoremacs[info['hwaddr']] = time.time() log.log({'info': 'Ignoring boot attempt by {0} no deployment profile specified (uuid {1}, hwaddr {2})'.format( - node, info['uuid'], info['hwaddr'] + node, info.get('uuid', 'NA'), info['hwaddr'] )}) return if addr: @@ -614,7 +630,7 @@ def check_reply(node, info, packet, sock, cfg, reqview, addr): return return reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock) else: - return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock) + return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock, requestor) def reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock): myaddrs = netutil.get_my_addresses(addr[-1], socket.AF_INET6) @@ -651,14 +667,16 @@ def reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock): ipass[4:16] = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x18' ipass[16:32] = socket.inet_pton(socket.AF_INET6, ipv6addr) ipass[32:40] = b'\x00\x00\x00\x78\x00\x00\x01\x2c' - elif (not packet['vci']) or not packet['vci'].startswith('HTTPClient:Arch:'): - return # do not send ip-less replies to anything but HTTPClient specifically - #1 msgtype - #3 txid - #22 - server ident - #len(packet[1]) + 4 - client ident - #len(ipass) + 4 or 0 - #len(url) + 4 + elif (not packet['vci']) or not packet['vci'].startswith( + 'HTTPClient:Arch:'): + # do not send ip-less replies to anything but HTTPClient specifically + return + # 1 msgtype + # 3 txid + # 22 - server ident + # len(packet[1]) + 4 - client ident + # len(ipass) + 4 or 0 + # len(url) + 4 replylen = 50 + len(bootfile) + len(packet[1]) + 4 if len(ipass): replylen += len(ipass) @@ -698,26 +716,31 @@ def get_my_duid(): return _myuuid -def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=None): +def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=None, requestor=None): replen = 275 # default is going to be 286 # while myipn is describing presumed destination, it's really # vague in the face of aliases, need to convert to ifidx and evaluate # aliases for best match to guess - + isboot = True + if requestor is None: + requestor = ('0.0.0.0', None) + if info.get('architecture', None) is None: + isboot = False rqtype = packet[53][0] - insecuremode = cfd.get(node, {}).get('deployment.useinsecureprotocols', - {}).get('value', 'never') - if not insecuremode: - insecuremode = 'never' - if insecuremode == 'never' and not httpboot: - if rqtype == 1 and info['architecture']: - log.log( - {'info': 'Boot attempt by {0} detected in insecure mode, but ' - 'insecure mode is disabled. Set the attribute ' - '`deployment.useinsecureprotocols` to `firmware` or ' - '`always` to enable support, or use UEFI HTTP boot ' - 'with HTTPS.'.format(node)}) - return + if isboot: + insecuremode = cfd.get(node, {}).get('deployment.useinsecureprotocols', + {}).get('value', 'never') + if not insecuremode: + insecuremode = 'never' + if insecuremode == 'never' and not httpboot: + if rqtype == 1 and info.get('architecture', None): + log.log( + {'info': 'Boot attempt by {0} detected in insecure mode, but ' + 'insecure mode is disabled. Set the attribute ' + '`deployment.useinsecureprotocols` to `firmware` or ' + '`always` to enable support, or use UEFI HTTP boot ' + 'with HTTPS.'.format(node)}) + return reply = bytearray(512) repview = memoryview(reply) repview[:20] = iphdr @@ -729,6 +752,9 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N repview[10:11] = b'\x80' # always set broadcast repview[28:44] = reqview[28:44] # copy chaddr field relayip = reqview[24:28].tobytes() + if (not isboot) and relayip == b'\x00\x00\x00\x00': + # Ignore local DHCP packets if it isn't a firmware request + return gateway = None netmask = None niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx'], relayipn=relayip) @@ -755,7 +781,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N gateway = None netmask = (2**32 - 1) ^ (2**(32 - netmask) - 1) netmask = struct.pack('!I', netmask) - elif (not packet['vci']) or not (packet['vci'].startswith('HTTPClient:Arch:') or packet['vci'].startswith('PXEClient')): + elif (not packet.get('vci', None)) or not (packet['vci'].startswith('HTTPClient:Arch:') or packet['vci'].startswith('PXEClient')): return # do not send ip-less replies to anything but netboot specifically myipn = niccfg['deploy_server'] if not myipn: @@ -775,9 +801,9 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N node, profile, len(bootfile) - 127)}) return repview[108:108 + len(bootfile)] = bootfile - elif info['architecture'] == 'uefi-aarch64' and packet.get(77, None) == b'iPXE': + elif info.get('architecture', None) == 'uefi-aarch64' and packet.get(77, None) == b'iPXE': if not profile: - profile = get_deployment_profile(node, cfg) + profile, stgprofile = get_deployment_profile(node, cfg) if not profile: log.log({'info': 'No pending profile for {0}, skipping proxyDHCP eply'.format(node)}) return @@ -798,17 +824,19 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N repview[245:249] = myipn repview[249:255] = b'\x33\x04\x00\x00\x00\xf0' # fixed short lease time repview[255:257] = b'\x61\x11' - repview[257:274] = packet[97] + if packet.get(97, None) is not None: + repview[257:274] = packet[97] # Note that sending PXEClient kicks off the proxyDHCP procedure, ignoring # boot filename and such in the DHCP packet # we will simply always do it to provide the boot payload in a consistent # matter to both dhcp-elsewhere and fixed ip clients - if info['architecture'] == 'uefi-httpboot': - repview[replen - 1:replen + 11] = b'\x3c\x0aHTTPClient' - replen += 12 - else: - repview[replen - 1:replen + 10] = b'\x3c\x09PXEClient' - replen += 11 + if isboot: + if info.get('architecture', None) == 'uefi-httpboot': + repview[replen - 1:replen + 11] = b'\x3c\x0aHTTPClient' + replen += 12 + else: + repview[replen - 1:replen + 10] = b'\x3c\x09PXEClient' + replen += 11 hwlen = bytearray(reqview[2:3].tobytes())[0] fulladdr = repview[28:28+hwlen].tobytes() myipbypeer[fulladdr] = myipn @@ -825,13 +853,14 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N repview[replen - 1:replen + 1] = b'\x03\x04' repview[replen + 1:replen + 5] = gateway replen += 6 + elif relayip != b'\x00\x00\x00\x00': + log.log({'error': 'Relay DHCP offer to {} will fail due to missing gateway information'.format(node)}) if 82 in packet: reloptionslen = len(packet[82]) reloptionshdr = struct.pack('BB', 82, reloptionslen) repview[replen - 1:replen + 1] = reloptionshdr repview[replen + 1:replen + reloptionslen + 1] = packet[82] replen += 2 + reloptionslen - repview[replen - 1:replen] = b'\xff' # end of options, should always be last byte repview = memoryview(reply) pktlen = struct.pack('!H', replen + 28) # ip+udp = 28 @@ -855,13 +884,18 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N ipinfo = 'with static address {0}'.format(niccfg['ipv4_address']) else: ipinfo = 'without address, served from {0}'.format(myip) - log.log({ - 'info': 'Offering {0} boot {1} to {2}'.format(boottype, ipinfo, node)}) + if isboot: + log.log({ + 'info': 'Offering {0} boot {1} to {2}'.format(boottype, ipinfo, node)}) + else: + log.log({ + 'info': 'Offering DHCP {} to {}'.format(ipinfo, node)}) if relayip != b'\x00\x00\x00\x00': - sock.sendto(repview[28:28 + replen], (socket.inet_ntoa(relayip), 67)) + sock.sendto(repview[28:28 + replen], requestor) else: send_raw_packet(repview, replen + 28, reqview, info) + def send_raw_packet(repview, replen, reqview, info): ifidx = info['netinfo']['ifidx'] tsock = socket.socket(socket.AF_PACKET, socket.SOCK_DGRAM, @@ -885,7 +919,7 @@ def send_raw_packet(repview, replen, reqview, info): sendto(tsock.fileno(), pkt, replen, 0, ctypes.byref(targ), ctypes.sizeof(targ)) -def ack_request(pkt, rq, info, sock=None): +def ack_request(pkt, rq, info, sock=None, requestor=None): hwlen = bytearray(rq[2:3].tobytes())[0] hwaddr = rq[28:28+hwlen].tobytes() relayip = rq[24:28].tobytes() @@ -908,17 +942,19 @@ def ack_request(pkt, rq, info, sock=None): datasum = ~datasum & 0xffff repview[26:28] = struct.pack('!H', datasum) if relayip != b'\x00\x00\x00\x00': - sock.sendto(repview[28:], (socket.inet_ntoa(relayip), 67)) + sock.sendto(repview[28:], requestor) else: send_raw_packet(repview, len(rply), rq, info) -def consider_discover(info, packet, sock, cfg, reqview, nodeguess, addr=None): - if info.get('hwaddr', None) in macmap and info.get('uuid', None): - check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview, addr) +def consider_discover(info, packet, sock, cfg, reqview, nodeguess, addr=None, requestor=None): + if packet.get(53, None) == b'\x03': + ack_request(packet, reqview, info, sock, requestor) + elif info.get('hwaddr', None) in macmap: # and info.get('uuid', None): + check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview, addr, requestor) elif info.get('uuid', None) in uuidmap: - check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview, addr) + check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview, addr, requestor) elif packet.get(53, None) == b'\x03': - ack_request(packet, reqview, info, sock) + ack_request(packet, reqview, info, sock, requestor) elif info.get('uuid', None) and info.get('hwaddr', None): if time.time() > ignoremacs.get(info['hwaddr'], 0) + 90: ignoremacs[info['hwaddr']] = time.time() From c7d87b755ab0183343c04d496b87dd9f6e9c0061 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 23 Aug 2024 18:02:15 -0400 Subject: [PATCH 293/319] Add logic to get remote deployers locked into current nic --- confluent_server/confluent/netutil.py | 40 +++++++++++++++++++++-- confluent_server/confluent/selfservice.py | 2 +- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index a852d96c..b4b1fbb5 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -408,7 +408,8 @@ def noneify(cfgdata): # the ip as reported by recvmsg to match the subnet of that net.* interface # if switch and port available, that should match. def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, - serverip=None, relayipn=b'\x00\x00\x00\x00'): + serverip=None, relayipn=b'\x00\x00\x00\x00', + clientip=None): """Fetch network configuration parameters for a nic For a given node and interface, find and retrieve the pertinent network @@ -429,6 +430,25 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, #TODO(jjohnson2): ip address, prefix length, mac address, # join a bond/bridge, vlan configs, etc. # also other nic criteria, physical location, driver and index... + clientfam = None + clientipn = None + serverfam = None + serveripn = None + llaipn = socket.inet_pton(socket.AF_INET6, 'fe80::') + if serverip is not None: + if '.' in serverip: + serverfam = socket.AF_INET + elif ':' in serverip: + serverfam = socket.AF_INET6 + if serverfam: + serveripn = socket.inet_pton(serverfam, serverip) + if clientip is not None: + if '.' in clientip: + clientfam = socket.AF_INET + elif ':' in clientip: + clientfam = socket.AF_INET6 + if clientfam: + clientipn = socket.inet_pton(clientfam, clientip) nodenetattribs = configmanager.get_node_attributes( node, 'net*').get(node, {}) cfgbyname = {} @@ -466,9 +486,22 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, cfgdata['ipv4_broken'] = True if v6broken: cfgdata['ipv6_broken'] = True + isremote = False if serverip is not None: dhcprequested = False myaddrs = get_addresses_by_serverip(serverip) + if serverfam == socket.AF_INET6 and ipn_on_same_subnet(serverfam, serveripn, llaipn, 64): + isremote = False + elif clientfam: + for myaddr in myaddrs: + # we may have received over a local vlan, wrong aliased subnet + # so have to check for *any* potential matches + fam, svrip, prefix = myaddr[:3] + if fam == clientfam: + if ipn_on_same_subnet(fam, clientipn, svrip, prefix): + break + else: + isremote = True genericmethod = 'static' ipbynodename = None ip6bynodename = None @@ -514,14 +547,15 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, candip, candprefix = candip.split('/') if fam == socket.AF_INET and relayipn != b'\x00\x00\x00\x00': prefix = int(candprefix) - if int(candprefix) != prefix: + if (not isremote) and int(candprefix) != prefix: continue candgw = cfgbyname[candidate].get('ipv{}_gateway'.format(nver), None) if candip: try: for inf in socket.getaddrinfo(candip, 0, fam, socket.SOCK_STREAM): candipn = socket.inet_pton(fam, inf[-1][0]) - if ipn_on_same_subnet(fam, bootsvrip, candipn, prefix): + if ((isremote and ipn_on_same_subnet(fam, clientipn, candipn, int(candprefix))) + or ipn_on_same_subnet(fam, bootsvrip, candipn, prefix)): bestsrvbyfam[fam] = svrip cfgdata['ipv{}_address'.format(nver)] = candip cfgdata['ipv{}_method'.format(nver)] = ipmethod diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index 438d5b3e..b7577b92 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -282,7 +282,7 @@ def handle_request(env, start_response): ifidx = int(nici.read()) ncfg = netutil.get_nic_config(cfg, nodename, ifidx=ifidx) else: - ncfg = netutil.get_nic_config(cfg, nodename, serverip=myip) + ncfg = netutil.get_nic_config(cfg, nodename, serverip=myip, clientip=clientip) if env['PATH_INFO'] == '/self/deploycfg': for key in list(ncfg): if 'v6' in key: From 77c5b70ad954eebc2916d6924a677f47837cf217 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 26 Aug 2024 09:34:58 -0400 Subject: [PATCH 294/319] Wire up '-a -e' for nodeconfig --- confluent_client/bin/nodeconfig | 11 ++++++++--- .../confluent/plugins/hardwaremanagement/ipmi.py | 5 ++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/confluent_client/bin/nodeconfig b/confluent_client/bin/nodeconfig index 06d512c7..4d3d17f3 100755 --- a/confluent_client/bin/nodeconfig +++ b/confluent_client/bin/nodeconfig @@ -303,9 +303,14 @@ else: '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange), session, printbmc, options, attrprefix='bmc.') if options.extra: - rcode |= client.print_attrib_path( - '/noderange/{0}/configuration/management_controller/extended/extra'.format(noderange), - session, printextbmc, options) + if options.advanced: + rcode |= client.print_attrib_path( + '/noderange/{0}/configuration/management_controller/extended/extra_advanced'.format(noderange), + session, printextbmc, options) + else: + rcode |= client.print_attrib_path( + '/noderange/{0}/configuration/management_controller/extended/extra'.format(noderange), + session, printextbmc, options) if printsys or options.exclude: if printsys == 'all': printsys = [] diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index 06a8c444..566cc478 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -660,6 +660,8 @@ class IpmiHandler(object): return self.handle_bmcconfig(True) elif self.element[1:4] == ['management_controller', 'extended', 'extra']: return self.handle_bmcconfig(True, extended=True) + elif self.element[1:4] == ['management_controller', 'extended', 'extra_advanced']: + return self.handle_bmcconfig(True, advanced=True, extended=True) elif self.element[1:3] == ['system', 'all']: return self.handle_sysconfig() elif self.element[1:3] == ['system', 'advanced']: @@ -1472,7 +1474,8 @@ class IpmiHandler(object): if 'read' == self.op: try: if extended: - bmccfg = self.ipmicmd.get_extended_bmc_configuration() + bmccfg = self.ipmicmd.get_extended_bmc_configuration( + hideadvanced=(not advanced)) else: bmccfg = self.ipmicmd.get_bmc_configuration() self.output.put(msg.ConfigSet(self.node, bmccfg)) From 4f1b6b1facd601f46972d8807fda13ddec4a2cb6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 26 Aug 2024 09:42:44 -0400 Subject: [PATCH 295/319] Add extra_advanced to core resource tree. --- confluent_server/confluent/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index ce792fcb..a4e311c4 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -358,6 +358,10 @@ def _init_core(): 'pluginattrs': ['hardwaremanagement.method'], 'default': 'ipmi', }), + 'extra_advanced': PluginRoute({ + 'pluginattrs': ['hardwaremanagement.method'], + 'default': 'ipmi', + }), }, }, 'storage': { From d84a76dbc6d21d3798abfa9f7d518e59842fa0b0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 26 Aug 2024 10:01:24 -0400 Subject: [PATCH 296/319] Fix -a and -e nodeconfig --- confluent_server/confluent/plugins/hardwaremanagement/ipmi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index 566cc478..b53eccb1 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -659,9 +659,9 @@ class IpmiHandler(object): elif self.element[1:4] == ['management_controller', 'extended', 'advanced']: return self.handle_bmcconfig(True) elif self.element[1:4] == ['management_controller', 'extended', 'extra']: - return self.handle_bmcconfig(True, extended=True) + return self.handle_bmcconfig(advanced=False, extended=True) elif self.element[1:4] == ['management_controller', 'extended', 'extra_advanced']: - return self.handle_bmcconfig(True, advanced=True, extended=True) + return self.handle_bmcconfig(advanced=True, extended=True) elif self.element[1:3] == ['system', 'all']: return self.handle_sysconfig() elif self.element[1:3] == ['system', 'advanced']: From 7304c8e3b7169afdda2b4b606a62c9e974550d8a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 26 Aug 2024 16:51:35 -0400 Subject: [PATCH 297/319] Fix LLA request for deploycfg If a client uses IPv6 LLA, the '%' location may slip in. In such a case, disable client ip matching, since fe80:: is useless for that anyway. --- confluent_server/confluent/netutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index b4b1fbb5..e5384f5d 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -443,7 +443,10 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None, if serverfam: serveripn = socket.inet_pton(serverfam, serverip) if clientip is not None: - if '.' in clientip: + if '%' in clientip: + # link local, don't even bother' + clientfam = None + elif '.' in clientip: clientfam = socket.AF_INET elif ':' in clientip: clientfam = socket.AF_INET6 From 1d80e2703caab15634aea826ec52a89155e74f0f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 27 Aug 2024 09:59:16 -0400 Subject: [PATCH 298/319] Add extended nodeconfig to confluent redfish --- .../plugins/hardwaremanagement/redfish.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/redfish.py b/confluent_server/confluent/plugins/hardwaremanagement/redfish.py index 2c2857de..f89d9a09 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/redfish.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/redfish.py @@ -516,6 +516,12 @@ class IpmiHandler(object): return self.handle_ntp() elif self.element[1:4] == ['management_controller', 'extended', 'all']: return self.handle_bmcconfig() + elif self.element[1:4] == ['management_controller', 'extended', 'advanced']: + return self.handle_bmcconfig(True) + elif self.element[1:4] == ['management_controller', 'extended', 'extra']: + return self.handle_bmcconfig(advanced=False, extended=True) + elif self.element[1:4] == ['management_controller', 'extended', 'extra_advanced']: + return self.handle_bmcconfig(advanced=True, extended=True) elif self.element[1:3] == ['system', 'all']: return self.handle_sysconfig() elif self.element[1:3] == ['system', 'advanced']: @@ -1312,12 +1318,15 @@ class IpmiHandler(object): if 'read' == self.op: lc = self.ipmicmd.get_location_information() - def handle_bmcconfig(self, advanced=False): + def handle_bmcconfig(self, advanced=False, extended=False): if 'read' == self.op: try: - self.output.put(msg.ConfigSet( - self.node, - self.ipmicmd.get_bmc_configuration())) + if extended: + bmccfg = self.ipmicmd.get_extended_bmc_configuration( + hideadvanced=(not advanced)) + else: + bmccfg = self.ipmicmd.get_bmc_configuration() + self.output.put(msg.ConfigSet(self.node, bmccfg)) except Exception as e: self.output.put( msg.ConfluentNodeError(self.node, str(e))) From b601cd97c7c09b6070b49193c49b8472b832f613 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 27 Aug 2024 10:36:30 -0400 Subject: [PATCH 299/319] Bump pyghmi to a higher explicit version --- confluent_server/builddeb | 4 ++-- confluent_server/confluent_server.spec.tmpl | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/confluent_server/builddeb b/confluent_server/builddeb index a63d9d4f..71080110 100755 --- a/confluent_server/builddeb +++ b/confluent_server/builddeb @@ -36,9 +36,9 @@ if [ "$OPKGNAME" = "confluent-server" ]; then if grep wheezy /etc/os-release; then sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control elif grep jammy /etc/os-release; then - sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil/' debian/control + sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi(>=1.5.71), python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil/' debian/control else - sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-pyasyncore/' debian/control + sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi(>=1.5.71), python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd, python3-netifaces, python3-yaml, python3-dateutil, python3-pyasyncore/' debian/control fi if grep wheezy /etc/os-release; then echo 'confluent_client python-confluent-client' >> debian/pydist-overrides diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 04e63b21..7f99ee11 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -14,15 +14,15 @@ Prefix: %{_prefix} BuildArch: noarch Requires: confluent_vtbufferd %if "%{dist}" == ".el7" -Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python-websocket-client python2-msgpack python-libarchive-c python-yaml python-monotonic cpio +Requires: python-pyghmi >= 1.5.71, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python-websocket-client python2-msgpack python-libarchive-c python-yaml python-monotonic cpio %else %if "%{dist}" == ".el8" -Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio +Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio %else %if "%{dist}" == ".el9" -Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio +Requires: python3-pyghmi >= 1.5.74, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio %else -Requires: python3-dbm,python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodome >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dnspython, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-PyYAML openssl iproute +Requires: python3-dbm,python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodome >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dnspython, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-PyYAML openssl iproute %endif %endif %endif From 08a3a0ee766ac5f1a85cf1365ac0ecb76f72c6ad Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 27 Aug 2024 10:43:47 -0400 Subject: [PATCH 300/319] Correct wrong pyghmi version in dependencies --- confluent_server/confluent_server.spec.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 7f99ee11..acaf9fc4 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -20,7 +20,7 @@ Requires: python-pyghmi >= 1.5.71, python-eventlet, python-greenlet, python-pycr Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio %else %if "%{dist}" == ".el9" -Requires: python3-pyghmi >= 1.5.74, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio +Requires: python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute cpio %else Requires: python3-dbm,python3-pyghmi >= 1.5.71, python3-eventlet, python3-greenlet, python3-pycryptodome >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dnspython, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-cffi, python3-pyOpenSSL, python3-websocket-client python3-msgpack python3-libarchive-c python3-PyYAML openssl iproute %endif From 4dc54b92d5a45a9fd8f988b90939854f1751596a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 27 Aug 2024 11:35:39 -0400 Subject: [PATCH 301/319] Correct nodeconsole syntaxwarning --- confluent_client/bin/nodeconsole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/bin/nodeconsole b/confluent_client/bin/nodeconsole index 076913be..f05d5783 100755 --- a/confluent_client/bin/nodeconsole +++ b/confluent_client/bin/nodeconsole @@ -243,7 +243,7 @@ if options.windowed: elif 'Height' in line: window_height = int(line.split(':')[1]) elif '-geometry' in line: - l = re.split(' |x|-|\+', line) + l = re.split(' |x|-|\\+', line) l_nosp = [ele for ele in l if ele.strip()] wmxo = int(l_nosp[1]) wmyo = int(l_nosp[2]) From cd91ed0b94b59e6b991dd482bc308cd4f4c22394 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 27 Aug 2024 15:55:54 -0400 Subject: [PATCH 302/319] Fix escape warning on newer python --- confluent_client/bin/confluent2hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/bin/confluent2hosts b/confluent_client/bin/confluent2hosts index bbf989b1..b467e5cc 100644 --- a/confluent_client/bin/confluent2hosts +++ b/confluent_client/bin/confluent2hosts @@ -157,7 +157,7 @@ def main(): elif attrib.endswith('.ipv6_address') and val: ip6bynode[node][currnet] = val.split('/', 1)[0] elif attrib.endswith('.hostname'): - namesbynode[node][currnet] = re.split('\s+|,', val) + namesbynode[node][currnet] = re.split(r'\s+|,', val) for node in ip4bynode: mydomain = domainsbynode.get(node, None) for ipdb in (ip4bynode, ip6bynode): From 53b9a13a515c7e09d2b888ea416f3bd3890d8c16 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 28 Aug 2024 19:18:43 -0400 Subject: [PATCH 303/319] Fix different invocations of check_fish Particularly nodediscover register can fail. Those invocations are XCC specific, so the targtype should not matter in those cases. --- confluent_server/confluent/discovery/protocols/ssdp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/ssdp.py b/confluent_server/confluent/discovery/protocols/ssdp.py index c7063838..5c27473b 100644 --- a/confluent_server/confluent/discovery/protocols/ssdp.py +++ b/confluent_server/confluent/discovery/protocols/ssdp.py @@ -451,7 +451,11 @@ def _find_service(service, target): def check_fish(urldata, port=443, verifycallback=None): if not verifycallback: verifycallback = lambda x: True - url, data, targtype = urldata + try: + url, data, targtype = urldata + except ValueError: + url, data = urldata + targtype = 'service:redfish-bmc' try: wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5) peerinfo = wc.grab_json_response(url, headers={'Accept': 'application/json'}) From fd301c609f3b10d2e46a174c4b9e8eba80f8b8f6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 29 Aug 2024 08:51:22 -0400 Subject: [PATCH 304/319] Add additional log data to relaydhcp --- confluent_server/confluent/discovery/protocols/pxe.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index f8f1254f..5e5c98d4 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -755,6 +755,9 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N if (not isboot) and relayip == b'\x00\x00\x00\x00': # Ignore local DHCP packets if it isn't a firmware request return + relayipa = None + if relayip != b'\x00\x00\x00\x00': + relayipa = socket.inet_ntoa(relayip) gateway = None netmask = None niccfg = netutil.get_nic_config(cfg, node, ifidx=info['netinfo']['ifidx'], relayipn=relayip) @@ -884,6 +887,8 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N ipinfo = 'with static address {0}'.format(niccfg['ipv4_address']) else: ipinfo = 'without address, served from {0}'.format(myip) + if relayipa: + ipinfo += ' (relayed to {} via {})'.format(relayipa, requestor[0]) if isboot: log.log({ 'info': 'Offering {0} boot {1} to {2}'.format(boottype, ipinfo, node)}) From f4f5b4d1b6869b783af722d601e7c285d036d7d3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 29 Aug 2024 08:57:39 -0400 Subject: [PATCH 305/319] Suppress bad gateway warning if the client ip is unknown In a scenario where we are doing the 'next-server' next to a real dhcp server, but through relay, the missing gateway would be expected. Rely on the final message about 'no address' as the clue to users that something went wrong on the node side. --- confluent_server/confluent/discovery/protocols/pxe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index 5e5c98d4..64e64c79 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -856,7 +856,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, sock=N repview[replen - 1:replen + 1] = b'\x03\x04' repview[replen + 1:replen + 5] = gateway replen += 6 - elif relayip != b'\x00\x00\x00\x00': + elif relayip != b'\x00\x00\x00\x00' and clipn: log.log({'error': 'Relay DHCP offer to {} will fail due to missing gateway information'.format(node)}) if 82 in packet: reloptionslen = len(packet[82]) From 1f6987bafcafd737aa9f9403bdfe9989cb8c9406 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sat, 31 Aug 2024 07:28:51 -0400 Subject: [PATCH 306/319] Fix nodedeploy -c with profile Remove vestigial statement near end, and put an up front clarification to a user trying to use both '-c' and a profile on the same command line. --- confluent_client/bin/nodedeploy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/confluent_client/bin/nodedeploy b/confluent_client/bin/nodedeploy index 52e3a7d9..15e78f37 100755 --- a/confluent_client/bin/nodedeploy +++ b/confluent_client/bin/nodedeploy @@ -81,6 +81,12 @@ def main(args): if not args.profile and args.network: sys.stderr.write('Both noderange and a profile name are required arguments to request a network deployment\n') return 1 + if args.clear and args.profile: + sys.stderr.write( + 'The -c/--clear option should not be used with a profile, ' + 'it is a request to not deploy any profile, and will clear ' + 'whatever the current profile is without being specified\n') + return 1 if extra: sys.stderr.write('Unrecognized arguments: ' + repr(extra) + '\n') c = client.Command() @@ -166,8 +172,6 @@ def main(args): ','.join(errnodes))) return 1 rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot') - if args.network and not args.prepareonly: - return rc return 0 if __name__ == '__main__': From e11c3516e98c451cbc98dbb240456b687a339676 Mon Sep 17 00:00:00 2001 From: Markus Hilger Date: Tue, 3 Sep 2024 18:25:15 +0200 Subject: [PATCH 307/319] Create README.md (#162) --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..9be6cc60 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# Confluent + +![Python 3](https://img.shields.io/badge/python-3-blue.svg) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/xcat2/confluent/blob/master/LICENSE) + +Confluent is a software package to handle essential bootstrap and operation of scale-out server configurations. +It supports stateful and stateless deployments for various operating systems. + +Check [this page](https://hpc.lenovo.com/users/documentation/whatisconfluent.html +) for a more detailed list of features. + +Confluent is the modern successor of [xCAT](https://github.com/xcat2/xcat-core). +If you're coming from xCAT, check out [this comparison](https://hpc.lenovo.com/users/documentation/confluentvxcat.html). + +# Documentation + +Confluent documentation is hosted on hpc.lenovo.com: https://hpc.lenovo.com/users/documentation/ + +# Download + +Get the latest version from: https://hpc.lenovo.com/users/downloads/ + +Check release notes on: https://hpc.lenovo.com/users/news/ + +# Open Source License + +Confluent is made available under the Apache 2.0 license: https://opensource.org/license/apache-2-0 + +# Developers + +Want to help? Submit a [Pull Request](https://github.com/xcat2/confluent/pulls). From cb67c8328751736ff3e234305a64aad537d7ba3e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 08:11:42 -0400 Subject: [PATCH 308/319] iAttempt to be consistent with ~ prerelease versioning We may not know where we are going, but at least bump the minor number. Ultimately we may not be building toward that number, or that number will be used in a different branch, but it can at least handle more cases --- confluent_server/makesetup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_server/makesetup b/confluent_server/makesetup index de39ea18..012fe1d4 100755 --- a/confluent_server/makesetup +++ b/confluent_server/makesetup @@ -2,7 +2,11 @@ cd `dirname $0` VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS+g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+g`git describe|cut -d- -f 3` fi echo $VERSION > VERSION sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py From 97e29a5655ad876d8360d5f8ec075166fe5629fd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 10:11:47 -0400 Subject: [PATCH 309/319] Change versioning to consistently produce prerelease consistent with rpm and deb --- confluent_client/confluent_client.spec.tmpl | 2 +- confluent_server/confluent_server.spec.tmpl | 2 +- confluent_server/makesetup | 2 +- confluent_vtbufferd/builddeb | 6 +++++- confluent_vtbufferd/buildrpm | 6 +++++- imgutil/builddeb | 6 +++++- imgutil/buildrpm | 6 +++++- 7 files changed, 23 insertions(+), 7 deletions(-) diff --git a/confluent_client/confluent_client.spec.tmpl b/confluent_client/confluent_client.spec.tmpl index 820b0bbb..b26155ea 100644 --- a/confluent_client/confluent_client.spec.tmpl +++ b/confluent_client/confluent_client.spec.tmpl @@ -21,7 +21,7 @@ This package enables python development and command line access to a confluent server. %prep -%setup -n %{name}-%{version} -n %{name}-%{version} +%setup -n %{name}-%{lua: print(string.gsub("#VERSION#", "[~+]", "-"))} %build %if "%{dist}" == ".el7" diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index acaf9fc4..284a487d 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -33,7 +33,7 @@ Url: https://github.com/lenovo/confluent Server for console management and systems management aggregation %prep -%setup -n %{name}-%{version} -n %{name}-%{version} +%setup -n %{name}-%{lua: print(string.gsub("#VERSION#", "[~+]", "-"))} %build %if "%{dist}" == ".el7" diff --git a/confluent_server/makesetup b/confluent_server/makesetup index 012fe1d4..a34438d3 100755 --- a/confluent_server/makesetup +++ b/confluent_server/makesetup @@ -6,7 +6,7 @@ if [ "$NUMCOMMITS" != "$VERSION" ]; then LASTNUM=$((LASTNUM+1)) FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) VERSION=${FIRSTPART}.${LASTNUM} - VERSION=$VERSION~dev$NUMCOMMITS+g`git describe|cut -d- -f 3` + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi echo $VERSION > VERSION sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py diff --git a/confluent_vtbufferd/builddeb b/confluent_vtbufferd/builddeb index 36f41218..3a98315a 100755 --- a/confluent_vtbufferd/builddeb +++ b/confluent_vtbufferd/builddeb @@ -8,7 +8,11 @@ DSCARGS="--with-python3=True --with-python2=False" VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi cd .. rm -rf /tmp/confluent diff --git a/confluent_vtbufferd/buildrpm b/confluent_vtbufferd/buildrpm index 35feec59..9a20844d 100755 --- a/confluent_vtbufferd/buildrpm +++ b/confluent_vtbufferd/buildrpm @@ -1,7 +1,11 @@ VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi mkdir -p dist/confluent_vtbufferd-$VERSION cp ../LICENSE NOTICE *.c *.h Makefile dist/confluent_vtbufferd-$VERSION diff --git a/imgutil/builddeb b/imgutil/builddeb index 7e12a6e6..a7cee375 100755 --- a/imgutil/builddeb +++ b/imgutil/builddeb @@ -2,7 +2,11 @@ VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi mkdir -p /tmp/confluent-imgutil cp -a * /tmp/confluent-imgutil diff --git a/imgutil/buildrpm b/imgutil/buildrpm index 87631ac0..4439b7ed 100755 --- a/imgutil/buildrpm +++ b/imgutil/buildrpm @@ -2,7 +2,11 @@ VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi sed -e "s/#VERSION#/$VERSION/" confluent_imgutil.spec.tmpl > confluent_imgutil.spec cp ../LICENSE . From 4a2e943f8432a9531253c4a7e3ac60fc7616778e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 10:19:11 -0400 Subject: [PATCH 310/319] Update osdeploy rpms to new version scheme for snapshots --- confluent_osdeploy/buildrpm | 6 +++++- confluent_osdeploy/buildrpm-aarch64 | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/buildrpm b/confluent_osdeploy/buildrpm index 6bd1d419..9e9cb582 100755 --- a/confluent_osdeploy/buildrpm +++ b/confluent_osdeploy/buildrpm @@ -1,7 +1,11 @@ VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec cd .. diff --git a/confluent_osdeploy/buildrpm-aarch64 b/confluent_osdeploy/buildrpm-aarch64 index 83ffc519..c269284b 100644 --- a/confluent_osdeploy/buildrpm-aarch64 +++ b/confluent_osdeploy/buildrpm-aarch64 @@ -2,7 +2,11 @@ cd $(dirname $0) VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then - VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3` + LASTNUM=$(echo $VERSION|rev|cut -d . -f 1|rev) + LASTNUM=$((LASTNUM+1)) + FIRSTPART=$(echo $VERSION|rev|cut -d . -f 2- |rev) + VERSION=${FIRSTPART}.${LASTNUM} + VERSION=$VERSION~dev$NUMCOMMITS+`git describe|cut -d- -f 3` fi sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy-aarch64.spec.tmpl > confluent_osdeploy-aarch64.spec cd .. From ee067fa3c065a62ac8c2c3d66ed5eeea2de0b787 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 10:27:00 -0400 Subject: [PATCH 311/319] Cleanup stray debian content in apt --- confluent_server/builddeb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/builddeb b/confluent_server/builddeb index 71080110..9aca0d0a 100755 --- a/confluent_server/builddeb +++ b/confluent_server/builddeb @@ -74,7 +74,7 @@ else rm -rf $PKGNAME.egg-info dist setup.py rm -rf $(find deb_dist -mindepth 1 -maxdepth 1 -type d) if [ ! -z "$1" ]; then - mv deb_dist/* $1/ + mv deb_dist/*.deb $1/ fi fi exit 0 From 7c8f85eb065db88066a041a20d32d8db8aac8f01 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 10:53:05 -0400 Subject: [PATCH 312/319] Handle python mangling of filename consistently for rpm build --- confluent_client/confluent_client.spec.tmpl | 8 ++++++-- confluent_server/confluent_server.spec.tmpl | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/confluent_client/confluent_client.spec.tmpl b/confluent_client/confluent_client.spec.tmpl index b26155ea..ee786175 100644 --- a/confluent_client/confluent_client.spec.tmpl +++ b/confluent_client/confluent_client.spec.tmpl @@ -1,12 +1,16 @@ %define name confluent_client %define version #VERSION# +%define fversion %{lua: +sv, _ = string.gsub("#VERSION#", "[~+]", "-") +print(sv) +} %define release 1 Summary: Client libraries and utilities for confluent Name: %{name} Version: %{version} Release: %{release} -Source0: %{name}-%{version}.tar.gz +Source0: %{name}-%{fversion}.tar.gz License: Apache2 Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot @@ -21,7 +25,7 @@ This package enables python development and command line access to a confluent server. %prep -%setup -n %{name}-%{lua: print(string.gsub("#VERSION#", "[~+]", "-"))} +%setup -n %{name}-%{fversion} %build %if "%{dist}" == ".el7" diff --git a/confluent_server/confluent_server.spec.tmpl b/confluent_server/confluent_server.spec.tmpl index 284a487d..0a36390c 100644 --- a/confluent_server/confluent_server.spec.tmpl +++ b/confluent_server/confluent_server.spec.tmpl @@ -1,12 +1,16 @@ %define name confluent_server %define version #VERSION# +%define fversion %{lua: +sv, _ = string.gsub("#VERSION#", "[~+]", "-") +print(sv) +} %define release 1 Summary: confluent systems management server Name: %{name} Version: %{version} Release: %{release} -Source0: %{name}-%{version}.tar.gz +Source0: %{name}-%{fversion}.tar.gz License: Apache2 Group: Development/Libraries BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot @@ -33,7 +37,7 @@ Url: https://github.com/lenovo/confluent Server for console management and systems management aggregation %prep -%setup -n %{name}-%{lua: print(string.gsub("#VERSION#", "[~+]", "-"))} +%setup -n %{name}-%{fversion} %build %if "%{dist}" == ".el7" From 2c76d94a6debf504ff4c78161be2de60f9b4f359 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 12:13:40 -0400 Subject: [PATCH 313/319] Vary version for debian build process --- confluent_server/builddeb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confluent_server/builddeb b/confluent_server/builddeb index 9aca0d0a..3a983694 100755 --- a/confluent_server/builddeb +++ b/confluent_server/builddeb @@ -17,7 +17,9 @@ cd /tmp/confluent/$PKGNAME if [ -x ./makeman ]; then ./makeman fi -./makesetup +sed -e 's/~/./' ./makesetup > ./makesetup.deb +chmod +x ./makesetup.deb +./makesetup.deb VERSION=`cat VERSION` cat > setup.cfg << EOF [install] From d17d5341f11325141a62a03070d3e12e65859072 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Sep 2024 19:16:41 -0400 Subject: [PATCH 314/319] Imprement basic vinz management --- confluent_server/confluent/core.py | 1 + confluent_server/confluent/messages.py | 13 +- confluent_server/confluent/vinzmanager.py | 166 ++++++++++++++++++++++ 3 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 confluent_server/confluent/vinzmanager.py diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index a4e311c4..0ab6b647 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -430,6 +430,7 @@ def _init_core(): 'pluginattrs': ['hardwaremanagement.method'], 'default': 'ipmi', }), + 'ikvm': PluginRoute({'handler': 'ikvm'}), }, 'description': PluginRoute({ 'pluginattrs': ['hardwaremanagement.method'], diff --git a/confluent_server/confluent/messages.py b/confluent_server/confluent/messages.py index ce36344d..fa8dbbcb 100644 --- a/confluent_server/confluent/messages.py +++ b/confluent_server/confluent/messages.py @@ -262,10 +262,10 @@ class Generic(ConfluentMessage): def json(self): return json.dumps(self.data) - + def raw(self): return self.data - + def html(self): return json.dumps(self.data) @@ -344,10 +344,10 @@ class ConfluentResourceCount(ConfluentMessage): self.myargs = [count] self.desc = 'Resource Count' self.kvpairs = {'count': count} - + def strip_node(self, node): pass - + class CreatedResource(ConfluentMessage): notnode = True readonly = True @@ -569,6 +569,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False, return InputLicense(path, nodes, inputdata, configmanager) elif path == ['deployment', 'ident_image']: return InputIdentImage(path, nodes, inputdata) + elif path == ['console', 'ikvm']: + return InputIkvmParams(path, nodes, inputdata) elif inputdata: raise exc.InvalidArgumentException( 'No known input handler for request') @@ -936,6 +938,9 @@ class InputIdentImage(ConfluentInputMessage): keyname = 'ident_image' valid_values = ['create'] +class InputIkvmParams(ConfluentInputMessage): + keyname = 'method' + valid_values = ['unix', 'wss'] class InputIdentifyMessage(ConfluentInputMessage): valid_values = set([ diff --git a/confluent_server/confluent/vinzmanager.py b/confluent_server/confluent/vinzmanager.py new file mode 100644 index 00000000..0eb2314a --- /dev/null +++ b/confluent_server/confluent/vinzmanager.py @@ -0,0 +1,166 @@ + +import confluent.auth as auth +import eventlet +import confluent.messages as msg +import confluent.exceptions as exc +import confluent.util as util +import confluent.config.configmanager as configmanager +import struct +import eventlet.green.socket as socket +import eventlet.green.subprocess as subprocess +import base64 +import os +import pwd +mountsbyuser = {} +_vinzfd = None +_vinztoken = None +webclient = eventlet.import_patched('pyghmi.util.webclient') + + +# Handle the vinz VNC session +def assure_vinz(): + global _vinzfd + global _vinztoken + if _vinzfd is None: + _vinztoken = base64.b64encode(os.urandom(33), altchars=b'_-').decode() + os.environ['VINZ_TOKEN'] = _vinztoken + os.makedirs('/var/run/confluent/vinz/sessions', exist_ok=True) + + _vinzfd = subprocess.Popen( + ['/opt/confluent/bin/vinz', + '-c', '/var/run/confluent/vinz/control', + '-w', '127.0.0.1:4007', + '-a', '/var/run/confluent/vinz/approval', + # vinz supports unix domain websocket, however apache reverse proxy is dicey that way in some versions + '-d', '/var/run/confluent/vinz/sessions']) + while not os.path.exists('/var/run/confluent/vinz/control'): + eventlet.sleep(0.5) + eventlet.spawn(monitor_requests) + + +def get_url(nodename, inputdata): + method = inputdata.inputbynode[nodename] + assure_vinz() + if method == 'wss': + return f'/vinz/kvmsession/{nodename}' + elif method == 'unix': + return request_session(nodename) + + +def send_grant(conn, nodename): + cfg = configmanager.ConfigManager(None) + c = cfg.get_node_attributes( + nodename, + ['secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword', + 'hardwaremanagement.manager'], decrypt=True) + bmcuser = c.get(nodename, {}).get( + 'secret.hardwaremanagementuser', {}).get('value', None) + bmcpass = c.get(nodename, {}).get( + 'secret.hardwaremanagementpassword', {}).get('value', None) + bmc = c.get(nodename, {}).get( + 'hardwaremanagement.manager', {}).get('value', None) + if bmcuser and bmcpass and bmc: + kv = util.TLSCertVerifier(cfg, nodename, + 'pubkeys.tls_hardwaremanager').verify_cert + wc = webclient.SecureHTTPConnection(bmc, 443, verifycallback=kv) + if not isinstance(bmcuser, str): + bmcuser = bmcuser.decode() + if not isinstance(bmcpass, str): + bmcpass = bmcpass.decode() + rsp = wc.grab_json_response_with_status( + '/login', {'data': [bmcuser, bmcpass]}, + headers={'Content-Type': 'application/json', + 'Accept': 'application/json'}) + sessionid = wc.cookies['SESSION'] + sessiontok = wc.cookies['XSRF-TOKEN'] + url = '/kvm/0' + fprintinfo = cfg.get_node_attributes(nodename, 'pubkeys.tls_hardwaremanager') + fprint = fprintinfo.get( + nodename, {}).get('pubkeys.tls_hardwaremanager', {}).get('value', None) + if not fprint: + return + fprint = fprint.split('$', 1)[1] + fprint = bytes.fromhex(fprint) + conn.send(struct.pack('!BI', 1, len(bmc))) + conn.send(bmc.encode()) + conn.send(struct.pack('!I', len(sessionid))) + conn.send(sessionid.encode()) + conn.send(struct.pack('!I', len(sessiontok))) + conn.send(sessiontok.encode()) + conn.send(struct.pack('!I', len(fprint))) + conn.send(fprint) + conn.send(struct.pack('!I', len(url))) + conn.send(url.encode()) + conn.send(b'\xff') + +def evaluate_request(conn): + try: + creds = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, + struct.calcsize('iII')) + pid, uid, gid = struct.unpack('iII', creds) + if uid != os.getuid(): + return + rqcode, fieldlen = struct.unpack('!BI', conn.recv(5)) + if rqcode != 1: + return + authtoken = conn.recv(fieldlen).decode() + if authtoken != _vinztoken: + return + fieldlen = struct.unpack('!I', conn.recv(4))[0] + nodename = conn.recv(fieldlen).decode() + idtype = struct.unpack('!B', conn.recv(1))[0] + if idtype == 1: + usernum = struct.unpack('!I', conn.recv(4))[0] + if usernum == 0: # root is a special guy + send_grant(conn, nodename) + return + try: + authname = pwd.getpwuid(usernum).pw_name + except Exception: + return + allow = auth.authorize(authname, f'/nodes/{nodename}/console/ikvm') + if not allow: + return + send_grant(conn, nodename) + else: + return + if conn.recv(1) != b'\xff': + return + + finally: + conn.close() + +def monitor_requests(): + a = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + os.remove('/var/run/confluent/vinz/approval') + except Exception: + pass + a.bind('/var/run/confluent/vinz/approval') + os.chmod('/var/run/confluent/vinz/approval', 0o600) + a.listen(8) + while True: + conn, addr = a.accept() + eventlet.spawn_n(evaluate_request, conn) + +def request_session(nodename): + assure_vinz() + a = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + a.connect('/var/run/confluent/vinz/control') + nodename = nodename.encode() + a.send(struct.pack('!BI', 1, len(nodename))) + a.send(nodename) + a.send(b'\xff') + rsp = a.recv(1) + retcode = struct.unpack('!B', rsp)[0] + if retcode != 1: + raise Exception("Bad return code") + rsp = a.recv(4) + nlen = struct.unpack('!I', rsp)[0] + sockname = a.recv(nlen).decode('utf8') + retcode = a.recv(1) + if retcode != b'\xff': + raise Exception("Unrecognized response") + return os.path.join('/var/run/confluent/vinz/sessions', sockname) + From f8715f4cb19a67cf520a9fb444ce7114c86496cd Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 5 Sep 2024 11:42:52 -0400 Subject: [PATCH 315/319] Implement logout on disconnect notification for vinz --- confluent_server/confluent/vinzmanager.py | 78 +++++++++++++++++------ 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/confluent_server/confluent/vinzmanager.py b/confluent_server/confluent/vinzmanager.py index 0eb2314a..991ba0be 100644 --- a/confluent_server/confluent/vinzmanager.py +++ b/confluent_server/confluent/vinzmanager.py @@ -46,6 +46,36 @@ def get_url(nodename, inputdata): elif method == 'unix': return request_session(nodename) +_usersessions = {} +def close_session(sessionid): + sessioninfo = _usersessions.get(sessionid, None) + if not sessioninfo: + return + del _usersessions[sessionid] + nodename = sessioninfo['nodename'] + wc = sessioninfo['webclient'] + cfg = configmanager.ConfigManager(None) + c = cfg.get_node_attributes( + nodename, + ['secret.hardwaremanagementuser', + 'secret.hardwaremanagementpassword', + ], decrypt=True) + bmcuser = c.get(nodename, {}).get( + 'secret.hardwaremanagementuser', {}).get('value', None) + bmcpass = c.get(nodename, {}).get( + 'secret.hardwaremanagementpassword', {}).get('value', None) + if not isinstance(bmcuser, str): + bmcuser = bmcuser.decode() + if not isinstance(bmcpass, str): + bmcpass = bmcpass.decode() + if bmcuser and bmcpass: + wc.grab_json_response_with_status( + '/logout', {'data': [bmcuser, bmcpass]}, + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'X-XSRF-TOKEN': wc.cookies['XSRF-TOKEN']}) + def send_grant(conn, nodename): cfg = configmanager.ConfigManager(None) @@ -74,6 +104,10 @@ def send_grant(conn, nodename): 'Accept': 'application/json'}) sessionid = wc.cookies['SESSION'] sessiontok = wc.cookies['XSRF-TOKEN'] + _usersessions[sessionid] = { + 'webclient': wc, + 'nodename': nodename, + } url = '/kvm/0' fprintinfo = cfg.get_node_attributes(nodename, 'pubkeys.tls_hardwaremanager') fprint = fprintinfo.get( @@ -102,32 +136,34 @@ def evaluate_request(conn): if uid != os.getuid(): return rqcode, fieldlen = struct.unpack('!BI', conn.recv(5)) - if rqcode != 1: - return authtoken = conn.recv(fieldlen).decode() if authtoken != _vinztoken: return - fieldlen = struct.unpack('!I', conn.recv(4))[0] - nodename = conn.recv(fieldlen).decode() - idtype = struct.unpack('!B', conn.recv(1))[0] - if idtype == 1: - usernum = struct.unpack('!I', conn.recv(4))[0] - if usernum == 0: # root is a special guy + if rqcode == 2: # disconnect notification + fieldlen = struct.unpack('!I', conn.recv(4))[0] + sessionid = conn.recv(fieldlen).decode() + close_session(sessionid) + conn.recv(1) # digest 0xff + if rqcode == 1: # request for new connection + fieldlen = struct.unpack('!I', conn.recv(4))[0] + nodename = conn.recv(fieldlen).decode() + idtype = struct.unpack('!B', conn.recv(1))[0] + if idtype == 1: + usernum = struct.unpack('!I', conn.recv(4))[0] + if usernum == 0: # root is a special guy + send_grant(conn, nodename) + return + try: + authname = pwd.getpwuid(usernum).pw_name + except Exception: + return + allow = auth.authorize(authname, f'/nodes/{nodename}/console/ikvm') + if not allow: + return send_grant(conn, nodename) + else: return - try: - authname = pwd.getpwuid(usernum).pw_name - except Exception: - return - allow = auth.authorize(authname, f'/nodes/{nodename}/console/ikvm') - if not allow: - return - send_grant(conn, nodename) - else: - return - if conn.recv(1) != b'\xff': - return - + conn.recv(1) # should be 0xff finally: conn.close() From c048439849198f039b0fb6d607118172697327ff Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 5 Sep 2024 11:44:29 -0400 Subject: [PATCH 316/319] Reuse existing vinz unix session for a node --- confluent_server/confluent/vinzmanager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/vinzmanager.py b/confluent_server/confluent/vinzmanager.py index 991ba0be..75b51294 100644 --- a/confluent_server/confluent/vinzmanager.py +++ b/confluent_server/confluent/vinzmanager.py @@ -37,14 +37,17 @@ def assure_vinz(): eventlet.sleep(0.5) eventlet.spawn(monitor_requests) - +_unix_by_nodename = {} def get_url(nodename, inputdata): method = inputdata.inputbynode[nodename] assure_vinz() if method == 'wss': return f'/vinz/kvmsession/{nodename}' elif method == 'unix': - return request_session(nodename) + if nodename not in _unix_by_nodename: + _unix_by_nodename[nodename] = request_session(nodename) + return _unix_by_nodename[nodename] + _usersessions = {} def close_session(sessionid): From 0fb19ce26360dbf22a358212b798647009370ed8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 6 Sep 2024 13:41:50 -0400 Subject: [PATCH 317/319] Wire up http sessions to vinzmanager --- confluent_server/confluent/httpapi.py | 14 ++++++++++++++ confluent_server/confluent/vinzmanager.py | 20 ++++++++++++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index 10ab8b86..cf6a42ee 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -72,6 +72,20 @@ opmap = { } +def get_user_for_session(sessionid, sessiontok): + if not isinstance(sessionid, str): + sessionid = sessionid.decode() + if not isinstance(sessiontok, str): + sessiontok = sessiontok.decode() + if not sessiontok or not sessionid: + raise Exception("invalid session id or token") + if sessiontok != httpsessions.get(sessionid, {}).get('csrftoken', None): + raise Exception("Invalid csrf token for session") + user = httpsessions[sessionid]['name'] + if not isinstance(user, str): + user = user.decode() + return user + def group_creation_resources(): yield confluent.messages.Attributes( kv={'name': None}, desc="Name of the group").html() + '
' diff --git a/confluent_server/confluent/vinzmanager.py b/confluent_server/confluent/vinzmanager.py index 75b51294..308acb59 100644 --- a/confluent_server/confluent/vinzmanager.py +++ b/confluent_server/confluent/vinzmanager.py @@ -11,6 +11,7 @@ import eventlet.green.subprocess as subprocess import base64 import os import pwd +import confluent.httpapi as httpapi mountsbyuser = {} _vinzfd = None _vinztoken = None @@ -44,7 +45,7 @@ def get_url(nodename, inputdata): if method == 'wss': return f'/vinz/kvmsession/{nodename}' elif method == 'unix': - if nodename not in _unix_by_nodename: + if nodename not in _unix_by_nodename or not os.path.exists(_unix_by_nodename[nodename]): _unix_by_nodename[nodename] = request_session(nodename) return _unix_by_nodename[nodename] @@ -132,6 +133,8 @@ def send_grant(conn, nodename): conn.send(b'\xff') def evaluate_request(conn): + allow = False + authname = None try: creds = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize('iII')) @@ -160,13 +163,22 @@ def evaluate_request(conn): authname = pwd.getpwuid(usernum).pw_name except Exception: return - allow = auth.authorize(authname, f'/nodes/{nodename}/console/ikvm') - if not allow: + elif idtype == 2: + fieldlen = struct.unpack('!I', conn.recv(4))[0] + sessionid = conn.recv(fieldlen) + fieldlen = struct.unpack('!I', conn.recv(4))[0] + sessiontok = conn.recv(fieldlen) + try: + authname = httpapi.get_user_for_session(sessionid, sessiontok) + except Exception: return - send_grant(conn, nodename) else: return conn.recv(1) # should be 0xff + if authname: + allow = auth.authorize(authname, f'/nodes/{nodename}/console/ikvm') + if allow: + send_grant(conn, nodename) finally: conn.close() From 8704afcee5f0aa7f38f7923cdfe82120b536c380 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 10 Sep 2024 11:28:00 -0400 Subject: [PATCH 318/319] Add glue to confluent api from vinzmanager --- .../confluent/plugins/console/ikvm.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 confluent_server/confluent/plugins/console/ikvm.py diff --git a/confluent_server/confluent/plugins/console/ikvm.py b/confluent_server/confluent/plugins/console/ikvm.py new file mode 100644 index 00000000..395ead60 --- /dev/null +++ b/confluent_server/confluent/plugins/console/ikvm.py @@ -0,0 +1,34 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2024 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This provides linkage between vinz and confluent, with support +# for getting session authorization from the BMC + +import confluent.vinzmanager as vinzmanager +import confluent.messages as msg + + +def create(nodes, element, configmanager, inputdata): + for node in nodes: + url = vinzmanager.get_url(node, inputdata) + yield msg.ChildCollection(url) + + +def update(nodes, element, configmanager, inputdata): + for node in nodes: + url = vinzmanager.get_url(node, inputdata) + yield msg.ChildCollection(url) From 7da3944b2b7b0720f36cd7ac9961ffbb139652d9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 11 Sep 2024 09:13:30 -0400 Subject: [PATCH 319/319] Allow blink for ipmi OEM IPMI may now do blink --- .../confluent/plugins/hardwaremanagement/ipmi.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index b53eccb1..32fabefe 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -1378,10 +1378,8 @@ class IpmiHandler(object): def identify(self): if 'update' == self.op: identifystate = self.inputdata.inputbynode[self.node] == 'on' - if self.inputdata.inputbynode[self.node] == 'blink': - raise exc.InvalidArgumentException( - '"blink" is not supported with ipmi') - self.ipmicmd.set_identify(on=identifystate) + blinkstate = self.inputdata.inputbynode[self.node] == 'blink' + self.ipmicmd.set_identify(on=identifystate, blink=blinkstate) self.output.put(msg.IdentifyState( node=self.node, state=self.inputdata.inputbynode[self.node])) return