From f1d3e47439f901075f2d02728f478705d88feac7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 22 Mar 2024 12:56:09 -0400 Subject: [PATCH 01/13] Logout prior to renaming user Some firmware cannot tolerate a web session being active during a rename. Make sure logout has been done, and give a retry if needed to let the session close out after logging out. --- .../confluent/discovery/handlers/xcc.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/discovery/handlers/xcc.py b/confluent_server/confluent/discovery/handlers/xcc.py index 23f31fb0..ff7ca042 100644 --- a/confluent_server/confluent/discovery/handlers/xcc.py +++ b/confluent_server/confluent/discovery/handlers/xcc.py @@ -247,6 +247,10 @@ class NodeHandler(immhandler.NodeHandler): if rsp.status == 200: pwdchanged = True password = newpassword + wc.set_header('Authorization', 'Bearer ' + rspdata['access_token']) + if '_csrf_token' in wc.cookies: + wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token']) + wc.grab_json_response_with_status('/api/providers/logout') else: if rspdata.get('locktime', 0) > 0: raise LockedUserException( @@ -280,6 +284,7 @@ class NodeHandler(immhandler.NodeHandler): rsp.read() if rsp.status != 200: return (None, None) + wc.grab_json_response_with_status('/api/providers/logout') self._currcreds = (username, newpassword) wc.set_basic_credentials(username, newpassword) pwdchanged = True @@ -434,6 +439,7 @@ class NodeHandler(immhandler.NodeHandler): '/api/function', {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)}) if status == 200 and rsp.get('return', 0) == 13: + wc.grab_json_response('/api/providers/logout') wc.set_basic_credentials(self._currcreds[0], self._currcreds[1]) status = 503 while status != 200: @@ -442,10 +448,13 @@ class NodeHandler(immhandler.NodeHandler): {'UserName': username}, method='PATCH') if status != 200: rsp = json.loads(rsp) - if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError'): - eventlet.sleep(10) + if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'): + eventlet.sleep(4) else: break + self.tmppasswd = None + self._currcreds = (username, passwd) + return self.tmppasswd = None wc.grab_json_response('/api/providers/logout') self._currcreds = (username, passwd) @@ -632,3 +641,4 @@ def remote_nodecfg(nodename, cfm): info = {'addresses': [ipaddr]} nh = NodeHandler(info, cfm) nh.config(nodename) + From e38cd5d3e56ef0621f5449501719c810975ff096 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 15 Mar 2024 15:50:58 -0400 Subject: [PATCH 02/13] Fallback to unverified noderange on candidate manager check in PXE When doing pxe and the noderange of the candidate managers fails, try again without validation in case the user omitted collective members from nodelist, but still used ',' to enumerate them. --- confluent_server/confluent/discovery/protocols/pxe.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/discovery/protocols/pxe.py b/confluent_server/confluent/discovery/protocols/pxe.py index a9a07963..ff473ebc 100644 --- a/confluent_server/confluent/discovery/protocols/pxe.py +++ b/confluent_server/confluent/discovery/protocols/pxe.py @@ -587,7 +587,10 @@ def get_deployment_profile(node, cfg, cfd=None): return None candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None) if candmgrs: - candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + try: + candmgrs = noderange.NodeRange(candmgrs, cfg).nodes + except Exception: # fallback to unverified noderange + candmgrs = noderange.NodeRange(candmgrs).nodes if collective.get_myname() not in candmgrs: return None return profile From 19e9c6910d609f856cbc0422271e108cea5839e0 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 Mar 2024 15:32:44 -0400 Subject: [PATCH 03/13] Fix nodeapply redoing a single node multiple times --- confluent_client/bin/nodeapply | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/bin/nodeapply b/confluent_client/bin/nodeapply index e39447bc..2e798742 100755 --- a/confluent_client/bin/nodeapply +++ b/confluent_client/bin/nodeapply @@ -102,9 +102,9 @@ def run(): cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0] if currprocs < concurrentprocs: currprocs += 1 - run_cmdv(node, cmdv, all, pipedesc) + run_cmdv(sshnode, cmdv, all, pipedesc) else: - pendingexecs.append((node, cmdv)) + pendingexecs.append((sshnode, cmdv)) if not all or exitcode: sys.exit(exitcode) rdy, _, _ = select.select(all, [], [], 10) From ac1f7c57b606674a1e05788d382e470349ef53b7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 12 Mar 2024 09:36:40 -0400 Subject: [PATCH 04/13] Fix lldp when peername is null Some neighbors result in a null name, handle that. --- confluent_server/confluent/networking/lldp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/networking/lldp.py b/confluent_server/confluent/networking/lldp.py index e1fd8d4e..e181d46f 100644 --- a/confluent_server/confluent/networking/lldp.py +++ b/confluent_server/confluent/networking/lldp.py @@ -381,9 +381,10 @@ def list_info(parms, requestedparameter): break else: candidate = info[requestedparameter] - candidate = candidate.strip() - if candidate != '': - results.add(_api_sanitize_string(candidate)) + if candidate: + candidate = candidate.strip() + if candidate != '': + results.add(_api_sanitize_string(candidate)) return [msg.ChildCollection(x + suffix) for x in util.natural_sort(results)] def _handle_neighbor_query(pathcomponents, configmanager): From c1afc144cb32d900fe4b9962e7581bd3612174de Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Feb 2024 15:05:56 -0500 Subject: [PATCH 05/13] Change to unix domain for vtbuffer communication The semaphore arbitrated single channel sharing was proving to be too slow. Make the communication lockless by having dedicated sockets per request. --- confluent_server/confluent/consoleserver.py | 56 +++--- confluent_vtbufferd/vtbufferd.c | 179 +++++++++++++++----- 2 files changed, 161 insertions(+), 74 deletions(-) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index 37274792..783d77de 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -62,39 +62,38 @@ def chunk_output(output, n): yield output[i:i + n] def get_buffer_output(nodename): - out = _bufferdaemon.stdin - instream = _bufferdaemon.stdout + out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) + out.connect("\x00confluent-vtbuffer") if not isinstance(nodename, bytes): nodename = nodename.encode('utf8') outdata = bytearray() - with _bufferlock: - out.write(struct.pack('I', len(nodename))) - out.write(nodename) - out.flush() - select.select((instream,), (), (), 30) - while not outdata or outdata[-1]: - try: - chunk = os.read(instream.fileno(), 128) - except IOError: - chunk = None - if chunk: - outdata.extend(chunk) - else: - select.select((instream,), (), (), 0) - return bytes(outdata[:-1]) + out.send(struct.pack('I', len(nodename))) + out.send(nodename) + select.select((out,), (), (), 30) + while not outdata or outdata[-1]: + try: + chunk = os.read(out.fileno(), 128) + except IOError: + chunk = None + if chunk: + outdata.extend(chunk) + else: + select.select((out,), (), (), 0) + return bytes(outdata[:-1]) def send_output(nodename, output): if not isinstance(nodename, bytes): nodename = nodename.encode('utf8') - with _bufferlock: - _bufferdaemon.stdin.write(struct.pack('I', len(nodename) | (1 << 29))) - _bufferdaemon.stdin.write(nodename) - _bufferdaemon.stdin.flush() - for chunk in chunk_output(output, 8192): - _bufferdaemon.stdin.write(struct.pack('I', len(chunk) | (2 << 29))) - _bufferdaemon.stdin.write(chunk) - _bufferdaemon.stdin.flush() + out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) + out.connect("\x00confluent-vtbuffer") + out.send(struct.pack('I', len(nodename) | (1 << 29))) + out.send(nodename) + for chunk in chunk_output(output, 8192): + out.send(struct.pack('I', len(chunk) | (2 << 29))) + out.send(chunk) def _utf8_normalize(data, decoder): # first we give the stateful decoder a crack at the byte stream, @@ -604,11 +603,8 @@ def initialize(): _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen( - ['/opt/confluent/bin/vtbufferd'], bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL) - fcntl.fcntl(_bufferdaemon.stdout.fileno(), - fcntl.F_SETFL, fl | os.O_NONBLOCK) + ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) def start_console_sessions(): configmodule.hook_new_configmanagers(_start_tenant_sessions) diff --git a/confluent_vtbufferd/vtbufferd.c b/confluent_vtbufferd/vtbufferd.c index e89269b4..055a5263 100644 --- a/confluent_vtbufferd/vtbufferd.c +++ b/confluent_vtbufferd/vtbufferd.c @@ -1,8 +1,14 @@ +#include +#define _GNU_SOURCE #include #include #include #include #include +#include +#include +#include +#include #include "tmt.h" #define HASHSIZE 2053 #define MAXNAMELEN 256 @@ -10,13 +16,17 @@ struct terment { struct terment *next; char *name; + int fd; TMT *vt; }; #define SETNODE 1 #define WRITE 2 #define READBUFF 0 +#define CLOSECONN 3 +#define MAXEVTS 16 static struct terment *buffers[HASHSIZE]; +static char* nodenames[HASHSIZE]; unsigned long hash(char *str) /* djb2a */ @@ -37,10 +47,13 @@ TMT *get_termentbyname(char *name) { return NULL; } -TMT *set_termentbyname(char *name) { +TMT *set_termentbyname(char *name, int fd) { struct terment *ret; int idx; + if (nodenames[fd] == NULL) { + nodenames[fd] = strdup(name); + } idx = hash(name); for (ret = buffers[idx]; ret != NULL; ret = ret->next) if (strcmp(name, ret->name) == 0) @@ -48,12 +61,13 @@ TMT *set_termentbyname(char *name) { ret = (struct terment *)malloc(sizeof(*ret)); ret->next = buffers[idx]; ret->name = strdup(name); + ret->fd = fd; ret->vt = tmt_open(31, 100, NULL, NULL, L"→←↑↓■◆▒°±▒┘┐┌└┼⎺───⎽├┤┴┬│≤≥π≠£•"); buffers[idx] = ret; return ret->vt; } -void dump_vt(TMT* outvt) { +void dump_vt(TMT* outvt, int outfd) { const TMTSCREEN *out = tmt_screen(outvt); const TMTPOINT *curs = tmt_cursor(outvt); int line, idx, maxcol, maxrow; @@ -67,9 +81,10 @@ void dump_vt(TMT* outvt) { tmt_color_t fg = TMT_COLOR_DEFAULT; tmt_color_t bg = TMT_COLOR_DEFAULT; wchar_t sgrline[30]; + char strbuffer[128]; size_t srgidx = 0; char colorcode = 0; - wprintf(L"\033c"); + write(outfd, "\033c", 2); maxcol = 0; maxrow = 0; for (line = out->nline - 1; line >= 0; --line) { @@ -148,60 +163,136 @@ void dump_vt(TMT* outvt) { } if (sgrline[0] != 0) { sgrline[wcslen(sgrline) - 1] = 0; // Trim last ; - wprintf(L"\033[%lsm", sgrline); + + snprintf(strbuffer, sizeof(strbuffer), "\033[%lsm", sgrline); + write(outfd, strbuffer, strlen(strbuffer)); + write(outfd, "\033[]", 3); } - wprintf(L"%lc", out->lines[line]->chars[idx].c); + snprintf(strbuffer, sizeof(strbuffer), "%lc", out->lines[line]->chars[idx].c); + write(outfd, strbuffer, strlen(strbuffer)); } if (line < maxrow) - wprintf(L"\r\n"); + write(outfd, "\r\n", 2); } - fflush(stdout); - wprintf(L"\x1b[%ld;%ldH", curs->r + 1, curs->c + 1); - fflush(stdout); + //fflush(stdout); + snprintf(strbuffer, sizeof(strbuffer), "\x1b[%ld;%ldH", curs->r + 1, curs->c + 1); + write(outfd, strbuffer, strlen(strbuffer)); + //fflush(stdout); +} + +int handle_traffic(int fd) { + int cmd, length; + char currnode[MAXNAMELEN]; + char cmdbuf[MAXDATALEN]; + char *nodename; + TMT *currvt = NULL; + TMT *outvt = NULL; + length = read(fd, &cmd, 4); + if (length <= 0) { + return 0; + } + length = cmd & 536870911; + cmd = cmd >> 29; + if (cmd == SETNODE) { + cmd = read(fd, currnode, length); + currnode[length] = 0; + if (cmd < 0) + return 0; + currvt = set_termentbyname(currnode, fd); + } else if (cmd == WRITE) { + if (currvt == NULL) { + nodename = nodenames[fd]; + currvt = set_termentbyname(nodename, fd); + } + cmd = read(fd, cmdbuf, length); + cmdbuf[length] = 0; + if (cmd < 0) + return 0; + tmt_write(currvt, cmdbuf, length); + } else if (cmd == READBUFF) { + cmd = read(fd, cmdbuf, length); + cmdbuf[length] = 0; + if (cmd < 0) + return 0; + outvt = get_termentbyname(cmdbuf); + if (outvt != NULL) + dump_vt(outvt, fd); + length = write(fd, "\x00", 1); + if (length < 0) + return 0; + } else if (cmd == CLOSECONN) { + return 0; + } + return 1; } int main(int argc, char* argv[]) { - int cmd, length; setlocale(LC_ALL, ""); - char cmdbuf[MAXDATALEN]; - char currnode[MAXNAMELEN]; - TMT *currvt = NULL; - TMT *outvt = NULL; + struct sockaddr_un addr; + int numevts; + int status; + int poller; + int n; + socklen_t len; + int ctlsock, currsock; + socklen_t addrlen; + struct ucred ucr; + + struct epoll_event epvt, evts[MAXEVTS]; stdin = freopen(NULL, "rb", stdin); if (stdin == NULL) { exit(1); } + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path + 1, argv[1], sizeof(addr.sun_path) - 2); // abstract namespace socket + ctlsock = socket(AF_UNIX, SOCK_STREAM, 0); + status = bind(ctlsock, (const struct sockaddr*)&addr, sizeof(sa_family_t) + strlen(argv[1]) + 1); //sizeof(struct sockaddr_un)); + if (status < 0) { + perror("Unable to open unix socket - "); + exit(1); + } + listen(ctlsock, 128); + poller = epoll_create(1); + memset(&epvt, 0, sizeof(struct epoll_event)); + epvt.events = EPOLLIN; + epvt.data.fd = ctlsock; + if (epoll_ctl(poller, EPOLL_CTL_ADD, ctlsock, &epvt) < 0) { + perror("Unable to poll the socket"); + exit(1); + } + // create a unix domain socket for accepting, each connection is only allowed to either read or write, not both while (1) { - length = fread(&cmd, 4, 1, stdin); - if (length < 0) - continue; - length = cmd & 536870911; - cmd = cmd >> 29; - if (cmd == SETNODE) { - cmd = fread(currnode, 1, length, stdin); - currnode[length] = 0; - if (cmd < 0) - continue; - currvt = set_termentbyname(currnode); - } else if (cmd == WRITE) { - if (currvt == NULL) - currvt = set_termentbyname(""); - cmd = fread(cmdbuf, 1, length, stdin); - cmdbuf[length] = 0; - if (cmd < 0) - continue; - tmt_write(currvt, cmdbuf, length); - } else if (cmd == READBUFF) { - cmd = fread(cmdbuf, 1, length, stdin); - cmdbuf[length] = 0; - if (cmd < 0) - continue; - outvt = get_termentbyname(cmdbuf); - if (outvt != NULL) - dump_vt(outvt); - length = write(1, "\x00", 1); - if (length < 0) - continue; + numevts = epoll_wait(poller, evts, MAXEVTS, -1); + if (numevts < 0) { + perror("Failed wait"); + exit(1); + } + for (n = 0; n < numevts; ++n) { + if (evts[n].data.fd == ctlsock) { + currsock = accept(ctlsock, (struct sockaddr *) &addr, &addrlen); + len = sizeof(ucr); + getsockopt(currsock, SOL_SOCKET, SO_PEERCRED, &ucr, &len); + if (ucr.uid != getuid()) { // block access for other users + close(currsock); + continue; + } + memset(&epvt, 0, sizeof(struct epoll_event)); + epvt.events = EPOLLIN; + epvt.data.fd = currsock; + epoll_ctl(poller, EPOLL_CTL_ADD, currsock, &epvt); + } else { + if (!handle_traffic(evts[n].data.fd)) { + epoll_ctl(poller, EPOLL_CTL_DEL, evts[n].data.fd, NULL); + close(evts[n].data.fd); + if (nodenames[evts[n].data.fd] != NULL) { + free(nodenames[evts[n].data.fd]); + nodenames[evts[n].data.fd] = NULL; + } + } + } } } } + + From b3b3627bf926c3f18e0578fd3f8033463af4855a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 22 Feb 2024 15:07:12 -0500 Subject: [PATCH 06/13] Remove disused bufferlock We no longer use a lock on buffer communication, eliminate the stale variable. --- confluent_server/confluent/consoleserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/confluent_server/confluent/consoleserver.py b/confluent_server/confluent/consoleserver.py index 783d77de..7b9530f2 100644 --- a/confluent_server/confluent/consoleserver.py +++ b/confluent_server/confluent/consoleserver.py @@ -49,7 +49,6 @@ _handled_consoles = {} _tracelog = None _bufferdaemon = None -_bufferlock = None try: range = xrange @@ -599,8 +598,6 @@ def _start_tenant_sessions(cfm): def initialize(): global _tracelog global _bufferdaemon - global _bufferlock - _bufferlock = semaphore.Semaphore() _tracelog = log.Logger('trace') _bufferdaemon = subprocess.Popen( ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL, From d183a3f99cb452c2a8e0a38dcc9a2c30bb834db4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 1 Feb 2024 08:50:44 -0500 Subject: [PATCH 07/13] Fix problem where one multicast/broadcast attempt could tank other interfaces Carrying over change from ssdp, ignore failures on transmit, particularly if firewall --- confluent_server/confluent/discovery/protocols/slp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/discovery/protocols/slp.py b/confluent_server/confluent/discovery/protocols/slp.py index e42c1577..ac332def 100644 --- a/confluent_server/confluent/discovery/protocols/slp.py +++ b/confluent_server/confluent/discovery/protocols/slp.py @@ -246,11 +246,11 @@ def _find_srvtype(net, net4, srvtype, addresses, xid): try: net4.sendto(data, ('239.255.255.253', 427)) except socket.error as se: - # On occasion, multicasting may be disabled - # tolerate this scenario and move on - if se.errno != 101: - raise - net4.sendto(data, (bcast, 427)) + pass + try: + net4.sendto(data, (bcast, 427)) + except socket.error as se: + pass def _grab_rsps(socks, rsps, interval, xidmap, deferrals): From 7b3129a1a2a736d8d3672f2f983eb2ccd0062400 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 30 Jan 2024 09:08:28 -0500 Subject: [PATCH 08/13] Fix FFDC preflight checks The code was comparing two string constants, instead of a variable to a constant. Correct the problem to enable the preflight checks to work as intended. --- confluent_server/confluent/firmwaremanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/firmwaremanager.py b/confluent_server/confluent/firmwaremanager.py index a7713943..eb5d4c86 100644 --- a/confluent_server/confluent/firmwaremanager.py +++ b/confluent_server/confluent/firmwaremanager.py @@ -53,7 +53,7 @@ def execupdate(handler, filename, updateobj, type, owner, node, datfile): return if type == 'ffdc' and os.path.isdir(filename): filename += '/' + node - if 'type' == 'ffdc': + if type == 'ffdc': errstr = False if os.path.exists(filename): errstr = '{0} already exists on {1}, cannot overwrite'.format( From 17fff4997baa5b344013b9f4003bbe166bd94db3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 12 Jan 2024 10:52:49 -0500 Subject: [PATCH 09/13] Fix omission of info dir in plugins --- confluent_server/setup.py.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/setup.py.tmpl b/confluent_server/setup.py.tmpl index e6bd08b2..871497e3 100644 --- a/confluent_server/setup.py.tmpl +++ b/confluent_server/setup.py.tmpl @@ -19,6 +19,7 @@ setup( 'confluent/plugins/hardwaremanagement/', 'confluent/plugins/deployment/', 'confluent/plugins/console/', + 'confluent/plugins/info/', 'confluent/plugins/shell/', 'confluent/collective/', 'confluent/plugins/configuration/'], From ddb8c4cce44ac8d0d30385cea7d466ea862d3c73 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 1 Dec 2023 15:55:17 -0500 Subject: [PATCH 10/13] Fix a few noderange abbreviations Also, add some test cases on abbreviation to help sanity check things in the future. --- confluent_server/confluent/noderange.py | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index df4552b8..cf99dd72 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -96,6 +96,7 @@ class Bracketer(object): txtnums = getnumbers_nodename(nodename) nums = [int(x) for x in txtnums] for n in range(self.count): + # First pass to see if we have exactly one different number padto = len(txtnums[n]) needpad = (padto != len('{}'.format(nums[n]))) if self.sequences[n] is None: @@ -105,7 +106,24 @@ class Bracketer(object): elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: continue # new nodename has no new number, keep going else: # if self.sequences[n][2] != nums[n] or : - if self.diffn is not None and (n != self.diffn or + if self.diffn is not None and (n != self.diffn or + (padto < self.numlens[n][1]) or + (needpad and padto != self.numlens[n][1])): + self.flush_current() + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [padto, padto] + self.diffn = n + for n in range(self.count): + padto = len(txtnums[n]) + needpad = (padto != len('{}'.format(nums[n]))) + if self.sequences[n] is None: + # We initialize to text pieces, 'currstart', and 'prev' number + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] + elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: + continue # new nodename has no new number, keep going + else: # if self.sequences[n][2] != nums[n] or : + if self.diffn is not None and (n != self.diffn or (padto < self.numlens[n][1]) or (needpad and padto != self.numlens[n][1])): self.flush_current() @@ -449,3 +467,29 @@ class NodeRange(object): if self.cfm is None: return set([element]) raise Exception(element + ' not a recognized node, group, or alias') + +if __name__ == '__main__': + cases = [ + (['r3u4', 'r5u6'], 'r3u4,r5u6'), # should not erroneously gather + (['r3u4s1', 'r5u6s3'], 'r3u4s1,r5u6s3'), # should not erroneously gather + (['r3u4s1', 'r3u4s2', 'r5u4s3'], 'r3u4s[1:2],r5u4s3'), # should not erroneously gather + (['r3u4', 'r3u5', 'r3u6', 'r3u9', 'r4u1'], 'r3u[4:6,9],r4u1'), + (['n01', 'n2', 'n03'], 'n01,n2,n03'), + (['n7', 'n8', 'n09', 'n10', 'n11', 'n12', 'n13', 'n14', 'n15', 'n16', + 'n17', 'n18', 'n19', 'n20'], 'n[7:8],n[09:20]') + ] + for case in cases: + gc = case[0] + bracketer = Bracketer(gc[0]) + for chnk in gc[1:]: + bracketer.extend(chnk) + br = bracketer.range + resnodes = NodeRange(br).nodes + if set(resnodes) != set(gc): + print('FAILED: ' + repr(sorted(gc))) + print('RESULT: ' + repr(sorted(resnodes))) + print('EXPECTED: ' + repr(case[1])) + print('ACTUAL: ' + br) + + + From 661b2ae81542deffa9c53b7df844389515470eef Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 27 Nov 2023 08:34:34 -0500 Subject: [PATCH 11/13] Filter out nvme 'c' devnames, that are used to refer to paths to nvme Some versions start manifesting nvme devnames with 'c', which are to be used to interact with multipath to have raw devices backing a traditional nvme device. --- .../el7-diskless/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk | 2 ++ .../el8-diskless/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk | 2 ++ .../el9-diskless/profiles/default/scripts/getinstalldisk | 2 ++ .../rhvh4/profiles/default/scripts/getinstalldisk | 2 ++ confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk | 2 ++ .../suse15/profiles/server/scripts/getinstalldisk | 2 ++ .../profiles/default/scripts/getinstalldisk | 2 ++ .../ubuntu20.04/profiles/default/scripts/getinstalldisk | 2 ++ .../ubuntu22.04/profiles/default/scripts/getinstalldisk | 2 ++ 11 files changed, 22 insertions(+) diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk +++ b/confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk index 522aba00..04c7708e 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk @@ -3,6 +3,8 @@ import os class DiskInfo(object): def __init__(self, devname): + if devname.startswith('nvme') and 'c' in devname: + raise Exception("Skipping multipath devname") self.name = devname self.wwn = None self.path = None From 03bdbfc8ed43f64e25a9b103fa1874e774dbe362 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 13 Feb 2024 15:58:08 -0500 Subject: [PATCH 12/13] Provide more useful error messages on mistakes within [] --- confluent_server/confluent/noderange.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index cf99dd72..4a5cb808 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -402,12 +402,16 @@ class NodeRange(object): def _expandstring(self, element, filternodes=None): prefix = '' if element[0][0] in ('/', '~'): + if self.purenumeric: + raise Exception('Regular expression not supported within "[]"') element = ''.join(element) nameexpression = element[1:] if self.cfm is None: raise Exception('Verification configmanager required') return set(self.cfm.filter_nodenames(nameexpression, filternodes)) elif '=' in element[0] or '!~' in element[0]: + if self.purenumeric: + raise Exception('The "=" character is invalid within "[]"') element = ''.join(element) if self.cfm is None: raise Exception('Verification configmanager required') From 12bb5d583a7d58d2326521d455426ea887e1449e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 13 Feb 2024 16:00:50 -0500 Subject: [PATCH 13/13] Correct the equality message in better messagesw --- confluent_server/confluent/noderange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 4a5cb808..7657292c 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -411,7 +411,7 @@ class NodeRange(object): return set(self.cfm.filter_nodenames(nameexpression, filternodes)) elif '=' in element[0] or '!~' in element[0]: if self.purenumeric: - raise Exception('The "=" character is invalid within "[]"') + raise Exception('Equality/Inequality operators (=, !=, =~, !~) are invalid within "[]"') element = ''.join(element) if self.cfm is None: raise Exception('Verification configmanager required')