From d9ffa10422071a58605157efcb1f06c9509af983 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 6 Nov 2017 10:24:32 -0500 Subject: [PATCH 1/6] Restore power query function Implement it in a fashion that does not block confetty while long running power query situations happen. --- confluent_client/bin/confetty | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/confluent_client/bin/confetty b/confluent_client/bin/confetty index 59dcac2f..8ef69a4e 100755 --- a/confluent_client/bin/confetty +++ b/confluent_client/bin/confetty @@ -115,6 +115,7 @@ def print_help(): def updatestatus(stateinfo={}): + global powerstate, powertime status = consolename info = [] for statekey in stateinfo: @@ -129,6 +130,11 @@ def updatestatus(stateinfo={}): # error will be repeated if relevant # avoid keeping it around as stale del laststate['error'] + if 'state' in stateinfo: # currently only read power means anything + newpowerstate = stateinfo['state']['value'] + if newpowerstate != powerstate and newpowerstate == 'off': + sys.stdout.write("\x1b[2J\x1b[;H[powered off]\r\n") + powerstate = newpowerstate if 'clientcount' in laststate and laststate['clientcount'] != 1: info.append('clients: %d' % laststate['clientcount']) if 'bufferage' in stateinfo and stateinfo['bufferage'] is not None: @@ -715,7 +721,7 @@ def conserver_command(filehandle, localcommand): else: print("Unknown power state.]\r") - #check_power_state() + check_power_state() elif localcommand[0] == '?': print("help]\r") @@ -830,21 +836,11 @@ powertime = None def check_power_state(): - global powerstate, powertime - for rsp in session.read('/nodes/' + consolename + '/power/state'): - if type(rsp) == dict and 'state' in rsp: - newpowerstate = rsp['state']['value'] - powertime = time.time() - if newpowerstate != powerstate and newpowerstate == 'off': - sys.stdout.write("\x1b[2J\x1b[;H[powered off]\r\n") - powerstate = newpowerstate - elif type(rsp) == dict and '_requestdone' in rsp: - break - elif type(rsp) == dict: - updatestatus(rsp) - else: - sys.stdout.write(rsp) - sys.stdout.flush() + tlvdata.send( + session.connection, + {'operation': 'retrieve', + 'path': '/nodes/' + consolename + '/power/state'}) + return while inconsole or not doexit: @@ -908,8 +904,10 @@ while inconsole or not doexit: tlvdata.send(session.connection, myinput) except IOError: pass - #if powerstate is None or powertime < time.time() - 60: # Check powerstate every 60 seconds - # check_power_state() + if powerstate is None or powertime < time.time() - 60: # Check powerstate every 60 seconds + powertime = time.time() + powerstate = True + check_power_state() else: currcommand = prompt() try: From 148329dd8e95d805792c6163b4957a9ce148f3b3 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 6 Nov 2017 11:09:55 -0500 Subject: [PATCH 2/6] Implement send resize commands Craft correct stty commands based on terminal size detected by confluent. --- confluent_client/bin/confetty | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/confluent_client/bin/confetty b/confluent_client/bin/confetty index 8ef69a4e..e490ebb1 100755 --- a/confluent_client/bin/confetty +++ b/confluent_client/bin/confetty @@ -722,7 +722,22 @@ def conserver_command(filehandle, localcommand): print("Unknown power state.]\r") check_power_state() - + elif localcommand[0] == 'r': + sys.stdout.write('\x1b7\x1b[999;999H\x1b[6n') + sys.stdout.flush() + reply = '' + while 'R' not in reply: + try: + reply += sys.stdin.read(1) + except IOError: + pass + reply = reply.replace('\x1b[', '') + reply = reply.replace('R', '') + height, width = reply.split(';') + sys.stdout.write('\x1b8') + sys.stdout.flush() + print('sending stty commands]') + return 'stty columns {0}\rstty rows {1}\r'.format(width, height) elif localcommand[0] == '?': print("help]\r") print(". exit console\r") @@ -733,6 +748,7 @@ def conserver_command(filehandle, localcommand): print("pbs boot to setup\r") print("pbn boot to network\r") print("pb boot to default\r") + print("r send stty command to resize terminal\r") print(" abort command\r") elif localcommand[0] == '\x0d': print("ignored]\r") From f5889e70292399b7325206e79c0473d6e7f651c7 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 6 Nov 2017 14:51:59 -0500 Subject: [PATCH 3/6] Add '-c' to manage noderun/nodeshell concurrency Allow user to specify custom parallel count. --- confluent_client/bin/noderun | 10 ++++++---- confluent_client/bin/nodeshell | 11 +++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/confluent_client/bin/noderun b/confluent_client/bin/noderun index 02e22eac..19d8f133 100755 --- a/confluent_client/bin/noderun +++ b/confluent_client/bin/noderun @@ -37,18 +37,20 @@ import confluent.client as client def run(): - concurrentprocs = 168 - # among other things, FD_SETSIZE limits. Besides, spawning too many - # processes can be unkind for the unaware on memory pressure and such... argparser = optparse.OptionParser( - usage="Usage: %prog node commandexpression", + usage="Usage: %prog noderange commandexpression", epilog="Expressions are the same as in attributes, e.g. " "'ipmitool -H {hardwaremanagement.manager}' will be expanded.") + argparser.add_option('-c', '--count', type='int', default=168, + help='Number of commands to run at a time') + # among other things, FD_SETSIZE limits. Besides, spawning too many + # processes can be unkind for the unaware on memory pressure and such... argparser.disable_interspersed_args() (options, args) = argparser.parse_args() if len(args) < 2: argparser.print_help() sys.exit(1) + concurrentprocs = options.count c = client.Command() cmdstr = " ".join(args[1:]) diff --git a/confluent_client/bin/nodeshell b/confluent_client/bin/nodeshell index 061bc869..6b8f6200 100755 --- a/confluent_client/bin/nodeshell +++ b/confluent_client/bin/nodeshell @@ -36,18 +36,21 @@ import confluent.client as client def run(): - concurrentprocs = 168 - # among other things, FD_SETSIZE limits. Besides, spawning too many - # processes can be unkind for the unaware on memory pressure and such... + argparser = optparse.OptionParser( - usage="Usage: %prog node commandexpression", + usage="Usage: %prog noderange commandexpression", epilog="Expressions are the same as in attributes, e.g. " "'ipmitool -H {hardwaremanagement.manager}' will be expanded.") + argparser.add_option('-c', '--count', type='int', default=168, + help='Number of commands to run at a time') + # among other things, FD_SETSIZE limits. Besides, spawning too many + # processes can be unkind for the unaware on memory pressure and such... argparser.disable_interspersed_args() (options, args) = argparser.parse_args() if len(args) < 2: argparser.print_help() sys.exit(1) + concurrentprocs = options.count c = client.Command() cmdstr = " ".join(args[1:]) From fcae11bf96affef383749a4fa051a8a5aad592f1 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 6 Nov 2017 14:57:15 -0500 Subject: [PATCH 4/6] Do natural sort on expression expansion This creates more logical behavior from nodeshell and noderun when dealing with many nodes, particularly when crossing the concurrency limit. --- .../confluent/plugins/configuration/attributes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/plugins/configuration/attributes.py b/confluent_server/confluent/plugins/configuration/attributes.py index 1139f3f5..06e061c0 100644 --- a/confluent_server/confluent/plugins/configuration/attributes.py +++ b/confluent_server/confluent/plugins/configuration/attributes.py @@ -16,6 +16,7 @@ import confluent.exceptions as exc import confluent.messages as msg import confluent.config.attributes as allattributes +import confluent.util as util def retrieve(nodes, element, configmanager, inputdata): @@ -175,8 +176,11 @@ def _expand_expression(nodes, configmanager, inputdata): expression = expression['expression'] if type(expression) is dict: expression = expression['expression'] + pernodeexpressions = {} for expanded in configmanager.expand_attrib_expression(nodes, expression): - yield msg.KeyValueData({'value': expanded[1]}, expanded[0]) + pernodeexpressions[expanded[0]] = expanded[1] + for node in util.natural_sort(pernodeexpressions): + yield msg.KeyValueData({'value': pernodeexpressions[node]}, node) def create(nodes, element, configmanager, inputdata): From cc4950ef758a661135ea5e5ab20710dc487681e9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 6 Nov 2017 15:47:59 -0500 Subject: [PATCH 5/6] Opportunistically grab and sort by node If output comes close enough together, make some effort to group it so that it will have a higher tendency of looking orderly. This of course only does so when it does not interfere with quickly presenting the data. --- confluent_client/bin/noderun | 43 ++++++++++++++++---------- confluent_client/bin/nodeshell | 43 ++++++++++++++++---------- confluent_client/confluent/sortutil.py | 43 ++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 confluent_client/confluent/sortutil.py diff --git a/confluent_client/bin/noderun b/confluent_client/bin/noderun index 19d8f133..8e380d84 100755 --- a/confluent_client/bin/noderun +++ b/confluent_client/bin/noderun @@ -34,6 +34,7 @@ if path.startswith('/opt'): sys.path.append(path) import confluent.client as client +import confluent.sortutil as sortutil def run(): @@ -78,26 +79,34 @@ def run(): sys.exit(exitcode) rdy, _, _ = select.select(all, [], [], 10) while all: + pernodeout = {} for r in rdy: - data = r.readline() desc = pipedesc[r] - if data: - node = desc['node'] - if desc['type'] == 'stdout': - sys.stdout.write('{0}: {1}'.format(node,data)) - sys.stdout.flush() + node = desc['node'] + data = True + while data and select.select([r], [], [], 0): + data = r.readline() + if data: + if desc['type'] == 'stdout': + if node not in pernodeout: + pernodeout[node] = [] + pernodeout[node].append(data) + else: + sys.stderr.write('{0}: {1}'.format(node, data)) + sys.stderr.flush() else: - sys.stderr.write('{0}: {1}'.format(node, data)) - sys.stderr.flush() - else: - pop = desc['popen'] - ret = pop.poll() - if ret is not None: - exitcode = exitcode | ret - all.discard(r) - if desc['type'] == 'stdout' and pendingexecs: - node, cmdv = pendingexecs.popleft() - run_cmdv(node, cmdv, all, pipedesc) + pop = desc['popen'] + ret = pop.poll() + if ret is not None: + exitcode = exitcode | ret + all.discard(r) + if desc['type'] == 'stdout' and pendingexecs: + node, cmdv = pendingexecs.popleft() + run_cmdv(node, cmdv, all, pipedesc) + for node in sortutil.natural_sort(pernodeout): + for line in pernodeout[node]: + sys.stdout.write('{0}: {1}'.format(node, line)) + sys.stdout.flush() if all: rdy, _, _ = select.select(all, [], [], 10) sys.exit(exitcode) diff --git a/confluent_client/bin/nodeshell b/confluent_client/bin/nodeshell index 6b8f6200..95706c55 100755 --- a/confluent_client/bin/nodeshell +++ b/confluent_client/bin/nodeshell @@ -33,6 +33,7 @@ if path.startswith('/opt'): sys.path.append(path) import confluent.client as client +import confluent.sortutil as sortutil def run(): @@ -79,26 +80,34 @@ def run(): sys.exit(exitcode) rdy, _, _ = select.select(all, [], [], 10) while all: + pernodeout = {} for r in rdy: - data = r.readline() desc = pipedesc[r] - if data: - node = desc['node'] - if desc['type'] == 'stdout': - sys.stdout.write('{0}: {1}'.format(node,data)) - sys.stdout.flush() + node = desc['node'] + data = True + while data and select.select([r], [], [], 0): + data = r.readline() + if data: + if desc['type'] == 'stdout': + if node not in pernodeout: + pernodeout[node] = [] + pernodeout[node].append(data) + else: + sys.stderr.write('{0}: {1}'.format(node, data)) + sys.stderr.flush() else: - sys.stderr.write('{0}: {1}'.format(node, data)) - sys.stderr.flush() - else: - pop = desc['popen'] - ret = pop.poll() - if ret is not None: - exitcode = exitcode | ret - all.discard(r) - if desc['type'] == 'stdout' and pendingexecs: - node, cmdv = pendingexecs.popleft() - run_cmdv(node, cmdv, all, pipedesc) + pop = desc['popen'] + ret = pop.poll() + if ret is not None: + exitcode = exitcode | ret + all.discard(r) + if desc['type'] == 'stdout' and pendingexecs: + node, cmdv = pendingexecs.popleft() + run_cmdv(node, cmdv, all, pipedesc) + for node in sortutil.natural_sort(pernodeout): + for line in pernodeout[node]: + sys.stdout.ouwrite('{0}: {1}'.format(node, line)) + sys.stdout.flush() if all: rdy, _, _ = select.select(all, [], [], 10) sys.exit(exitcode) diff --git a/confluent_client/confluent/sortutil.py b/confluent_client/confluent/sortutil.py new file mode 100644 index 00000000..109eacbc --- /dev/null +++ b/confluent_client/confluent/sortutil.py @@ -0,0 +1,43 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014 IBM Corporation +# Copyright 2015-2016 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +numregex = re.compile('([0-9]+)') + + +def naturalize_string(key): + """Analyzes string in a human way to enable natural sort + + :param key: The node name to analyze + :returns: A structure that can be consumed by 'sorted' + """ + return [int(text) if text.isdigit() else text.lower() + for text in re.split(numregex, key)] + + +def natural_sort(iterable): + """Return a sort using natural sort if possible + + :param iterable: + :return: + """ + try: + return sorted(iterable, key=naturalize_string) + except TypeError: + # The natural sort attempt failed, fallback to ascii sort + return sorted(iterable) From 80864d78b3be9dad785b0831df6a0fcaa9c5b253 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 8 Nov 2017 13:36:36 -0500 Subject: [PATCH 6/6] Remove unused nestedmode Without the ability to reliably tell a nodename from an input key, must not do the nestedmode detection. --- confluent_server/confluent/messages.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/confluent_server/confluent/messages.py b/confluent_server/confluent/messages.py index e81060e4..b89f4505 100644 --- a/confluent_server/confluent/messages.py +++ b/confluent_server/confluent/messages.py @@ -446,23 +446,13 @@ class InputExpression(ConfluentMessage): # so that it can make it intact to the pertinent configmanager function def __init__(self, path, inputdata, nodes=None): self.nodeattribs = {} - nestedmode = False if not inputdata: raise exc.InvalidArgumentException('no request data provided') if nodes is None: self.attribs = inputdata return for node in nodes: - if node in inputdata: - nestedmode = True - self.nodeattribs[node] = inputdata[node] - if nestedmode: - for key in inputdata: - if key not in nodes: - raise exc.InvalidArgumentException - else: - for node in nodes: - self.nodeattribs[node] = inputdata + self.nodeattribs[node] = inputdata def get_attributes(self, node): if node not in self.nodeattribs: @@ -484,7 +474,6 @@ class InputAttributes(ConfluentMessage): # to the client def __init__(self, path, inputdata, nodes=None): self.nodeattribs = {} - nestedmode = False if not inputdata: raise exc.InvalidArgumentException('no request data provided') if nodes is None: @@ -509,16 +498,7 @@ class InputAttributes(ConfluentMessage): 'expression': self.attribs[attrib]} return for node in nodes: - if node in inputdata: - nestedmode = True - self.nodeattribs[node] = inputdata[node] - if nestedmode: - for key in inputdata: - if key not in nodes: - raise exc.InvalidArgumentException - else: - for node in nodes: - self.nodeattribs[node] = inputdata + self.nodeattribs[node] = inputdata def get_attributes(self, node): if node not in self.nodeattribs: @@ -555,7 +535,6 @@ class InputCredential(ConfluentMessage): def __init__(self, path, inputdata, nodes=None): self.credentials = {} - nestedmode = False if not inputdata: raise exc.InvalidArgumentException('no request data provided')