From db15992894d30b487525a8151f061e4d1b70d493 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Wed, 12 Apr 2017 10:36:27 -0400 Subject: [PATCH 1/5] Adding nodegroupattrib to update nodegroups --- confluent_client/bin/nodeattrib | 17 ++- confluent_client/bin/nodegroupattrib | 98 +++++++++++++ confluent_client/confluent/client.py | 133 +++++++++++++++--- .../confluent/config/configmanager.py | 32 +++-- 4 files changed, 242 insertions(+), 38 deletions(-) create mode 100644 confluent_client/bin/nodegroupattrib diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index 4d6678eb..c36942d2 100644 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -54,14 +54,18 @@ exitcode = 0 nodetype="noderange" if len(args) > 1: - exitcode=client.updateattrib(session,args,nodetype, noderange, options) + if "=" in args[1]: + exitcode=client.updateattrib(session,args,nodetype, noderange, options) try: # setting user output to what the user inputs if args[1] == 'all': showtype = 'all' + requestargs=args[2:] elif args[1] == 'current': showtype = 'current' - requestargs=args[2:] + requestargs=args[2:] + else: + requestargs=args[1:] except: pass @@ -76,8 +80,13 @@ if len(args) > 0: elif requestargs == []: #showtype already set pass - - exitcode = client.printattributes(session, args, requestargs, showtype,nodetype, noderange, options) + else: + try: + requestargs.remove('all') + requestargs.remove('current') + except ValueError: + pass + exitcode = client.printattributes(session, requestargs, showtype,nodetype, noderange, options) else: for res in session.read(nodelist): if 'error' in res: diff --git a/confluent_client/bin/nodegroupattrib b/confluent_client/bin/nodegroupattrib new file mode 100644 index 00000000..f0fa7051 --- /dev/null +++ b/confluent_client/bin/nodegroupattrib @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2017 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'alin37' + +import optparse +import os +import sys + +path = os.path.dirname(os.path.realpath(__file__)) +path = os.path.realpath(os.path.join(path, '..', 'lib', 'python')) +if path.startswith('/opt'): + sys.path.append(path) + +import confluent.client as client + +argparser = optparse.OptionParser( + usage='''\n %prog [options] \ + \n %prog [options] nodegroup [list of attributes] \ + \n %prog [options] nodegroup group=value1,value2 \ + \n %prog [options] nodegroup group=value1,value2 + \n ''') +argparser.add_option('-b', '--blame', action='store_true', + help='Show information about how attributes inherited') +argparser.add_option('-c', '--clear', action='store_true', + help='Clear variables') +(options, args) = argparser.parse_args() + + +#setting minimal output to only output current information +showtype = 'current' +requestargs=None +nodetype="nodegroups" + +try: + nodegroups = args[0] + nodelist = '/{0}/{1}/'.format(nodetype,nodegroups) +except IndexError: + nodelist = '/nodegroups/' +session = client.Command() +exitcode = 0 + +#Sets attributes + +if len(args) > 1: + exitcode=client.updateattrib(session,args,nodetype, nodegroups, options) + try: + # setting user output to what the user inputs + if args[1] == 'all': + showtype = 'all' + elif args[1] == 'current': + showtype = 'current' + + requestargs=args[1:] + except Exception as e: + print str(e) + +if exitcode != 0: + sys.exit(exitcode) + +# Lists all attributes +if len(args) > 0: + # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank. + if requestargs is None: + showtype = 'current' + elif requestargs == []: + #showtype already set + pass + else: + try: + requestargs.remove('all') + requestargs.remove('current') + except ValueError: + pass + exitcode = client.printgroupattributes(session, requestargs, showtype,nodetype, nodegroups, options) +else: + for res in session.read(nodelist): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + else: + print res['item']['href'].replace('/', '') + +sys.exit(exitcode) \ No newline at end of file diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py index cfa9d663..3d14ef90 100644 --- a/confluent_client/confluent/client.py +++ b/confluent_client/confluent/client.py @@ -42,7 +42,6 @@ def _parseserver(string): class Command(object): - def __init__(self, server=None): self._prevkeyname = None self.connection = None @@ -135,7 +134,29 @@ class Command(object): print('') return 0 - + def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs): + try: + rc = 0 + if resource[0] == '/': + resource = resource[1:] + # The implicit key is the resource basename + if key is None: + ikey = resource.rpartition('/')[-1] + else: + ikey = key + if input is None: + for res in self.read('/nodegroups/{0}/{1}'.format( + noderange, resource)): + rc = self.handle_results(ikey, rc, res) + else: + kwargs[ikey] = input + for res in self.update('/nodegroups/{0}/{1}'.format( + noderange, resource), kwargs): + rc = self.handle_results(ikey, rc, res) + return rc + except KeyboardInterrupt: + print('') + return 0 def read(self, path, parameters=None): if not self.authenticated: @@ -206,7 +227,7 @@ class Command(object): if knownhosts: certdata = self.connection.getpeercert(binary_form=True) fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest() - hostid = '@'.join((port,server)) + hostid = '@'.join((port, server)) khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384) if hostid in khf: if fingerprint == khf[hostid]: @@ -220,7 +241,6 @@ class Command(object): khf[hostid] = fingerprint - def send_request(operation, path, server, parameters=None): """This function iterates over all the responses received from the server. @@ -245,6 +265,7 @@ def send_request(operation, path, server, parameters=None): raise result = tlvdata.recv(server) + def attrrequested(attr, attrlist, seenattributes): for candidate in attrlist: truename = candidate @@ -258,10 +279,11 @@ def attrrequested(attr, attrlist, seenattributes): return True return False -def printattributes(session,args,requestargs,showtype,nodetype, noderange,options): - exitcode=0 + +def printattributes(session, requestargs, showtype, nodetype, noderange, options): + exitcode = 0 seenattributes = set([]) - for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype,noderange,showtype)): + for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)): if 'error' in res: sys.stderr.write(res['error'] + '\n') exitcode = 1 @@ -270,7 +292,7 @@ def printattributes(session,args,requestargs,showtype,nodetype, noderange,option for attr in res['databynode'][node]: seenattributes.add(attr) currattr = res['databynode'][node][attr] - if (requestargs is None or requestargs == [] or attrrequested(attr, args[1:], seenattributes)): + if (requestargs is None or requestargs == [] or attrrequested(attr, requestargs, seenattributes)): if 'value' in currattr: if currattr['value'] is not None: attrout = '{0}: {1}: {2}'.format( @@ -311,20 +333,78 @@ def printattributes(session,args,requestargs,showtype,nodetype, noderange,option print attrout if not exitcode: if requestargs: - for attr in args[1:]: + for attr in requestargs: if attr not in seenattributes: sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) exitcode = 1 return exitcode -def updateattrib(session,args,nodetype, noderange, options ): - #update attribute - exitcode=0 +def printgroupattributes(session, requestargs, showtype, nodetype, noderange, options): + exitcode = 0 + seenattributes = set([]) + for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + continue + for attr in res: + seenattributes.add(attr) + currattr = res[attr] + if (requestargs is None or requestargs == [] or attrrequested(attr, requestargs, seenattributes)): + if 'value' in currattr: + if currattr['value'] is not None: + attrout = '{0}: {1}: {2}'.format( + noderange, attr, currattr['value']) + else: + attrout = '{0}: {1}:'.format(noderange, attr) + elif 'isset' in currattr: + if currattr['isset']: + attrout = '{0}: {1}: ********'.format(noderange, attr) + else: + attrout = '{0}: {1}:'.format(noderange, attr) + elif 'broken' in currattr: + attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \ + '{2}'.format(noderange, attr, + currattr['broken']) + elif isinstance(currattr, list) or isinstance(currattr, tuple): + attrout = '{0}: {1}: {2}'.format(noderange, attr, ', '.join(map(str, currattr))) + elif isinstance(currattr, dict): + dictout = [] + for k, v in currattr.items: + dictout.append("{0}={1}".format(k, v)) + attrout = '{0}: {1}: {2}'.format(noderange, attr, ', '.join(map(str, dictout))) + else: + print ("CODE ERROR" + repr(attr)) + + if options.blame or 'broken' in currattr: + blamedata = [] + if 'inheritedfrom' in currattr: + blamedata.append('inherited from group {0}'.format( + currattr['inheritedfrom'] + )) + if 'expression' in currattr: + blamedata.append( + 'derived from expression "{0}"'.format( + currattr['expression'])) + if blamedata: + attrout += ' (' + ', '.join(blamedata) + ')' + print attrout + if not exitcode: + if requestargs: + for attr in requestargs: + if attr not in seenattributes: + sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) + exitcode = 1 + return exitcode + +def updateattrib(session, updateargs, nodetype, noderange, options): + # update attribute + exitcode = 0 if options.clear: - targpath = '/{0}/{1}/attributes/all'.format(nodetype,noderange) + targpath = '/{0}/{1}/attributes/all'.format(nodetype, noderange) keydata = {} - for attrib in args[1:]: + for attrib in updateargs[1:]: keydata[attrib] = None for res in session.update(targpath, keydata): if 'error' in res: @@ -333,18 +413,27 @@ def updateattrib(session,args,nodetype, noderange, options ): sys.stderr.write('Error: ' + res['error'] + '\n') sys.exit(exitcode) else: - if "=" in args[1]: + if "=" in updateargs[1]: try: - if len(args[1:]) > 1: - for val in args[1:]: + if len(updateargs[1:]) > 1: + for val in updateargs[1:]: val = val.split('=') - exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0]) + if (nodetype == "nodegroups"): + exitcode = session.simple_nodegroups_command(noderange, 'attributes/all'.format(noderange), + val[1],val[0]) + else: + exitcode = session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), + val[1], val[0]) else: - val=args[1].split('=') - exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0]) + val = updateargs[1].split('=') + if nodetype == "nodegroups" : + exitcode = session.simple_nodegroups_command(noderange, 'attributes/all'.format(noderange), + val[1], val[0]) + else: + exitcode = session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), + val[1], val[0]) except: - sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:]))) + sys.stderr.write('Error: {0} not a valid expression\n'.format(str(updateargs[1:]))) exitcode = 1 sys.exit(exitcode) return exitcode - diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 4a36a97d..329b8320 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -920,11 +920,13 @@ class ConfigManager(object): ('type' in allattributes.node[attr] and not isinstance(attribmap[group][attr], allattributes.node[attr]['type'])))): - raise ValueError + raise ValueError("nodes attribute is invalid") if attr == 'nodes': if not isinstance(attribmap[group][attr], list): - raise ValueError( - "nodes attribute on group must be list") + if type(attribmap[group][attr]) is unicode or type(attribmap[group][attr]) is str: + attribmap[group][attr]=attribmap[group][attr].split(",") + else: + raise ValueError("nodes attribute on group must be list") for node in attribmap[group]['nodes']: if node not in self._cfgstore['nodes']: raise ValueError( @@ -1136,7 +1138,21 @@ class ConfigManager(object): raise ValueError("node {0} does not exist".format(node)) for attrname in attribmap[node].iterkeys(): attrval = attribmap[node][attrname] - if (attrname not in allattributes.node or + try: + if (allattributes.node[attrname]['type'] == 'list' and + type(attrval) in (str, unicode)): + attrval = attrval.split(",") + except KeyError: + pass + if attrname == 'groups': + for group in attribmap[node]['groups'].split(","): + if group not in self._cfgstore['nodegroups']: + raise ValueError( + "group {0} does not exist".format(group)) + if ('everything' in self._cfgstore['nodegroups'] and + 'everything' not in attribmap[node]['groups']): + attribmap[node]['groups'].append('everything') + elif (attrname not in allattributes.node or ('type' in allattributes.node[attrname] and not isinstance( attrval, @@ -1144,14 +1160,6 @@ class ConfigManager(object): errstr = "{0} attribute on node {1} is invalid".format( attrname, node) raise ValueError(errstr) - if attrname == 'groups': - for group in attribmap[node]['groups']: - if group not in self._cfgstore['nodegroups']: - raise ValueError( - "group {0} does not exist".format(group)) - if ('everything' in self._cfgstore['nodegroups'] and - 'everything' not in attribmap[node]['groups']): - attribmap[node]['groups'].append('everything') for node in attribmap.iterkeys(): node = node.encode('utf-8') exprmgr = None From 43211cf9a7fd040e4ac070e70f12331ce9c96e8a Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Wed, 12 Apr 2017 10:50:44 -0400 Subject: [PATCH 2/5] Updating file permissions on nodegroupattrib and nodeattrib --- confluent_client/bin/nodeattrib | 0 confluent_client/bin/nodegroupattrib | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 confluent_client/bin/nodeattrib mode change 100644 => 100755 confluent_client/bin/nodegroupattrib diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib old mode 100644 new mode 100755 diff --git a/confluent_client/bin/nodegroupattrib b/confluent_client/bin/nodegroupattrib old mode 100644 new mode 100755 From 2042bcb8f096c762acd1dfd2178177a06831fa45 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 13 Apr 2017 17:36:41 -0400 Subject: [PATCH 3/5] updating server side for use with nodegroupattrib --- confluent_server/confluent/config/configmanager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 329b8320..7a54895a 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -896,6 +896,11 @@ class ConfigManager(object): if node not in self._cfgstore['nodes']: self._cfgstore['nodes'][node] = {'groups': [group]} _mark_dirtykey('nodes', node, self.tenant) + elif 'value' in self._cfgstore['nodes'][node]['groups']: + tmplist=self._cfgstore['nodes'][node]['groups']['value'].split(",") + tmplist.insert(0, unicode(group)) + self._cfgstore['nodes'][node]['groups']['value']=",".join(tmplist) + _mark_dirtykey('nodes', node, self.tenant) elif group not in self._cfgstore['nodes'][node]['groups']: self._cfgstore['nodes'][node]['groups'].insert(0, group) _mark_dirtykey('nodes', node, self.tenant) @@ -1138,14 +1143,18 @@ class ConfigManager(object): raise ValueError("node {0} does not exist".format(node)) for attrname in attribmap[node].iterkeys(): attrval = attribmap[node][attrname] + print attrval + print allattributes.node[attrname] try: if (allattributes.node[attrname]['type'] == 'list' and - type(attrval) in (str, unicode)): + type(attrval) in (str, unicode)) and type(attrval)==list: attrval = attrval.split(",") except KeyError: pass if attrname == 'groups': - for group in attribmap[node]['groups'].split(","): + if type(attribmap[node]['groups']) != list: + attribmap[node]['groups']=attribmap[node]['groups'].split(",") + for group in attribmap[node]['groups']: if group not in self._cfgstore['nodegroups']: raise ValueError( "group {0} does not exist".format(group)) From 86e502a47a87cb6ddc2062c8467fa471af7acde2 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 13 Apr 2017 22:07:47 -0400 Subject: [PATCH 4/5] remove debug and fixing issue checking attribute --- confluent_server/confluent/config/configmanager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 7a54895a..71efda93 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -1143,11 +1143,9 @@ class ConfigManager(object): raise ValueError("node {0} does not exist".format(node)) for attrname in attribmap[node].iterkeys(): attrval = attribmap[node][attrname] - print attrval - print allattributes.node[attrname] try: if (allattributes.node[attrname]['type'] == 'list' and - type(attrval) in (str, unicode)) and type(attrval)==list: + type(attrval) in (str, unicode)): attrval = attrval.split(",") except KeyError: pass From b64858e0b3c9d971cd829593294f4a8340ccc242 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 20 Apr 2017 15:41:11 -0400 Subject: [PATCH 5/5] Removing unnecessary case if groups ever has a key of 'value' --- confluent_server/confluent/config/configmanager.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 71efda93..d9600a3a 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -896,11 +896,6 @@ class ConfigManager(object): if node not in self._cfgstore['nodes']: self._cfgstore['nodes'][node] = {'groups': [group]} _mark_dirtykey('nodes', node, self.tenant) - elif 'value' in self._cfgstore['nodes'][node]['groups']: - tmplist=self._cfgstore['nodes'][node]['groups']['value'].split(",") - tmplist.insert(0, unicode(group)) - self._cfgstore['nodes'][node]['groups']['value']=",".join(tmplist) - _mark_dirtykey('nodes', node, self.tenant) elif group not in self._cfgstore['nodes'][node]['groups']: self._cfgstore['nodes'][node]['groups'].insert(0, group) _mark_dirtykey('nodes', node, self.tenant)