From 27d5dbae081ae2198114bf10c1040d7745babb1a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sun, 18 Aug 2013 19:11:49 -0400 Subject: [PATCH] Rework a few things, get the config module to the point of basic functionality --- confluent/config.py | 197 +++++++++++++++++++------ confluent/{cryptutils.py => crypto.py} | 14 +- confluent/httpapi.py | 2 +- 3 files changed, 158 insertions(+), 55 deletions(-) rename confluent/{cryptutils.py => crypto.py} (91%) diff --git a/confluent/config.py b/confluent/config.py index 1b4d58b2..2483879c 100644 --- a/confluent/config.py +++ b/confluent/config.py @@ -2,40 +2,121 @@ # All rights reserved +# Ultimately, the design is to handle all the complicated stuff at set +# rather than get tiime. When something is set on a group, then all +# members of that group are examined and 'inheritedfrom' attributes +# are pushed. as expression definned values are iinserted, their +# dependdentt attributes are added to a private dict to aid in auto +# calculation. When a name is changed, all attributes are re-evaluated +# on get, should be simple read value *except* for encrypted values, +# which are only decrypted when explicitly requested +# encrypted fields do not support expressions, either as a source or +# destination + +# For multi-node operation, each instance opens and retains a TLS connection +# to each other instance. 'set' operations push to queue for writeback and +# returns. The writeback thread writes to local disk and to other instances. +# A function is provided to wait for pending output to disk and peers to complete +# to assure that aa neww requesst to peer does not beat configuration data to +# the target + +# on disk format is cpickle. No data shall be in the configuration db required +# to get started. For example, argv shall indicate ports rather than cfg store + # Note on the cryptography. Default behavior is mostly just to pave the -# way to meaningful security. Root all potentially sensitive data in +# way to meaningful security. Root all potentially sensitive data in # one key. That key is in plain sight, so not meaningfully protected # However, the key can be protected in the following ways: # - Passphrase protected (requiring human interaction every restart) # - TPM sealing (which would forgo the interactive assuming risk of # physical attack on TPM is not a concern) -# This time around, expression based values will be parsed when set, and the -# parsing results will be stored rather than parsing on every evaluation -# Additionally, the option will be made available to use other attributes -# as well as the $1, $2, etc extracted from nodename. Left hand side can -# be requested to customize $1 and $2, but it is not required - -#Actually, may override one of the python string formatters: -# 2.6 String.Formatter, e.g. "hello {world}" -# 2.4 string.Template, e.g. "hello $world" - -# In JSON mode, will just read and write entire thing, with a comment -# to dissuade people from hand editing. - -# In JSON mode, a file for different categories (site, nodes, etc) -# in redis, each category is a different database number import array +import ast import collections +import copy import math +import operator import os +import re +import string -_masterintegritykey = None -_cfgstore = {} -def _expand_expression(attribute, nodeobj): +class _ExpressionFormat(string.Formatter): + posmatch = re.compile('^n([0-9]*)$') + nummatch = re.compile('[0-9]+') + _supported_ops = { + ast.Mult: operator.mul, + ast.Div: operator.floordiv, + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.LShift: operator.lshift, + ast.RShift: operator.rshift, + ast.BitAnd: operator.and_, + ast.BitXor: operator.xor, + ast.BitOr: operator.or_, + } + + def __init__(self, nodeobj): + self._nodeobj = nodeobj + self._numbers = re.findall(self.nummatch, nodeobj['name']['value']) + + def get_field(self, field_name, args, kwargs): + parsed = ast.parse(field_name) + return (self._handle_ast_node(parsed.body[0].value), field_name) + + def _handle_ast_node(self, node): + if isinstance(node, ast.Num): + return node.n + elif isinstance(node, ast.Attribute): + #ok, we have something with a dot + left = node.value.id + right = node.attr + key = left + '.' + right + val = _decode_attribute(key, self._nodeobj, + formatter=self) + return val['value'] if 'value' in val else "" + elif isinstance(node, ast.Name): + var = node.id + if var == 'nodename': + return self._nodeobj['name']['value'] + mg = re.match(self.posmatch, var) + if mg: + idx = int(mg.group(1)) + return int(self._numbers[idx - 1]) + else: + if var in self._nodeobj: + val = _decode_attribute(var, self._nodeobj, + formatter=self) + return val['value'] if 'value' in val else "" + elif isinstance(node, ast.BinOp): + optype = type(node.op) + if optype not in self._supported_ops: + raise Exception("Unsupported operation") + op = self._supported_ops[optype] + return op(self._handle_ast_node(node.left), + self._handle_ast_node(node.right)) + + +def _decode_attribute(attribute, nodeobj, formatter, decrypt=False): + if attribute not in nodeobj: + return None + if 'value' in nodeobj[attribute]: + return nodeobj[attribute] + elif 'expression' in nodeobj[attribute]: + retdict = copy.deepcopy(nodeobj[attribute]) + retdict['value'] = formatter.format(retdict['expression']) + return retdict + elif 'cryptvalue' in nodeobj[attribute] and decrypt: + retdict = copy.deepcopy(nodeobj[attribute]) + retdict['value'] = crypto.decrypt_value( + nodeobj[attribute]['cryptvalue']) + return nodeobj[attribute] + + +def _expand_expression(attribute, nodeobj, decrypt=False): # here is where we may avail ourselves of string.Formatter or # string.Template # we would then take the string that is identifier and do @@ -98,36 +179,58 @@ def _expand_expression(attribute, nodeobj): # raise TypeError(node) pass +_cfgstore = {} -class NodeAttribs(object): - def __init__(self, nodes=[], attributes=[], tenant=0): - self._nodelist = collecitons.dequeue(nodes) +# my thinking at this point is that noderange and configdata objects +# will be constructed and passed as part of a context object to plugins +# reasoning being that the main program will handle establishing the +# tenant context and then modules need not consider the current tenant +# most of the time as things are automatic + +class ConfigData(object): + def __init__(self, tenant=0, decrypt=False): self._tenant = tenant - self._attributes=attributes + self.decrypt = decrypt - def __iter__(self): - return self - - def next(): - node = self._nodelist.popleft() - onodeobj = _cfgstore['node'][(self._tenant,node)] - nodeobj = - attriblist = [] - #if there is a filter, delete irrelevant keys - if self._attributes.length > 0: - for attribute in nodeobj.keys(): - if attribute not in self._attributes: - del nodeobj[attribute] - #now that attributes are filtered, seek out and evaluate expressions - for attribute in nodeobj.keys(): - if ('value' not in nodeobj[attribute] and - 'cryptvalue' in nodeobj[attribute]): - nodeobj[attribute]['value'] = _decrypt_value( - nodeobj[attribute]['cryptvalue']) - if ('value' not in nodeobj[attribute] and - 'expression' in nodeobj[attribute]): - nodeobj[attribute]['value'] = _expand_expression( - attribute=attribute, - nodeobj=nodeobj) + def get_node_attributes(self, nodelist, attributes=[]): + if 'node' not in _cfgstore: + return None + retdict = {} + if isinstance(nodelist,str): + nodelist = [nodelist] + for node in nodelist: + if (self._tenant,node) not in _cfgstore['node']: + continue + cfgnodeobj = _cfgstore['node'][(self._tenant,node)] + exprmgr = _ExpressionFormat(cfgnodeobj) + nodeobj = {} + if len(attributes) == 0: + attributes = cfgnodeobj.keys() + for attribute in attributes: + if attribute not in cfgnodeobj: + continue + nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj, + formatter=exprmgr, + decrypt=self.decrypt) + retdict[node] = nodeobj + return retdict + def set_node_attributes(self, attribmap): + if 'node' not in _cfgstore: + _cfgstore['node'] = {} + for node in attribmap.keys(): + key = (self._tenant, node) + if key not in _cfgstore['node']: + _cfgstore['node'][key] = {'name': {'value': node}} + for attrname in attribmap[node].keys(): + newdict = {} + if isinstance(attribmap[node][attrname], dict): + newdict = attribmap[node][attrname] + else: + newdict = {'value': attribmap[node][attrname] } + if 'value' in newdict and attrname.startswith("credential"): + newdict['cryptvalue' ] = \ + crypto.crypt_value(newdict['value']) + del newdict['value'] + _cfgstore['node'][key][attrname] = newdict diff --git a/confluent/cryptutils.py b/confluent/crypto.py similarity index 91% rename from confluent/cryptutils.py rename to confluent/crypto.py index 8516315a..a5e74f93 100644 --- a/confluent/cryptutils.py +++ b/confluent/crypto.py @@ -75,23 +75,23 @@ def _format_key(key, passphrase=None): return {"unencryptedvalue": key} -def _init_masterkey(passphrase=None): - if 'master_privacy_key' in _cfgstore['globals']: +def init_masterkey(cfgstore, passphrase=None, cfgstore): + if 'master_privacy_key' in cfgstore['globals']: _masterkey = _get_protected_key( - _cfgstore['globals']['master_privacy_key'], + cfgstore['globals']['master_privacy_key'], passphrase=passphrase) else: _masterkey = os.urandom(32) - _cfgstore['globals']['master_privacy_key'] = _format_key(_masterkey, + cfgstore['globals']['master_privacy_key'] = _format_key(_masterkey, passphrase=passphrase) - if 'master_integrity_key' in _cfgstore['globals']: + if 'master_integrity_key' in cfgstore['globals']: _masterintegritykey = _get_protected_key( - _cfgstore['globals']['master_integrity_key'], + cfgstore['globals']['master_integrity_key'], passphrase=passphrase ) else: _masterintegritykey = os.urandom(64) - _cfgstore['globals']['master_integrity_key'] = _format_key( + cfgstore['globals']['master_integrity_key'] = _format_key( _masterintegritykey, passphrase=passphrase ) diff --git a/confluent/httpapi.py b/confluent/httpapi.py index edc6986f..7bc2dfd6 100644 --- a/confluent/httpapi.py +++ b/confluent/httpapi.py @@ -49,7 +49,7 @@ def _pick_mimetype(env): Note that as it gets into the ACCEPT header honoring, it only looks for application/json and else gives up and assumes html. This is because - browsers are very chaotic about ACCEPT header. It is assumed that + browsers are very chaotic about ACCEPT HEADER. It is assumed that XMLHttpRequest.setRequestHeader will be used by clever javascript if the '.json' scheme doesn't cut it. """