mirror of
https://github.com/xcat2/confluent.git
synced 2024-11-25 19:10:10 +00:00
Rework a few things, get the config module to the point of basic functionality
This commit is contained in:
Jarrod Johnson
parent
dc8c9d091c
commit
27d5dbae08
@ -2,40 +2,121 @@
|
||||
# All rights reserved
|
||||
|
||||
|
||||
# Ultimately, the design is to handle all the complicated stuff at set
|
||||
# rather than get tiime. When something is set on a group, then all
|
||||
# members of that group are examined and 'inheritedfrom' attributes
|
||||
# are pushed. as expression definned values are iinserted, their
|
||||
# dependdentt attributes are added to a private dict to aid in auto
|
||||
# calculation. When a name is changed, all attributes are re-evaluated
|
||||
# on get, should be simple read value *except* for encrypted values,
|
||||
# which are only decrypted when explicitly requested
|
||||
# encrypted fields do not support expressions, either as a source or
|
||||
# destination
|
||||
|
||||
# For multi-node operation, each instance opens and retains a TLS connection
|
||||
# to each other instance. 'set' operations push to queue for writeback and
|
||||
# returns. The writeback thread writes to local disk and to other instances.
|
||||
# A function is provided to wait for pending output to disk and peers to complete
|
||||
# to assure that aa neww requesst to peer does not beat configuration data to
|
||||
# the target
|
||||
|
||||
# on disk format is cpickle. No data shall be in the configuration db required
|
||||
# to get started. For example, argv shall indicate ports rather than cfg store
|
||||
|
||||
# Note on the cryptography. Default behavior is mostly just to pave the
|
||||
# way to meaningful security. Root all potentially sensitive data in
|
||||
# way to meaningful security. Root all potentially sensitive data in
|
||||
# one key. That key is in plain sight, so not meaningfully protected
|
||||
# However, the key can be protected in the following ways:
|
||||
# - Passphrase protected (requiring human interaction every restart)
|
||||
# - TPM sealing (which would forgo the interactive assuming risk of
|
||||
# physical attack on TPM is not a concern)
|
||||
|
||||
# This time around, expression based values will be parsed when set, and the
|
||||
# parsing results will be stored rather than parsing on every evaluation
|
||||
# Additionally, the option will be made available to use other attributes
|
||||
# as well as the $1, $2, etc extracted from nodename. Left hand side can
|
||||
# be requested to customize $1 and $2, but it is not required
|
||||
|
||||
#Actually, may override one of the python string formatters:
|
||||
# 2.6 String.Formatter, e.g. "hello {world}"
|
||||
# 2.4 string.Template, e.g. "hello $world"
|
||||
|
||||
# In JSON mode, will just read and write entire thing, with a comment
|
||||
# to dissuade people from hand editing.
|
||||
|
||||
# In JSON mode, a file for different categories (site, nodes, etc)
|
||||
# in redis, each category is a different database number
|
||||
|
||||
import array
|
||||
import ast
|
||||
import collections
|
||||
import copy
|
||||
import math
|
||||
import operator
|
||||
import os
|
||||
import re
|
||||
import string
|
||||
|
||||
|
||||
_masterintegritykey = None
|
||||
_cfgstore = {}
|
||||
|
||||
def _expand_expression(attribute, nodeobj):
|
||||
class _ExpressionFormat(string.Formatter):
|
||||
posmatch = re.compile('^n([0-9]*)$')
|
||||
nummatch = re.compile('[0-9]+')
|
||||
_supported_ops = {
|
||||
ast.Mult: operator.mul,
|
||||
ast.Div: operator.floordiv,
|
||||
ast.Add: operator.add,
|
||||
ast.Sub: operator.sub,
|
||||
ast.LShift: operator.lshift,
|
||||
ast.RShift: operator.rshift,
|
||||
ast.BitAnd: operator.and_,
|
||||
ast.BitXor: operator.xor,
|
||||
ast.BitOr: operator.or_,
|
||||
}
|
||||
|
||||
def __init__(self, nodeobj):
|
||||
self._nodeobj = nodeobj
|
||||
self._numbers = re.findall(self.nummatch, nodeobj['name']['value'])
|
||||
|
||||
def get_field(self, field_name, args, kwargs):
|
||||
parsed = ast.parse(field_name)
|
||||
return (self._handle_ast_node(parsed.body[0].value), field_name)
|
||||
|
||||
def _handle_ast_node(self, node):
|
||||
if isinstance(node, ast.Num):
|
||||
return node.n
|
||||
elif isinstance(node, ast.Attribute):
|
||||
#ok, we have something with a dot
|
||||
left = node.value.id
|
||||
right = node.attr
|
||||
key = left + '.' + right
|
||||
val = _decode_attribute(key, self._nodeobj,
|
||||
formatter=self)
|
||||
return val['value'] if 'value' in val else ""
|
||||
elif isinstance(node, ast.Name):
|
||||
var = node.id
|
||||
if var == 'nodename':
|
||||
return self._nodeobj['name']['value']
|
||||
mg = re.match(self.posmatch, var)
|
||||
if mg:
|
||||
idx = int(mg.group(1))
|
||||
return int(self._numbers[idx - 1])
|
||||
else:
|
||||
if var in self._nodeobj:
|
||||
val = _decode_attribute(var, self._nodeobj,
|
||||
formatter=self)
|
||||
return val['value'] if 'value' in val else ""
|
||||
elif isinstance(node, ast.BinOp):
|
||||
optype = type(node.op)
|
||||
if optype not in self._supported_ops:
|
||||
raise Exception("Unsupported operation")
|
||||
op = self._supported_ops[optype]
|
||||
return op(self._handle_ast_node(node.left),
|
||||
self._handle_ast_node(node.right))
|
||||
|
||||
|
||||
def _decode_attribute(attribute, nodeobj, formatter, decrypt=False):
|
||||
if attribute not in nodeobj:
|
||||
return None
|
||||
if 'value' in nodeobj[attribute]:
|
||||
return nodeobj[attribute]
|
||||
elif 'expression' in nodeobj[attribute]:
|
||||
retdict = copy.deepcopy(nodeobj[attribute])
|
||||
retdict['value'] = formatter.format(retdict['expression'])
|
||||
return retdict
|
||||
elif 'cryptvalue' in nodeobj[attribute] and decrypt:
|
||||
retdict = copy.deepcopy(nodeobj[attribute])
|
||||
retdict['value'] = crypto.decrypt_value(
|
||||
nodeobj[attribute]['cryptvalue'])
|
||||
return nodeobj[attribute]
|
||||
|
||||
|
||||
def _expand_expression(attribute, nodeobj, decrypt=False):
|
||||
# here is where we may avail ourselves of string.Formatter or
|
||||
# string.Template
|
||||
# we would then take the string that is identifier and do
|
||||
@ -98,36 +179,58 @@ def _expand_expression(attribute, nodeobj):
|
||||
# raise TypeError(node)
|
||||
pass
|
||||
|
||||
_cfgstore = {}
|
||||
|
||||
class NodeAttribs(object):
|
||||
def __init__(self, nodes=[], attributes=[], tenant=0):
|
||||
self._nodelist = collecitons.dequeue(nodes)
|
||||
# my thinking at this point is that noderange and configdata objects
|
||||
# will be constructed and passed as part of a context object to plugins
|
||||
# reasoning being that the main program will handle establishing the
|
||||
# tenant context and then modules need not consider the current tenant
|
||||
# most of the time as things are automatic
|
||||
|
||||
class ConfigData(object):
|
||||
def __init__(self, tenant=0, decrypt=False):
|
||||
self._tenant = tenant
|
||||
self._attributes=attributes
|
||||
self.decrypt = decrypt
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next():
|
||||
node = self._nodelist.popleft()
|
||||
onodeobj = _cfgstore['node'][(self._tenant,node)]
|
||||
nodeobj =
|
||||
attriblist = []
|
||||
#if there is a filter, delete irrelevant keys
|
||||
if self._attributes.length > 0:
|
||||
for attribute in nodeobj.keys():
|
||||
if attribute not in self._attributes:
|
||||
del nodeobj[attribute]
|
||||
#now that attributes are filtered, seek out and evaluate expressions
|
||||
for attribute in nodeobj.keys():
|
||||
if ('value' not in nodeobj[attribute] and
|
||||
'cryptvalue' in nodeobj[attribute]):
|
||||
nodeobj[attribute]['value'] = _decrypt_value(
|
||||
nodeobj[attribute]['cryptvalue'])
|
||||
if ('value' not in nodeobj[attribute] and
|
||||
'expression' in nodeobj[attribute]):
|
||||
nodeobj[attribute]['value'] = _expand_expression(
|
||||
attribute=attribute,
|
||||
nodeobj=nodeobj)
|
||||
def get_node_attributes(self, nodelist, attributes=[]):
|
||||
if 'node' not in _cfgstore:
|
||||
return None
|
||||
retdict = {}
|
||||
if isinstance(nodelist,str):
|
||||
nodelist = [nodelist]
|
||||
for node in nodelist:
|
||||
if (self._tenant,node) not in _cfgstore['node']:
|
||||
continue
|
||||
cfgnodeobj = _cfgstore['node'][(self._tenant,node)]
|
||||
exprmgr = _ExpressionFormat(cfgnodeobj)
|
||||
nodeobj = {}
|
||||
if len(attributes) == 0:
|
||||
attributes = cfgnodeobj.keys()
|
||||
for attribute in attributes:
|
||||
if attribute not in cfgnodeobj:
|
||||
continue
|
||||
nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj,
|
||||
formatter=exprmgr,
|
||||
decrypt=self.decrypt)
|
||||
retdict[node] = nodeobj
|
||||
return retdict
|
||||
|
||||
def set_node_attributes(self, attribmap):
|
||||
if 'node' not in _cfgstore:
|
||||
_cfgstore['node'] = {}
|
||||
for node in attribmap.keys():
|
||||
key = (self._tenant, node)
|
||||
if key not in _cfgstore['node']:
|
||||
_cfgstore['node'][key] = {'name': {'value': node}}
|
||||
for attrname in attribmap[node].keys():
|
||||
newdict = {}
|
||||
if isinstance(attribmap[node][attrname], dict):
|
||||
newdict = attribmap[node][attrname]
|
||||
else:
|
||||
newdict = {'value': attribmap[node][attrname] }
|
||||
if 'value' in newdict and attrname.startswith("credential"):
|
||||
newdict['cryptvalue' ] = \
|
||||
crypto.crypt_value(newdict['value'])
|
||||
del newdict['value']
|
||||
_cfgstore['node'][key][attrname] = newdict
|
||||
|
||||
|
||||
@ -75,23 +75,23 @@ def _format_key(key, passphrase=None):
|
||||
return {"unencryptedvalue": key}
|
||||
|
||||
|
||||
def _init_masterkey(passphrase=None):
|
||||
if 'master_privacy_key' in _cfgstore['globals']:
|
||||
def init_masterkey(cfgstore, passphrase=None, cfgstore):
|
||||
if 'master_privacy_key' in cfgstore['globals']:
|
||||
_masterkey = _get_protected_key(
|
||||
_cfgstore['globals']['master_privacy_key'],
|
||||
cfgstore['globals']['master_privacy_key'],
|
||||
passphrase=passphrase)
|
||||
else:
|
||||
_masterkey = os.urandom(32)
|
||||
_cfgstore['globals']['master_privacy_key'] = _format_key(_masterkey,
|
||||
cfgstore['globals']['master_privacy_key'] = _format_key(_masterkey,
|
||||
passphrase=passphrase)
|
||||
if 'master_integrity_key' in _cfgstore['globals']:
|
||||
if 'master_integrity_key' in cfgstore['globals']:
|
||||
_masterintegritykey = _get_protected_key(
|
||||
_cfgstore['globals']['master_integrity_key'],
|
||||
cfgstore['globals']['master_integrity_key'],
|
||||
passphrase=passphrase
|
||||
)
|
||||
else:
|
||||
_masterintegritykey = os.urandom(64)
|
||||
_cfgstore['globals']['master_integrity_key'] = _format_key(
|
||||
cfgstore['globals']['master_integrity_key'] = _format_key(
|
||||
_masterintegritykey,
|
||||
passphrase=passphrase
|
||||
)
|
||||
@ -49,7 +49,7 @@ def _pick_mimetype(env):
|
||||
|
||||
Note that as it gets into the ACCEPT header honoring, it only looks for
|
||||
application/json and else gives up and assumes html. This is because
|
||||
browsers are very chaotic about ACCEPT header. It is assumed that
|
||||
browsers are very chaotic about ACCEPT HEADER. It is assumed that
|
||||
XMLHttpRequest.setRequestHeader will be used by clever javascript
|
||||
if the '.json' scheme doesn't cut it.
|
||||
"""
|
||||
|
||||
Loading…
Reference in New Issue
Block a user