confluent/confluent_server/confluentd/config/configmanager.py

# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2014 IBM Corporation
# Copyright 2015 Lenovo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# Ultimately, the design is to handle all the complicated stuff at set
# rather than get tiime.  When something is set on a group, then all
# members of that group are examined  and 'inheritedfrom' attributes
# are pushed.  as expression definned values are iinserted, their
#  dependdentt attributes are added to a private dict to aid in auto
# calculation.  When a name is changed, all attributes are re-evaluated
# on get, should be simple read value *except* for encrypted values,
# which are only decrypted when explicitly requested
# encrypted fields do not support expressions, either as a source or
# destination

#TODO: clustered mode
# In clustered case, only one instance is the 'master'.  If some 'def set'
# is requested on a slave, it creates a transaction id and an event, firing it
# to master.  It then waits on the event.  When the master reflects the data
# back and that reflection data goes into memory, the wait will be satisfied
# this means that set on a slave will be much longer.
# the assumption is that only the calls to 'def set' need be pushed to/from
# master and all the implicit activity that ensues will pan out since
# the master is ensuring a strict ordering of transactions
# for missed transactions, transaction log will be used to track transactions
# transaction log can have a constrained size if we want, in which case full
# replication will trigger.
# uuid.uuid4() will be used for transaction ids


# Note on the cryptography.  Default behavior is mostly just to pave the
# way to meaningful security.  Root all potentially sensitive data in
# one key.  That key is in plain sight, so not meaningfully protected
# However, the key can be protected in the following ways:
#   - Passphrase protected (requiring human interaction every restart)
#   - TPM sealing (which would forgo the interactive assuming risk of
#           physical attack on TPM is not a concern)
# This module provides cryptographic convenience functions, largely to be
# used by config.py to protect/unlock configuration as appropriopriate.
# The default behavior provides no meaningful protection, all encrypted
# values are linked to a master key that is stored in the clear.
# meanigful protection comes when the user elects to protect the key
# by passphrase and optionally TPM


import Crypto.Protocol.KDF as KDF
from Crypto.Cipher import AES
from Crypto.Hash import HMAC
from Crypto.Hash import SHA256
import anydbm as dbm
import ast
import base64
import confluentd.config.attributes as allattributes
import confluentd.log
import confluentd.util
import confluentd.exceptions as exc
import copy
import cPickle
import errno
import json
import operator
import os
import random
import re
import string
import sys
import threading


_masterkey = None
_masterintegritykey = None
_dirtylock = threading.RLock()
_config_areas = ('nodegroups', 'nodes', 'usergroups', 'users')
tracelog = None

def _mkpath(pathname):
    try:
        os.makedirs(pathname)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(pathname):
            pass
        else:
            raise


def _derive_keys(password, salt):
    #implement our specific combination of pbkdf2 transforms to get at
    #key.  We bump the iterations up because we can afford to
    #TODO: WORKERPOOL PBKDF2 is expensive
    tmpkey = KDF.PBKDF2(password, salt, 32, 50000,
                        lambda p, s: HMAC.new(p, s, SHA256).digest())
    finalkey = KDF.PBKDF2(tmpkey, salt, 32, 50000,
                          lambda p, s: HMAC.new(p, s, SHA256).digest())
    return finalkey[:16], finalkey[16:]


def _get_protected_key(keydict, password, paramname):
    if password and 'unencryptedvalue' in keydict:
        set_global(paramname, _format_key(
            keydict['unencryptedvalue'],
            password=password))
    if 'unencryptedvalue' in keydict:
        return keydict['unencryptedvalue']
    # TODO(jbjohnso): check for TPM sealing
    if 'passphraseprotected' in keydict:
        if password is None:
            raise exc.LockedCredentials("Passphrase protected secret requires password")
        pp = keydict['passphraseprotected']
        salt = pp[0]
        privkey, integkey = _derive_keys(password, salt)
        return decrypt_value(pp[1:], key=privkey, integritykey=integkey)
    else:
        raise exc.LockedCredentials("No available decryption key")


def _format_key(key, password=None):
    if password is not None:
        salt = os.urandom(32)
        privkey, integkey = _derive_keys(password, salt)
        cval = crypt_value(key, key=privkey, integritykey=integkey)
        return {"passphraseprotected": (salt,) + cval}
    else:
        return {"unencryptedvalue": key}


def init_masterkey(password=None):
    global _masterkey
    global _masterintegritykey
    cfgn = get_global('master_privacy_key')

    if cfgn:
        _masterkey = _get_protected_key(cfgn, password, 'master_privacy_key')
    else:
        _masterkey = os.urandom(32)
        set_global('master_privacy_key', _format_key(
            _masterkey,
            password=password))
    cfgn = get_global('master_integrity_key')
    if cfgn:
        _masterintegritykey = _get_protected_key(cfgn, password,
                                                 'master_integrity_key')
    else:
        _masterintegritykey = os.urandom(64)
        set_global('master_integrity_key', _format_key(
            _masterintegritykey,
            password=password))


def decrypt_value(cryptvalue,
                  key=None,
                  integritykey=None):
    iv, cipherdata, hmac = cryptvalue
    if key is None and integritykey is None:
        if _masterkey is None or _masterintegritykey is None:
            init_masterkey()
        key = _masterkey
        integritykey = _masterintegritykey
    check_hmac = HMAC.new(integritykey, cipherdata, SHA256).digest()
    if hmac != check_hmac:
        raise Exception("bad HMAC value on crypted value")
    decrypter = AES.new(key, AES.MODE_CBC, iv)
    value = decrypter.decrypt(cipherdata)
    padsize = ord(value[-1])
    pad = value[-padsize:]
    # Note that I cannot grasp what could be done with a subliminal
    # channel in padding in this case, but check the padding anyway
    for padbyte in pad:
        if ord(padbyte) != padsize:
            raise Exception("bad padding in encrypted value")
    return value[0:-padsize]


def crypt_value(value,
                key=None,
                integritykey=None):
    # encrypt given value
    # PKCS7 is the padding scheme to employ, if no padded needed, pad with 16
    # check HMAC prior to attempting decrypt
    if key is None or integritykey is None:
        if _masterkey is None or _masterintegritykey is None:
            init_masterkey()
        key = _masterkey
        integritykey = _masterintegritykey
    iv = os.urandom(16)
    crypter = AES.new(key, AES.MODE_CBC, iv)
    neededpad = 16 - (len(value) % 16)
    pad = chr(neededpad) * neededpad
    value += pad
    cryptval = crypter.encrypt(value)
    hmac = HMAC.new(integritykey, cryptval, SHA256).digest()
    return iv, cryptval, hmac


def _load_dict_from_dbm(dpath, tdb):
    try:
        dbe = dbm.open(tdb, 'r')
        currdict = _cfgstore
        for elem in dpath:
            if elem not in currdict:
                currdict[elem] = {}
            currdict = currdict[elem]
        for tk in dbe.iterkeys():
            currdict[tk] = cPickle.loads(dbe[tk])
    except dbm.error:
        return


def is_tenant(tenant):
    try:
        return tenant in _cfgstore['tenant']
    except KeyError:
        return False


def get_global(globalname):
    """Get a global variable


    :param globalname: The global parameter name to read
    """
    try:
        return _cfgstore['globals'][globalname]
    except KeyError:
        return None


def set_global(globalname, value):
    """Set a global variable.

    Globals should be rarely ever used.  Almost everything should be under a
    tenant scope.  Some things like master key and socket numbers/paths can be
    reasonably considered global in nature.

    :param globalname:  The global parameter name to store
    :param value: The value to set the global parameter to.
    """
    with _dirtylock:
        if 'dirtyglobals' not in _cfgstore:
            _cfgstore['dirtyglobals'] = set()
        _cfgstore['dirtyglobals'].add(globalname)
    if 'globals' not in _cfgstore:
        _cfgstore['globals'] = {globalname: value}
    else:
        _cfgstore['globals'][globalname] = value
    ConfigManager._bg_sync_to_file()


def _mark_dirtykey(category, key, tenant=None):
    if type(key) in (str, unicode):
        key = key.encode('utf-8')
    with _dirtylock:
        if 'dirtykeys' not in _cfgstore:
            _cfgstore['dirtykeys'] = {}
        if tenant not in _cfgstore['dirtykeys']:
            _cfgstore['dirtykeys'][tenant] = {}
        if category not in _cfgstore['dirtykeys'][tenant]:
            _cfgstore['dirtykeys'][tenant][category] = set()
        _cfgstore['dirtykeys'][tenant][category].add(key)


def _generate_new_id():
    # generate a random id outside the usual ranges used for normal users in
    # /etc/passwd.  Leave an equivalent amount of space near the end disused,
    # just in case
    uid = str(confluentd.util.securerandomnumber(65537, 4294901759))
    if 'idmap' not in _cfgstore['main']:
        return uid
    while uid in _cfgstore['main']['idmap']:
        uid = str(confluentd.util.securerandomnumber(65537, 4294901759))
    return uid


class _ExpressionFormat(string.Formatter):
    # This class is used to extract the literal value from an expression
    # in the db
    # This is made easier by subclassing one of the 'fprintf' mechanisms
    # baked into python
    posmatch = re.compile('^n([0-9]*)$')
    nummatch = re.compile('[0-9]+')
    _supported_ops = {
        ast.Mult: operator.mul,
        ast.Div: operator.floordiv,
        ast.Mod: operator.mod,
        ast.Add: operator.add,
        ast.Sub: operator.sub,
        ast.LShift: operator.lshift,
        ast.RShift: operator.rshift,
        ast.BitAnd: operator.and_,
        ast.BitXor: operator.xor,
        ast.BitOr: operator.or_,
    }

    def __init__(self, nodeobj, nodename):
        self._nodeobj = nodeobj
        self._nodename = nodename
        self._numbers = None

    def get_field(self, field_name, args, kwargs):
        parsed = ast.parse(field_name)
        return self._handle_ast_node(parsed.body[0].value), field_name

    def _handle_ast_node(self, node):
        if isinstance(node, ast.Num):
            return node.n
        elif isinstance(node, ast.Attribute):
            #ok, we have something with a dot
            left = node.value.id
            right = node.attr
            key = left + '.' + right
            if '_expressionkeys' not in self._nodeobj:
                self._nodeobj['_expressionkeys'] = set([key])
            else:
                self._nodeobj['_expressionkeys'].add(key)
            val = _decode_attribute(key, self._nodeobj,
                                    formatter=self)
            return val['value'] if 'value' in val else ""
        elif isinstance(node, ast.Name):
            var = node.id
            if var == 'nodename':
                return self._nodename
            mg = re.match(self.posmatch, var)
            if mg:
                idx = int(mg.group(1))
                if self._numbers is None:
                    self._numbers = re.findall(self.nummatch, self._nodename)
                return int(self._numbers[idx - 1])
            else:
                if var in self._nodeobj:
                    if '_expressionkeys' not in self._nodeobj:
                        self._nodeobj['_expressionkeys'] = set([var])
                    else:
                        self._nodeobj['_expressionkeys'].add(var)
                    val = _decode_attribute(var, self._nodeobj,
                                            formatter=self)
                    return val['value'] if 'value' in val else ""
        elif isinstance(node, ast.BinOp):
            optype = type(node.op)
            if optype not in self._supported_ops:
                raise Exception("Unsupported operation")
            op = self._supported_ops[optype]
            return op(self._handle_ast_node(node.left),
                      self._handle_ast_node(node.right))


def _decode_attribute(attribute, nodeobj, formatter=None, decrypt=False):
    if attribute not in nodeobj:
        return None
    # if we have an expression and a formatter, that overrides 'value'
    # which may be out of date
    # get methods will skip the formatter allowing value to come on through
    # set methods induce recalculation as appropriate to get a cached value
    if 'expression' in nodeobj[attribute] and formatter is not None:
        retdict = copy.deepcopy(nodeobj[attribute])
        if 'value' in retdict:
            del retdict['value']
        try:
            retdict['value'] = formatter.format(retdict['expression'])
        except Exception as e:
            retdict['broken'] = str(e)
        return retdict
    elif 'value' in nodeobj[attribute]:
        return nodeobj[attribute]
    elif 'cryptvalue' in nodeobj[attribute] and decrypt:
        retdict = copy.deepcopy(nodeobj[attribute])
        retdict['value'] = decrypt_value(nodeobj[attribute]['cryptvalue'])
        return retdict
    return nodeobj[attribute]


# my thinking at this point is that noderange and configdata  objects
# will be constructed and passed as part of a context object to plugins
# reasoning being that the main program will handle establishing the
# tenant context and then modules need not consider the current tenant
# most of the time as things are automatic

def _addchange(changeset, node, attrname):
    if node not in changeset:
        changeset[node] = {attrname: 1}
    else:
        changeset[node][attrname] = 1


def hook_new_configmanagers(callback):
    """Register callback for new tenants

    From the point when this function is called until the end,
    callback may be invoked to indicate a new tenant and
    callback is notified to perform whatever tasks appropriate for
    a new tenant

    :param callback: Function to call for each possible config manager
    :returns: identifier that can be used to cancel this registration
    """
    #TODO(jbjohnso): actually live up to the promise of ongoing callbacks
    callback(ConfigManager(None))
    try:
        for tenant in _cfgstore['tenant'].iterkeys():
            callback(ConfigManager(tenant))
    except KeyError:
        pass


class ConfigManager(object):
    if os.name == 'nt':
        _cfgdir = os.path.join(
            os.getenv('SystemDrive'), '\\ProgramData', 'confluent', 'cfg')
    else:
        _cfgdir = "/etc/confluent/cfg"
    _cfgwriter = None
    _writepending = False
    _syncrunning = False
    _syncstate = threading.RLock()
    _attribwatchers = {}
    _nodecollwatchers = {}
    _notifierids = {}

    def __init__(self, tenant, decrypt=False):
        global _cfgstore
        self.decrypt = decrypt
        if tenant is None:
            self.tenant = None
            if 'main' not in _cfgstore:
                _cfgstore['main'] = {}
                self._bg_sync_to_file()
            self._cfgstore = _cfgstore['main']
            if 'nodegroups' not in self._cfgstore:
                self._cfgstore['nodegroups'] = {'everything': {'nodes': set()}}
                self._bg_sync_to_file()
            if 'nodes' not in self._cfgstore:
                self._cfgstore['nodes'] = {}
                self._bg_sync_to_file()
            return
        elif 'tenant' not in _cfgstore:
            _cfgstore['tenant'] = {tenant: {}}
            self._bg_sync_to_file()
        elif tenant not in _cfgstore['tenant']:
            _cfgstore['tenant'][tenant] = {}
            self._bg_sync_to_file()
        self.tenant = tenant
        self._cfgstore = _cfgstore['tenant'][tenant]
        if 'nodegroups' not in self._cfgstore:
            self._cfgstore['nodegroups'] = {'everything': {}}
        if 'nodes' not in self._cfgstore:
            self._cfgstore['nodes'] = {}
        self._bg_sync_to_file()

    def filter_node_attributes(self, expression, nodes=None):
        """Filtered nodelist according to expression

        expression may be:
        attribute.name=value
        attribute.name==value
        attribute.name=~value
        attribute.name!=value
        attribute.name!~value

        == and != do strict equality.  The ~ operators do a regular expression.
        ! negates the sense of the match

        :param expression: The expression containing the criteria to match
        :param nodes: Optional iterable set of nodes to limit the check
        """
        exmatch = None
        yieldmatches = True
        if nodes is None:
            nodes = self._cfgstore['nodes']
        if '==' in expression:
            attribute, match = expression.split('==')
        elif '!=' in expression:
            attribute, match = expression.split('!=')
            yieldmatches = False
        elif '=~' in expression:
            attribute, match = expression.split('=~')
            exmatch = re.compile(match)
        elif '!~' in expression:
            attribute, match = expression.split('!~')
            exmatch = re.compile(match)
            yieldmatches = False
        elif '=' in expression:
            attribute, match = expression.split('=')
        else:
            raise Exception('Invalid Expression')
        for node in nodes:
            try:
                currval = self._cfgstore['nodes'][node][attribute]['value']
            except KeyError:
                # Let's treat 'not set' as being an empty string for this path
                currval = ''
            if exmatch:
                if yieldmatches:
                    if exmatch.search(currval):
                        yield node
                else:
                    if not exmatch.search(currval):
                        yield node
            else:
                if yieldmatches:
                    if match == currval:
                        yield node
                else:
                    if match != currval:
                        yield node

    def filter_nodenames(self, expression, nodes=None):
        """Filter nodenames by regular expression

        :param expression: Regular expression for matching nodenames
        :param nodes: Optional iterable of candidates
        """
        if nodes is None:
            nodes = self._cfgstore['nodes']
        expression = re.compile(expression)
        for node in nodes:
            if expression.search(node):
                yield node

    def watch_attributes(self, nodes, attributes, callback):
        """
        Watch a list of attributes for changes on a list of nodes

        :param nodes: An iterable of node names to be watching
        :param attributes: An iterable of attribute names to be notified about
        :param callback: A callback to process a notification

        Returns an identifier that can be used to unsubscribe from these
        notifications using remove_watcher
        """
        notifierid = random.randint(0, sys.maxint)
        while notifierid in self._notifierids:
            notifierid = random.randint(0, sys.maxint)
        self._notifierids[notifierid] = {'attriblist': []}
        if self.tenant not in self._attribwatchers:
            self._attribwatchers[self.tenant] = {}
        attribwatchers = self._attribwatchers[self.tenant]
        for node in nodes:
            if node not in attribwatchers:
                attribwatchers[node] = {}
            for attribute in attributes:
                self._notifierids[notifierid]['attriblist'].append(
                    (node, attribute))
                if attribute not in attribwatchers[node]:
                    attribwatchers[node][attribute] = {
                        notifierid: callback
                    }
                else:
                    attribwatchers[node][attribute][notifierid] = callback
        return notifierid

    def watch_nodecollection(self, callback):
        """
        Watch the nodecollection for addition or removal of nodes.

        A watcher is notified prior after node has been added and before node
        is actually removed.

        :param callback: Function to call when a node is added or removed

        Returns an identifier that can be used to unsubscribe from these
        notifications using remove_watcher
        """
        # first provide an identifier for the calling code to
        # use in case of cancellation.
        # I anticipate no more than a handful of watchers of this sort, so
        # this loop should not have to iterate too many times
        notifierid = random.randint(0, sys.maxint)
        while notifierid in self._notifierids:
            notifierid = random.randint(0, sys.maxint)
        # going to track that this is a nodecollection type watcher,
        # but there is no additional data associated.
        self._notifierids[notifierid] = set(['nodecollection'])
        if self.tenant not in self._nodecollwatchers:
            self._nodecollwatchers[self.tenant] = {}
        self._nodecollwatchers[self.tenant][notifierid] = callback
        return notifierid

    def remove_watcher(self, watcher):
        # identifier of int would be a collection watcher
        if watcher not in self._notifierids:
            raise Exception("Invalid")
            # return
        if 'attriblist' in self._notifierids[watcher]:
            attribwatchers = self._attribwatchers[self.tenant]
            for nodeattrib in self._notifierids[watcher]['attriblist']:
                node, attrib = nodeattrib
                del attribwatchers[node][attrib][watcher]
        elif 'nodecollection' in self._notifierids[watcher]:
            del self._nodecollwatchers[self.tenant][watcher]
        else:
            raise Exception("Completely not a valid place to be")
        del self._notifierids[watcher]

    def list_users(self):
        try:
            return self._cfgstore['users'].iterkeys()
        except KeyError:
            return []

    def get_user(self, name):
        """Get user information from DB

        :param name: Name of the user

        Returns a dictionary describing parameters of a user.  These parameters
        may include numeric id (id), certificate thumbprint (certthumb),
        password hash (passhash, which currently is going to be PBKDF2 derived)
        full name (displayname), ...

        """
        try:
            return copy.deepcopy(self._cfgstore['users'][name])
        except KeyError:
            return None

    def get_usergroup(self, groupname):
        """Get user group information from DB

        :param groupname: Name of the group

        Returns a dictionary describing parameters of a user group.
        This may include the role for users in the group to receive
        if no more specific information is found.

        """
        try:
            return copy.deepcopy(self._cfgstore['usergroups'][groupname])
        except KeyError:
            return None

    def set_usergroup(self, groupname, attributemap):
        """Set usergroup attribute(s)

        :param groupname: the name of teh group to modify
        :param attributemap: The mapping of keys to values to set
        """

        for attribute in attributemap.iterkeys():
            self._cfgstore['usergroups'][attribute] = attributemap[attribute]
        _mark_dirtykey('usergroups', groupname, self.tenant)

    def create_usergroup(self, groupname, role="Administrator"):
        if 'usergroups' not in self._cfgstore:
            self._cfgstore['usergroups'] = {}
        groupname = groupname.encode('utf-8')
        if groupname in self._cfgstore['usergroups']:
            raise Exception("Duplicate groupname requested")
        self._cfgstore['usergroups'][groupname] = {'role': role}
        _mark_dirtykey('usergroups', groupname, self.tenant)

    def set_user(self, name, attributemap):
        """Set user attribute(s)

        :param name: The login name of the user
        :param attributemap: A dict of key values to set
        """
        user = self._cfgstore['users'][name]
        for attribute in attributemap:
            if attribute == 'password':
                salt = os.urandom(8)
                #TODO: WORKERPOOL, offload password set to a worker
                crypted = KDF.PBKDF2(
                    attributemap[attribute], salt, 32, 10000,
                    lambda p, s: HMAC.new(p, s, SHA256).digest()
                )
                user['cryptpass'] = (salt, crypted)
            else:
                user[attribute] = attributemap[attribute]
        _mark_dirtykey('users', name, self.tenant)
        self._bg_sync_to_file()

    def del_user(self, name):
        if name in self._cfgstore['users']:
            del self._cfgstore['users'][name]
            _mark_dirtykey('users', name, self.tenant)
        self._bg_sync_to_file()

    def create_user(self, name,
                    role="Administrator", uid=None, displayname=None,
                    attributemap=None):
        """Create a new user

        :param name: The login name of the user
        :param role: The role the user should be considered.  Can be
                     "Administrator" or "Technician", defaults to
                     "Administrator"
        :param uid: Custom identifier number if desired.  Defaults to random.
        :param displayname: Optional long format name for UI consumption
        """
        if uid is None:
            uid = _generate_new_id()
        else:
            if uid in _cfgstore['main']['idmap']:
                raise Exception("Duplicate id requested")
        if 'users' not in self._cfgstore:
            self._cfgstore['users'] = {}
        name = name.encode('utf-8')
        if name in self._cfgstore['users']:
            raise Exception("Duplicate username requested")
        self._cfgstore['users'][name] = {'id': uid}
        if displayname is not None:
            self._cfgstore['users'][name]['displayname'] = displayname
        if 'idmap' not in _cfgstore['main']:
            _cfgstore['main']['idmap'] = {}
        _cfgstore['main']['idmap'][uid] = {
            'tenant': self.tenant,
            'username': name
        }
        if attributemap is not None:
            self.set_user(name, attributemap)
        _mark_dirtykey('users', name, self.tenant)
        _mark_dirtykey('idmap', uid)
        self._bg_sync_to_file()

    def is_node(self, node):
        return node in self._cfgstore['nodes']

    def is_nodegroup(self, nodegroup):
        return nodegroup in self._cfgstore['nodegroups']

    def get_groups(self):
        return self._cfgstore['nodegroups'].iterkeys()

    def list_nodes(self):
        try:
            return self._cfgstore['nodes'].iterkeys()
        except KeyError:
            return []

    def get_nodegroup_attributes(self, nodegroup, attributes=()):
        cfgnodeobj = self._cfgstore['nodegroups'][nodegroup]
        if not attributes:
            attributes = cfgnodeobj.iterkeys()
        nodeobj = {}
        for attribute in attributes:
            if attribute.startswith('_'):
                continue
            if attribute not in cfgnodeobj:
                continue
            nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj,
                                                   decrypt=self.decrypt)
        return nodeobj

    def get_node_attributes(self, nodelist, attributes=()):
        retdict = {}
        relattribs = attributes
        if isinstance(nodelist, str) or isinstance(nodelist, unicode):
            nodelist = [nodelist]
        for node in nodelist:
            if node not in self._cfgstore['nodes']:
                continue
            cfgnodeobj = self._cfgstore['nodes'][node]
            nodeobj = {}
            if len(attributes) == 0:
                relattribs = cfgnodeobj
            for attribute in relattribs:
                if attribute.startswith('_'):
                    # skip private things
                    continue
                if attribute not in cfgnodeobj:
                    continue
                # since the formatter is not passed in, the calculator is
                # skipped.  The decryption, however, we want to do only on
                # demand
                nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj,
                                                       decrypt=self.decrypt)
            retdict[node] = nodeobj
        return retdict

    def _node_added_to_group(self, node, group, changeset):
        try:
            nodecfg = self._cfgstore['nodes'][node]
            groupcfg = self._cfgstore['nodegroups'][group]
        except KeyError:  # something did not exist, nothing to do
            return
        for attrib in groupcfg.iterkeys():
            self._do_inheritance(nodecfg, attrib, node, changeset)
            _addchange(changeset, node, attrib)

    def _node_removed_from_group(self, node, group, changeset):
        try:
            nodecfg = self._cfgstore['nodes'][node]
        except KeyError:  # node did not exist, nothing to do
            return
        for attrib in nodecfg.keys():
            if attrib.startswith("_"):
                continue
            if attrib == 'groups':
                continue
            try:
                if nodecfg[attrib]['inheritedfrom'] == group:
                    del nodecfg[attrib]  # remove invalid inherited data
                    self._do_inheritance(nodecfg, attrib, node, changeset)
                    _addchange(changeset, node, attrib)
                    _mark_dirtykey('nodes', node, self.tenant)
            except KeyError:  # inheritedfrom not set, move on
                pass

    def _do_inheritance(self, nodecfg, attrib, nodename, changeset,
                        srcgroup=None):
        # for now, just do single inheritance
        # TODO: concatenating inheritance if requested
        if attrib in ('nodes', 'groups'):
            #not attributes that should be considered here
            return
        if attrib in nodecfg and 'inheritedfrom' not in nodecfg[attrib]:
            return  # already has a non-inherited value set, nothing to do
        # if the attribute is not set, this will search for a candidate
        # if it is set, but inheritedfrom, search for a replacement, just
        # in case
        if not 'groups' in nodecfg:
            return
        for group in nodecfg['groups']:
            if attrib in self._cfgstore['nodegroups'][group]:
                if srcgroup is not None and group != srcgroup:
                    # skip needless deepcopy
                    return
                nodecfg[attrib] = \
                    copy.deepcopy(self._cfgstore['nodegroups'][group][attrib])
                nodecfg[attrib]['inheritedfrom'] = group
                self._refresh_nodecfg(nodecfg, attrib, nodename,
                                      changeset=changeset)
                _mark_dirtykey('nodes', nodename, self.tenant)
                return
            if srcgroup is not None and group == srcgroup:
                # break out
                return

    def _sync_groups_to_node(self, groups, node, changeset):
        for group in self._cfgstore['nodegroups'].iterkeys():
            if group not in groups:
                if node in self._cfgstore['nodegroups'][group]['nodes']:
                    self._cfgstore['nodegroups'][group]['nodes'].discard(node)
                    self._node_removed_from_group(node, group, changeset)
                    _mark_dirtykey('nodegroups', group, self.tenant)
        for group in groups:
            if group not in self._cfgstore['nodegroups']:
                self._cfgstore['nodegroups'][group] = {'nodes': set([node])}
                _mark_dirtykey('nodegroups', group, self.tenant)
            elif node not in self._cfgstore['nodegroups'][group]['nodes']:
                self._cfgstore['nodegroups'][group]['nodes'].add(node)
                _mark_dirtykey('nodegroups', group, self.tenant)
            # node was not already in given group, perform inheritence fixup
            self._node_added_to_group(node, group, changeset)

    def _sync_nodes_to_group(self, nodes, group, changeset):
        for node in self._cfgstore['nodes'].iterkeys():
            if node not in nodes and 'groups' in self._cfgstore['nodes'][node]:
                if group in self._cfgstore['nodes'][node]['groups']:
                    self._cfgstore['nodes'][node]['groups'].remove(group)
                    self._node_removed_from_group(node, group, changeset)
        for node in nodes:
            if node not in self._cfgstore['nodes']:
                self._cfgstore['nodes'][node] = {'groups': [group]}
                _mark_dirtykey('nodes', node, self.tenant)
            elif group not in self._cfgstore['nodes'][node]['groups']:
                self._cfgstore['nodes'][node]['groups'].insert(0, group)
                _mark_dirtykey('nodes', node, self.tenant)
            else:
                continue  # next node, this node already in
            self._node_added_to_group(node, group, changeset)

    def add_group_attributes(self, attribmap):
        self.set_group_attributes(attribmap, autocreate=True)

    def set_group_attributes(self, attribmap, autocreate=False):
        changeset = {}
        for group in attribmap.iterkeys():
            if group == '':
                raise ValueError('"{0}" is not a valid group name'.format(
                    group))
            if not autocreate and group not in self._cfgstore['nodegroups']:
                raise ValueError("{0} group does not exist".format(group))
            for attr in attribmap[group].iterkeys():
                if (attr not in ('nodes', 'noderange') and
                        (attr not in allattributes.node or
                         ('type' in allattributes.node[attr] and
                          not isinstance(attribmap[group][attr],
                                         allattributes.node[attr]['type'])))):
                    raise ValueError
                if attr == 'nodes':
                    if not isinstance(attribmap[group][attr], list):
                        raise ValueError(
                            "nodes attribute on group must be list")
                    for node in attribmap[group]['nodes']:
                        if node not in self._cfgstore['nodes']:
                            raise ValueError(
                                "{0} node does not exist to add to {1}".format(
                                    node, group))
        for group in attribmap.iterkeys():
            group = group.encode('utf-8')
            if group not in self._cfgstore['nodegroups']:
                self._cfgstore['nodegroups'][group] = {'nodes': set()}
            cfgobj = self._cfgstore['nodegroups'][group]
            for attr in attribmap[group].iterkeys():
                if attr == 'nodes':
                    newdict = set(attribmap[group][attr])
                elif (isinstance(attribmap[group][attr], str) or
                        isinstance(attribmap[group][attr], unicode)):
                    newdict = {'value': attribmap[group][attr]}
                else:
                    newdict = attribmap[group][attr]
                if 'value' in newdict and attr.startswith("secret."):
                    newdict['cryptvalue'] = crypt_value(newdict['value'])
                    del newdict['value']
                cfgobj[attr] = newdict
                if attr == 'nodes':
                    self._sync_nodes_to_group(group=group,
                                              nodes=attribmap[group]['nodes'],
                                              changeset=changeset)
                elif attr != 'noderange':  # update inheritence
                    for node in cfgobj['nodes']:
                        nodecfg = self._cfgstore['nodes'][node]
                        self._do_inheritance(nodecfg, attr, node, changeset,
                                             srcgroup=group)
                        _addchange(changeset, node, attr)
            _mark_dirtykey('nodegroups', group, self.tenant)
        self._notif_attribwatchers(changeset)
        self._bg_sync_to_file()

    def clear_group_attributes(self, groups, attributes):
        changeset = {}
        if type(groups) in (str, unicode):
            groups = (groups,)
        for group in groups:
                group = group.encode('utf-8')
                try:
                    groupentry = self._cfgstore['nodegroups'][group]
                except KeyError:
                    continue
                for attrib in attributes:
                    if attrib == 'nodes':
                        groupentry['nodes'] = set()
                        self._sync_nodes_to_group(
                            group=group, nodes=(), changeset=changeset)
                    else:
                        try:
                            del groupentry[attrib]
                        except KeyError:
                            pass
                        for node in groupentry['nodes']:
                            nodecfg = self._cfgstore['nodes'][node]
                            try:
                                delnodeattrib = (
                                    nodecfg[attrib]['inheritedfrom'] == group)
                            except KeyError:
                                delnodeattrib = False
                            if delnodeattrib:
                                del nodecfg[attrib]
                                self._do_inheritance(nodecfg, attrib, node,
                                                     changeset)
                                _addchange(changeset, node, attrib)
                                _mark_dirtykey('nodes', node, self.tenant)
                _mark_dirtykey('nodegroups', group, self.tenant)
        self._notif_attribwatchers(changeset)
        self._bg_sync_to_file()

    def _refresh_nodecfg(self, cfgobj, attrname, node, changeset):
        exprmgr = None
        if 'expression' in cfgobj[attrname]:  # evaluate now
            if exprmgr is None:
                exprmgr = _ExpressionFormat(cfgobj, node)
            cfgobj[attrname] = _decode_attribute(attrname, cfgobj,
                                                 formatter=exprmgr)
        if ('_expressionkeys' in cfgobj and
                attrname in cfgobj['_expressionkeys']):
            if exprmgr is None:
                exprmgr = _ExpressionFormat(cfgobj, node)
            self._recalculate_expressions(cfgobj, formatter=exprmgr, node=node,
                                          changeset=changeset)

    def _notif_attribwatchers(self, nodeattrs):
        if self.tenant not in self._attribwatchers:
            return
        notifdata = {}
        attribwatchers = self._attribwatchers[self.tenant]
        for node in nodeattrs.iterkeys():
            if node not in attribwatchers:
                continue
            attribwatcher = attribwatchers[node]
            for attrname in nodeattrs[node].iterkeys():
                if attrname not in attribwatcher:
                    continue
                for notifierid in attribwatcher[attrname].iterkeys():
                    if notifierid in notifdata:
                        if node in notifdata[notifierid]['nodeattrs']:
                            notifdata[notifierid]['nodeattrs'][node].append(
                                attrname)
                        else:
                            notifdata[notifierid]['nodeattrs'][node] = [
                                attrname]
                    else:
                        notifdata[notifierid] = {
                            'nodeattrs': {node: [attrname]},
                            'callback': attribwatcher[attrname][notifierid]
                        }
        for watcher in notifdata.itervalues():
            callback = watcher['callback']
            try:
                callback(nodeattribs=watcher['nodeattrs'], configmanager=self)
            except Exception:
                global tracelog
                if tracelog is None:
                    tracelog = confluentd.log.Logger('trace')
                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                             event=log.Events.stacktrace)

    def del_nodes(self, nodes):
        if self.tenant in self._nodecollwatchers:
            for watcher in self._nodecollwatchers[self.tenant].itervalues():
                watcher(added=[], deleting=nodes, configmanager=self)
        changeset = {}
        for node in nodes:
            node = node.encode('utf-8')
            if node in self._cfgstore['nodes']:
                self._sync_groups_to_node(node=node, groups=[],
                                          changeset=changeset)
                del self._cfgstore['nodes'][node]
                _mark_dirtykey('nodes', node, self.tenant)
        self._notif_attribwatchers(changeset)
        self._bg_sync_to_file()

    def del_groups(self, groups):
        changeset = {}
        for group in groups:
            if group in self._cfgstore['nodegroups']:
                self._sync_nodes_to_group(group=group, nodes=[],
                                          changeset=changeset)
                del self._cfgstore['nodegroups'][group]
                _mark_dirtykey('nodegroups', group, self.tenant)
        self._notif_attribwatchers(changeset)
        self._bg_sync_to_file()

    def clear_node_attributes(self, nodes, attributes):
        # accumulate all changes into a changeset and push in one go
        changeset = {}
        for node in nodes:
            node = node.encode('utf-8')
            try:
                nodek = self._cfgstore['nodes'][node]
            except KeyError:
                continue
            recalcexpressions = False
            for attrib in attributes:
                if attrib in nodek and 'inheritedfrom' not in nodek[attrib]:
                    # if the attribute is set and not inherited,
                    # delete it and check for inheritence to backfil data
                    del nodek[attrib]
                    self._do_inheritance(nodek, attrib, node, changeset)
                    _addchange(changeset, node, attrib)
                    _mark_dirtykey('nodes', node, self.tenant)
                if ('_expressionkeys' in nodek and
                        attrib in nodek['_expressionkeys']):
                    recalcexpressions = True
            if recalcexpressions:
                exprmgr = _ExpressionFormat(nodek, node)
                self._recalculate_expressions(nodek, formatter=exprmgr,
                                              node=node, changeset=changeset)
        self._notif_attribwatchers(changeset)
        self._bg_sync_to_file()

    def add_node_attributes(self, attribmap):
        for node in attribmap.iterkeys():
            if 'groups' not in attribmap[node]:
                attribmap[node]['groups'] = []
        self.set_node_attributes(attribmap, autocreate=True)

    def set_node_attributes(self, attribmap, autocreate=False):
        # TODO(jbjohnso): multi mgr support, here if we have peers,
        # pickle the arguments and fire them off in eventlet
        # flows to peers, all should have the same result
        newnodes = []
        changeset = {}
        # first do a sanity check of the input upfront
        # this mitigates risk of arguments being partially applied
        for node in attribmap.iterkeys():
            node = node.encode('utf-8')
            if node == '':
                raise ValueError('"{0}" is not a valid node name'.format(node))
            if autocreate is False and node not in self._cfgstore['nodes']:
                raise ValueError("node {0} does not exist".format(node))
            for attrname in attribmap[node].iterkeys():
                attrval = attribmap[node][attrname]
                if (attrname not in allattributes.node or
                        ('type' in allattributes.node[attrname] and
                         not isinstance(
                             attrval,
                             allattributes.node[attrname]['type']))):
                    errstr = "{0} attribute on node {1} is invalid".format(
                        attrname, node)
                    raise ValueError(errstr)
                if attrname == 'groups':
                    for group in attribmap[node]['groups']:
                        if group not in self._cfgstore['nodegroups']:
                            raise ValueError(
                                "group {0} does not exist".format(group))
                    if ('everything' in self._cfgstore['nodegroups'] and
                            'everything' not in attribmap[node]['groups']):
                        attribmap[node]['groups'].append('everything')
        for node in attribmap.iterkeys():
            node = node.encode('utf-8')
            exprmgr = None
            if node not in self._cfgstore['nodes']:
                newnodes.append(node)
                self._cfgstore['nodes'][node] = {}
            cfgobj = self._cfgstore['nodes'][node]
            recalcexpressions = False
            for attrname in attribmap[node].iterkeys():
                if (isinstance(attribmap[node][attrname], str) or
                        isinstance(attribmap[node][attrname], unicode)):
                    newdict = {'value': attribmap[node][attrname]}
                else:
                    newdict = attribmap[node][attrname]
                if 'value' in newdict and attrname.startswith("secret."):
                    newdict['cryptvalue'] = crypt_value(newdict['value'])
                    del newdict['value']
                cfgobj[attrname] = newdict
                if attrname == 'groups':
                    self._sync_groups_to_node(node=node,
                                              groups=attribmap[node]['groups'],
                                              changeset=changeset)
                if ('_expressionkeys' in cfgobj and
                        attrname in cfgobj['_expressionkeys']):
                    recalcexpressions = True
                if 'expression' in cfgobj[attrname]:  # evaluate now
                    if exprmgr is None:
                        exprmgr = _ExpressionFormat(cfgobj, node)
                    cfgobj[attrname] = _decode_attribute(attrname, cfgobj,
                                                         formatter=exprmgr)
                # if any code is watching these attributes, notify
                # them of the change
                _addchange(changeset, node, attrname)
                _mark_dirtykey('nodes', node, self.tenant)
            if recalcexpressions:
                if exprmgr is None:
                    exprmgr = _ExpressionFormat(cfgobj, node)
                self._recalculate_expressions(cfgobj, formatter=exprmgr,
                                              node=node, changeset=changeset)
        self._notif_attribwatchers(changeset)
        if newnodes:
            if self.tenant in self._nodecollwatchers:
                nodecollwatchers = self._nodecollwatchers[self.tenant]
                for watcher in nodecollwatchers.itervalues():
                    watcher(added=newnodes, deleting=[], configmanager=self)
        self._bg_sync_to_file()
        #TODO: wait for synchronization to suceed/fail??)

    def _dump_to_json(self, redact=None):
        """Dump the configuration in json form to output

        password is used to protect the 'secret' attributes in liue of the
        actual in-configuration master key (which will have no clear form
        in the dump

        :param redact: If True, then sensitive password data will be redacted.
                       Other values may be used one day to redact in more
                       complex and interesting ways for non-secret
                       data.

        """
        dumpdata = {}
        for confarea in _config_areas:
            if confarea not in self._cfgstore:
                continue
            dumpdata[confarea] = {}
            for element in self._cfgstore[confarea].iterkeys():
                dumpdata[confarea][element] = \
                    copy.deepcopy(self._cfgstore[confarea][element])
                for attribute in self._cfgstore[confarea][element].iterkeys():
                    if 'inheritedfrom' in dumpdata[confarea][element][attribute]:
                        del dumpdata[confarea][element][attribute]
                    elif (attribute == 'cryptpass' or
                                  'cryptvalue' in
                                  dumpdata[confarea][element][attribute]):
                        if redact is not None:
                            dumpdata[confarea][element][attribute] = '*REDACTED*'
                        else:
                            if attribute == 'cryptpass':
                                target = dumpdata[confarea][element][attribute]
                            else:
                                target = dumpdata[confarea][element][attribute]['cryptvalue']
                            cryptval = []
                            for value in target:
                                cryptval.append(base64.b64encode(value))
                            if attribute == 'cryptpass':
                                dumpdata[confarea][element][attribute] = '!'.join(cryptval)
                            else:
                                dumpdata[confarea][element][attribute]['cryptvalue'] = '!'.join(cryptval)
                    elif isinstance(dumpdata[confarea][element][attribute], set):
                        dumpdata[confarea][element][attribute] = \
                            list(dumpdata[confarea][element][attribute])
        return json.dumps(
            dumpdata, sort_keys=True, indent=4, separators=(',', ': '))



    @classmethod
    def _read_from_path(cls):
        global _cfgstore
        _cfgstore = {}
        rootpath = cls._cfgdir
        _load_dict_from_dbm(['globals'], os.path.join(rootpath, "globals"))
        for confarea in _config_areas:
            _load_dict_from_dbm(['main', confarea], os.path.join(rootpath, confarea))
        try:
            for tenant in os.listdir(os.path.join(rootpath, 'tenants')):
                for confarea in _config_areas:
                    _load_dict_from_dbm(
                        ['main', tenant, confarea],
                        os.path.join(rootpath, tenant, confarea))
        except OSError:
            pass

    @classmethod
    def shutdown(cls):
        cls._bg_sync_to_file()
        if cls._cfgwriter is not None:
            cls._cfgwriter.join()
        sys.exit(0)

    @classmethod
    def _bg_sync_to_file(cls):
        with cls._syncstate:
            if cls._syncrunning:
                cls._writepending = True
                return
            cls._syncrunning = True
        # if the thread is exiting, join it to let it close, just in case
        if cls._cfgwriter is not None:
            cls._cfgwriter.join()
        cls._cfgwriter = threading.Thread(target=cls._sync_to_file)
        cls._cfgwriter.start()

    @classmethod
    def _sync_to_file(cls):
        if 'dirtyglobals' in _cfgstore:
            with _dirtylock:
                dirtyglobals = copy.deepcopy(_cfgstore['dirtyglobals'])
                del _cfgstore['dirtyglobals']
            _mkpath(cls._cfgdir)
            globalf = dbm.open(os.path.join(cls._cfgdir, "globals"), 'c', 384)  # 0600
            try:
                for globalkey in dirtyglobals:
                    if globalkey in _cfgstore['globals']:
                        globalf[globalkey] = \
                            cPickle.dumps(_cfgstore['globals'][globalkey])
                    else:
                        if globalkey in globalf:
                            del globalf[globalkey]
            finally:
                globalf.close()
        if 'dirtykeys' in _cfgstore:
            with _dirtylock:
                currdirt = copy.deepcopy(_cfgstore['dirtykeys'])
                del _cfgstore['dirtykeys']
            for tenant in currdirt.iterkeys():
                dkdict = currdirt[tenant]
                if tenant is None:
                    pathname = cls._cfgdir
                    currdict = _cfgstore['main']
                else:
                    pathname = os.path.join(cls._cfgdir, 'tenants', tenant)
                    currdict = _cfgstore['tenant'][tenant]
                for category in dkdict.iterkeys():
                    _mkpath(pathname)
                    dbf = dbm.open(os.path.join(pathname, category), 'c', 384)  # 0600
                    try:
                        for ck in dkdict[category]:
                            if ck not in currdict[category]:
                                if ck in dbf:
                                    del dbf[ck]
                            else:
                                dbf[ck] = cPickle.dumps(currdict[category][ck])
                    finally:
                        dbf.close()
        willrun = False
        with cls._syncstate:
            if cls._writepending:
                cls._writepending = False
                willrun = True
            else:
                cls._syncrunning = False
        if willrun:
            return cls._sync_to_file()

    def _recalculate_expressions(self, cfgobj, formatter, node, changeset):
        for key in cfgobj.iterkeys():
            if not isinstance(cfgobj[key], dict):
                continue
            if 'expression' in cfgobj[key]:
                cfgobj[key] = _decode_attribute(key, cfgobj,
                                                formatter=formatter)
                _addchange(changeset, node, key)
            elif ('cryptvalue' not in cfgobj[key] and
                    'value' not in cfgobj[key]):
                # recurse for nested structures, with some hint that
                # it might indeed be a nested structure
                self._recalculate_expressions(cfgobj[key], formatter, node,
                                              changeset)

def _dump_keys(password):
    if _masterkey is None or _masterintegritykey is None:
        init_masterkey()
    cryptkey = _format_key(_masterkey, password=password)
    cryptkey = '!'.join(map(base64.b64encode, cryptkey['passphraseprotected']))
    integritykey = _format_key(_masterintegritykey, password=password)
    integritykey = '!'.join(map(base64.b64encode, integritykey['passphraseprotected']))
    return json.dumps({'cryptkey': cryptkey, 'integritykey': integritykey},
                      sort_keys=True, indent=4, separators=(',', ': '))


def dump_db_to_directory(location, password, redact=None):
    with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
        cfgfile.write(_dump_keys(password))
        cfgfile.write('\n')
    with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
        cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
        cfgfile.write('\n')
    try:
        for tenant in os.listdir(
                os.path.join(ConfigManager._cfgdir, '/tenants/')):
            with open(os.path.join(location, tenant + '.json'), 'w') as cfgfile:
                cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
                    redact=redact))
                cfgfile.write('\n')
    except OSError:
        pass


try:
    ConfigManager._read_from_path()
except IOError:
    _cfgstore = {}


# some unit tests worth implementing:
# set group attribute on lower priority group, result is that node should not
# change
# after that point, then unset on the higher priority group, lower priority
# group should get it then
# rinse and repeat for set on node versus set on group
# clear group attribute and assure than it becomes unset on all nodes
# set various expressions