From de383630cfd67ae610914c7f3f636d2227ff6c54 Mon Sep 17 00:00:00 2001
From: Jarrod Johnon <jjohnson2@lenovo.com>
Date: Mon, 19 Jan 2015 17:05:29 -0500
Subject: [PATCH] Gracefully trap config change callback errors

When a change callback handler raises an exception,
log it and move on to next handler.  This prevents
the handler from screwing up the caller or fellow
callback handlers.  Unfortunately this means that
no one notices the issue until checking the trace
log file, but it produces more peculiar behavior
as it stands without this change.
---
 confluent_server/confluent/config/configmanager.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py
index 7594794f..072d1351 100644
--- a/confluent_server/confluent/config/configmanager.py
+++ b/confluent_server/confluent/config/configmanager.py
@@ -1,6 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
+# Copyright 2015 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -64,6 +65,7 @@ import anydbm as dbm
 import ast
 import base64
 import confluent.config.attributes as allattributes
+import confluent.log as log
 import confluent.util
 import copy
 import cPickle
@@ -82,6 +84,7 @@ _masterkey = None
 _masterintegritykey = None
 _dirtylock = threading.RLock()
 _config_areas = ('nodegroups', 'nodes', 'usergroups', 'users')
+tracelog = None
 
 def _mkpath(pathname):
     try:
@@ -917,7 +920,14 @@ class ConfigManager(object):
                         }
         for watcher in notifdata.itervalues():
             callback = watcher['callback']
-            callback(nodeattribs=watcher['nodeattrs'], configmanager=self)
+            try:
+                callback(nodeattribs=watcher['nodeattrs'], configmanager=self)
+            except Exception:
+                global tracelog
+                if tracelog is None:
+                    tracelog = log.Logger('trace')
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)
 
     def del_nodes(self, nodes):
         if self.tenant in self._nodecollwatchers: