From 563a2b11c3e6984b639d84309f008edc42e76813 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 11 Apr 2017 14:39:29 -0400 Subject: [PATCH 1/2] Change the stdout/stderr/trace logs to not buffer things over time If something went completely off the rails, it could easily fill up lots of memory with log entries in the 2 seconds it would buffer. For now disable the buffering on key debug logs, as the main purpose was reducing IOPs in the per-node console logs anyway. A future behavior may be to also limit the size and/or number of outstanding log entries before committing to disk. --- confluent_server/confluent/log.py | 12 ++++++++---- confluent_server/confluent/main.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/confluent_server/confluent/log.py b/confluent_server/confluent/log.py index 5e7c33a1..c916dcfd 100644 --- a/confluent_server/confluent/log.py +++ b/confluent_server/confluent/log.py @@ -505,11 +505,12 @@ class Logger(object): else: return object.__new__(cls) - def __init__(self, logname, console=False, tenant=None): + def __init__(self, logname, console=False, tenant=None, buffered=True): if hasattr(self, 'initialized'): # we are just a copy of the same object return self.initialized = True + self.buffered = buffered self.filepath = confluent.config.configmanager.get_global("logdirectory") if self.filepath is None: if os.name == 'nt': @@ -727,8 +728,11 @@ class Logger(object): else: self.logentries.append( [ltype, timestamp, logdata, event, eventdata]) - if self.writer is None: - self.writer = eventlet.spawn_after(2, self.writedata) + if self.buffered: + if self.writer is None: + self.writer = eventlet.spawn_after(2, self.writedata) + else: + self.writedata() def closelog(self): self.handler.close() @@ -746,6 +750,6 @@ def log(logdata=None, ltype=None, event=0, eventdata=None): def logtrace(): global tracelog if tracelog is None: - tracelog = Logger('trace') + tracelog = Logger('trace', buffered=False) tracelog.log(traceback.format_exc(), ltype=DataTypes.event, event=Events.stacktrace) \ No newline at end of file diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py index cd26fba5..7457dbdf 100644 --- a/confluent_server/confluent/main.py +++ b/confluent_server/confluent/main.py @@ -77,8 +77,8 @@ def _daemonize(): os.open(os.devnull, os.O_RDWR) os.dup2(0, 1) os.dup2(0, 2) - sys.stdout = log.Logger('stdout') - sys.stderr = log.Logger('stderr') + sys.stdout = log.Logger('stdout', buffered=False) + sys.stderr = log.Logger('stderr', buffered=False) def _updatepidfile(): From e230f803ce85be23415bb3fdaedb403686e08888 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 11 Apr 2017 15:41:26 -0400 Subject: [PATCH 2/2] Correct missing argument __new__ was missed in the last commit. --- confluent_server/confluent/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/log.py b/confluent_server/confluent/log.py index c916dcfd..c63ddf12 100644 --- a/confluent_server/confluent/log.py +++ b/confluent_server/confluent/log.py @@ -494,7 +494,7 @@ class Logger(object): False, events will be formatted like syslog: date: message """ - def __new__(cls, logname, console=False, tenant=None): + def __new__(cls, logname, console=False, tenant=None, buffered=True): global _loggers if console: relpath = 'consoles/' + logname