Merge branch 'windowssupport' of git://github.com/jjohnson42/confluent into jjohnson42-windowssupport

Additionally, rework locking back to not require any extra files

confluent_client/bin/confetty

@@ -41,7 +41,6 @@
 # esc-( would interfere with normal esc use too much
 # ~ I will not use for now...
 
-import fcntl
 import math
 import getpass
 import optparse
@@ -50,9 +49,13 @@ import select
 import shlex
 import socket
 import sys
-import termios
 import time
-import tty
+try:
+    import fcntl
+    import termios
+    import tty
+except ImportError:
+    pass
 
 exitcode = 0
 consoleonly = False
@@ -72,8 +75,11 @@ conserversequence = '\x05c'  # ctrl-e, c
 
 oldtcattr = None
 fd = sys.stdin
-if fd.isatty():
-    oldtcattr = termios.tcgetattr(fd.fileno())
+try:
+    if fd.isatty():
+        oldtcattr = termios.tcgetattr(fd.fileno())
+except NameError:
+    pass
 netserver = None
 laststate = {}
 

confluent_server/bin/confluent

@@ -22,14 +22,17 @@ path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     # if installed into system path, do not muck with things
     sys.path.append(path)
-from confluent import main
+import confluent.main
 
 #import cProfile
 #import time
 #p = cProfile.Profile(time.clock)
 #p.enable()
 #try:
-main.run()
+import multiprocessing
+if __name__ == '__main__':
+    multiprocessing.freeze_support()
+    confluent.main.run()
 #except:
 #   pass
 #p.disable()

confluent_server/bin/confluentsrv.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2014 IBM Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    # if installed into system path, do not muck with things
+    sys.path.append(path)
+import confluent.main
+
+#import cProfile
+#import time
+#p = cProfile.Profile(time.clock)
+#p.enable()
+#try:
+import multiprocessing
+if __name__ == '__main__':
+    multiprocessing.freeze_support()
+    confluent.main.run()
+#except:
+#   pass
+#p.disable()
+#p.print_stats(sort='cumulative')
+#p.print_stats(sort='time')

confluent_server/confluent/auth.py

@@ -26,7 +26,10 @@ import Crypto.Protocol.KDF as KDF
 import hashlib
 import hmac
 import multiprocessing
-import PAM
+try:
+    import PAM
+except ImportError:
+    pass
 import time
 
 _pamservice = 'confluent'
@@ -161,6 +164,8 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
         pammy.acct_mgmt()
         del pammy
         return authorize(user, element, tenant, skipuserobj=False)
+    except NameError:
+        pass
     except PAM.error:
         if credobj.haspam:
             return None

confluent_server/confluent/config/conf.py

@@ -17,12 +17,16 @@
 
 #This defines config variable to store the global configuration for confluent
 import ConfigParser
+import os
 
 _config = None
 
 def init_config():
     global _config
     configfile = "/etc/confluent/service.cfg"
+    if os.name == 'nt':
+        configfile = os.path.join(os.getentv('SystemDrive'), '\\ProgramData',
+                                  'confluent', 'cfg', 'service.cfg')
     _config = ConfigParser.ConfigParser()
     _config.read(configfile)
 

confluent_server/confluent/config/configmanager.py

@@ -415,7 +415,11 @@ def hook_new_configmanagers(callback):
 
 
 class ConfigManager(object):
-    _cfgdir = "/etc/confluent/cfg/"
+    if os.name == 'nt':
+        _cfgdir = os.path.join(
+            os.getenv('SystemDrive'), '\\ProgramData', 'confluent', 'cfg')
+    else:
+        _cfgdir = "/etc/confluent/cfg"
     _cfgwriter = None
     _writepending = False
     _syncrunning = False
@@ -1207,15 +1211,15 @@ class ConfigManager(object):
         global _cfgstore
         _cfgstore = {}
         rootpath = cls._cfgdir
-        _load_dict_from_dbm(['globals'], rootpath + "/globals")
+        _load_dict_from_dbm(['globals'], os.path.join(rootpath, "globals"))
         for confarea in _config_areas:
-            _load_dict_from_dbm(['main', confarea], rootpath + "/" + confarea)
+            _load_dict_from_dbm(['main', confarea], os.path.join(rootpath, confarea))
         try:
-            for tenant in os.listdir(rootpath + '/tenants/'):
+            for tenant in os.listdir(os.path.join(rootpath, 'tenants')):
                 for confarea in _config_areas:
                     _load_dict_from_dbm(
                         ['main', tenant, confarea],
-                        "%s/%s/%s" % (rootpath, tenant, confarea))
+                        os.path.join(rootpath, tenant, confarea))
         except OSError:
             pass
 
@@ -1246,7 +1250,7 @@ class ConfigManager(object):
                 dirtyglobals = copy.deepcopy(_cfgstore['dirtyglobals'])
                 del _cfgstore['dirtyglobals']
             _mkpath(cls._cfgdir)
-            globalf = dbm.open(cls._cfgdir + "/globals", 'c', 384)  # 0600
+            globalf = dbm.open(os.path.join(cls._cfgdir, "globals"), 'c', 384)  # 0600
             try:
                 for globalkey in dirtyglobals:
                     if globalkey in _cfgstore['globals']:
@@ -1267,11 +1271,11 @@ class ConfigManager(object):
                     pathname = cls._cfgdir
                     currdict = _cfgstore['main']
                 else:
-                    pathname = cls._cfgdir + '/tenants/' + tenant + '/'
+                    pathname = os.path.join(cls._cfgdir, 'tenants', tenant)
                     currdict = _cfgstore['tenant'][tenant]
                 for category in dkdict.iterkeys():
                     _mkpath(pathname)
-                    dbf = dbm.open(pathname + category, 'c', 384)  # 0600
+                    dbf = dbm.open(os.path.join(pathname, category), 'c', 384)  # 0600
                     try:
                         for ck in dkdict[category]:
                             if ck not in currdict[category]:
@@ -1325,7 +1329,8 @@ def dump_db_to_directory(location, password, redact=None):
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
         cfgfile.write('\n')
     try:
-        for tenant in os.listdir(ConfigManager._cfgdir + '/tenants/'):
+        for tenant in os.listdir(
+                os.path.join(ConfigManager._cfgdir, '/tenants/')):
             with open(os.path.join(location, tenant + '.json'), 'w') as cfgfile:
                 cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
                     redact=redact))

confluent_server/confluent/core.py

@@ -39,7 +39,10 @@ import confluent.interface.console as console
 import confluent.exceptions as exc
 import confluent.messages as msg
 import confluent.noderange as noderange
-import confluent.shellmodule as shellmodule
+try:
+    import confluent.shellmodule as shellmodule
+except ImportError:
+    pass
 import itertools
 import os
 import sys

confluent_server/confluent/httpapi.py

@@ -25,7 +25,7 @@ import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages
 import confluent.core as pluginapi
-import confluent.tlvdata as tlvdata
+import confluent.tlvdata
 import confluent.util as util
 import copy
 import eventlet
@@ -36,6 +36,7 @@ import time
 import urlparse
 import eventlet.wsgi
 #scgi = eventlet.import_patched('flup.server.scgi')
+tlvdata = confluent.tlvdata
 
 
 auditlog = None

confluent_server/confluent/log.py

@@ -64,7 +64,6 @@ import confluent.config.configmanager
 import confluent.config.conf as conf
 import confluent.exceptions as exc
 import eventlet
-import fcntl
 import glob
 import json
 import os
@@ -74,6 +73,22 @@ import struct
 import time
 import traceback
 
+try:
+    from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_SH
+except ImportError:
+    if os.name == 'nt':
+        import msvcrt
+        LOCK_SH = msvcrt.LK_LOCK  # no shared, degrade to exclusive
+        LOCK_EX = msvcrt.LK_LOCK
+        LOCK_UN = msvcrt.LK_UNLCK
+        def flock(file, flag):
+            oldoffset = file.tell()
+            file.seek(0)
+            msvcrt.locking(file.fileno(), flag, 1)
+            file.seek(oldoffset)
+    else:
+        raise
+
 # on conserving filehandles:
 # upon write, if file not open, open it for append
 # upon write, schedule/reschedule closing filehandle in 15 seconds
@@ -114,16 +129,18 @@ class BaseRotatingHandler(object):
         Use the specified filename for streamed logging
         """
         self.filepath = filepath
-        self.textpath = self.filepath +logname
-        self.binpath = self.filepath + logname + ".cbl"
+        self.textpath = os.path.join(self.filepath, logname)
+        self.binpath = os.path.join(self.filepath, logname + ".cbl")
         self.textfile = None
         self.binfile = None
 
     def open(self):
         if self.textfile is None:
             self.textfile = open(self.textpath, mode='ab')
+            self.textfile.seek(0, 2)
         if self.binfile is None:
             self.binfile = open(self.binpath, mode='ab')
+            self.binfile.seek(0, 2)
         return self.textfile, self.binfile
 
     def try_emit(self, binrecord, textrecord):
@@ -487,10 +504,15 @@ class Logger(object):
         self.initialized = True
         self.filepath = confluent.config.configmanager.get_global("logdirectory")
         if self.filepath is None:
-            self.filepath = "/var/log/confluent/"
+            if os.name == 'nt':
+                self.filepath = os.path.join(
+                    os.getenv('SystemDrive'), '\\ProgramData', 'confluent',
+                    'logs')
+            else:
+                self.filepath = "/var/log/confluent"
         self.isconsole = console
         if console:
-            self.filepath += "consoles/"
+            self.filepath = os.path.join(self.filepath, "consoles")
         if not os.path.isdir(self.filepath):
             os.makedirs(self.filepath, 448)
         self.writer = None
@@ -524,7 +546,7 @@ class Logger(object):
             elif not self.isconsole:
                 textdate = time.strftime(
                     '%b %d %H:%M:%S ', time.localtime(tstamp))
-            self._lock(fcntl.LOCK_EX)
+            flock(self.textfile, LOCK_EX)
             offset = textfile.tell() + len(textdate)
             datalen = len(data)
             eventaux = entry[4]
@@ -547,6 +569,9 @@ class Logger(object):
             if not files:
                 self.handler.emit(binrecord, textrecord)
             else:
+                # Release the lock that was held to preserve the correct offset
+                # binrecord value is to be discarded anyway
+                flock(self.textfile, LOCK_UN)
                 # Log the rolling event at first, then log the last data
                 # which cause the rolling event.
                 to_bfile, to_tfile = files
@@ -554,7 +579,6 @@ class Logger(object):
                 roll_data = "rename:%s>%s" % (self.handler.textpath, to_tfile)
                 self.logentries.appendleft([DataTypes.event, tstamp, roll_data,
                                             Events.logrollover, None])
-            self._lock(fcntl.LOCK_UN)
         if self.closer is None:
             self.closer = eventlet.spawn_after(15, self.closelog)
         self.writer = None
@@ -578,7 +602,7 @@ class Logger(object):
             binfile = open(binpath, mode='r')
         except IOError:
             return '', 0, 0
-        self._lock(fcntl.LOCK_SH)
+        flock(binfile, LOCK_SH)
         binfile.seek(0, 2)
         binidx = binfile.tell()
         currsize = 0
@@ -586,6 +610,7 @@ class Logger(object):
         termstate = None
         recenttimestamp = 0
         access_last_rename = False
+        flock(textfile, LOCK_SH)
         while binidx > 0 and currsize < size:
             binidx -= 16
             binfile.seek(binidx, 0)
@@ -604,11 +629,13 @@ class Logger(object):
                                                              datalen)
                 # Rolling event detected, close the current bin file, then open
                 # the renamed bin file.
+                flock(binfile, LOCK_UN)
                 binfile.close()
                 try:
                     binfile = open(binpath, mode='r')
                 except IOError:
                     return '', 0, 0
+                flock(binfile, LOCK_SH)
                 binfile.seek(0, 2)
                 binidx = binfile.tell()
             elif ltype != 2:
@@ -619,11 +646,13 @@ class Logger(object):
             offsets.append((offset, datalen, textpath))
             if termstate is None:
                 termstate = eventaux
+        flock(binfile, LOCK_UN)
         binfile.close()
         textdata = ''
         while offsets:
             (offset, length, textpath) = offsets.pop()
             if textfile.name != textpath:
+                flock(textfile, LOCK_UN)
                 textfile.close()
                 try:
                     textfile = open(textpath)
@@ -631,7 +660,7 @@ class Logger(object):
                     return '', 0, 0
             textfile.seek(offset, 0)
             textdata += textfile.read(length)
-        self._lock(fcntl.LOCK_UN)
+        flock(textfile, LOCK_UN)
         textfile.close()
         if termstate is None:
             termstate = 0

confluent_server/confluent/main.py

@@ -33,10 +33,18 @@ import confluent.consoleserver as consoleserver
 import confluent.core as confluentcore
 import confluent.httpapi as httpapi
 import confluent.log as log
-import confluent.sockapi as sockapi
+try:
+    import confluent.sockapi as sockapi
+except ImportError:
+    #On platforms without pwd, give up on the sockapi in general and be http
+    #only for now
+    pass
 import eventlet
 #import eventlet.backdoor as backdoor
-import fcntl
+try:
+    import fcntl
+except ImportError:
+    pass
 #import multiprocessing
 import sys
 import os
@@ -44,6 +52,8 @@ import signal
 
 
 def _daemonize():
+    if not 'fork' in os.__dict__:
+        return
     thispid = os.fork()
     if thispid > 0:
         os.waitpid(thispid, 0)
@@ -110,6 +120,8 @@ def terminate(signalname, frame):
 
 
 def doexit():
+    if 'fcntl' not in locals():
+        return
     pidfile = open('/var/run/confluent/pid')
     pid = pidfile.read()
     if pid == str(os.getpid()):
@@ -125,7 +137,8 @@ def _initsecurity(config):
 
 
 def run():
-    _checkpidfile()
+    if 'fcntl' in locals():
+        _checkpidfile()
     conf.init_config()
     try:
         config = conf.get_config()
@@ -140,7 +153,8 @@ def run():
         doexit()
         raise
     _daemonize()
-    _updatepidfile()
+    if 'fcntl' in locals():
+        _updatepidfile()
     auth.init_auth()
     signal.signal(signal.SIGINT, terminate)
     signal.signal(signal.SIGTERM, terminate)
@@ -154,8 +168,11 @@ def run():
     consoleserver.start_console_sessions()
     webservice = httpapi.HttpApi(http_bind_host, http_bind_port)
     webservice.start()
-    sockservice = sockapi.SockApi(sock_bind_host, sock_bind_port)
-    sockservice.start()
+    try:
+        sockservice = sockapi.SockApi(sock_bind_host, sock_bind_port)
+        sockservice.start()
+    except NameError:
+        pass
     atexit.register(doexit)
     while 1:
         eventlet.sleep(100)

confluent_server/confluentsrv.spec

@@ -0,0 +1,30 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+a = Analysis(['c:/Python27/Scripts/confluentsrv.py'],
+             pathex=[],
+             hiddenimports=['pyghmi.constants', 'pyghmi.exceptions', 'pyghmi.ipmi.console', 'pyghmi.ipmi.private.constants', 'pyghmi.ipmi.private', 'pyghmi.ipmi.private.session', 'pyghmi.ipmi.command', 'pyghmi.ipmi.events', 'pyghmi.ipmi.fru', 'pyghmi.ipmi.private.spd', 'pyghmi.ipmi.oem.lookup', 'pyghmi.ipmi.oem.generic', 'pyghmi.ipmi.oem.lenovo', 'pyghmi.ipmi.private.util', 'pyghmi.ipmi.sdr'],
+             hookspath=None,
+             runtime_hooks=None,
+             excludes=None,
+             cipher=block_cipher)
+pyz = PYZ(a.pure,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          exclude_binaries=True,
+          name='confluentsrv.exe',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+	       Tree('confluent/plugins', prefix='confluent/plugins'),
+               strip=None,
+               upx=True,
+               name='confluentsrv')