xCAT/confluent
2
0
Fork 0
mirror of https://github.com/xcat2/confluent.git synced 2025-01-17 13:13:18 +00:00
Code Issues Projects Releases Wiki Activity

After 60 seconds of 'connect' limbo, kick a connection attempt

Browse Source 
Occasionally it was observed that systems would be just stuck in 'connect',
provide a backup system to detect and forcibly kick the console in such a case.
This commit is contained in:
Jarrod Johnson 2016-05-19 15:39:04 -04:00
parent ec02097b52
commit e4aa873141
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
confluent_server/confluent/consoleserver.py
View File

@ -180,6 +180,8 @@ class ConsoleHandler(object):
self.connectionthread.kill()
self.connectionthread = None
self.connectionthread = eventlet.spawn(self._connect_backend)
# in 60 seconds, kick connect() if it didn't complete
self.kickconnect = eventlet.spawn_after(60, self._connect)
def _connect_backend(self):
if self._console:
@ -217,6 +219,9 @@ class ConsoleHandler(object):
try:
self._console.connect(self.get_console_output)
except exc.TargetEndpointBadCredentials:
if self.kickconnect:
self.kickconnect.cancel()
self.kickconnect = None
self.error = 'badcredentials'
self.connectstate = 'unconnected'
self._send_rcpts({'connectstate': self.connectstate,
@ -226,6 +231,9 @@ class ConsoleHandler(object):
self.reconnect = eventlet.spawn_after(retrytime, self._connect)
return
except exc.TargetEndpointUnreachable:
if self.kickconnect:
self.kickconnect.cancel()
self.kickconnect = None
self.error = 'unreachable'
self.connectstate = 'unconnected'
self._send_rcpts({'connectstate': self.connectstate,
@ -237,6 +245,9 @@ class ConsoleHandler(object):
except Exception:
_tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
event=log.Events.stacktrace)
if self.kickconnect:
self.kickconnect.cancel()
self.kickconnect = None
self.error = 'unknown'
self.connectstate = 'unconnected'
self._send_rcpts({'connectstate': self.connectstate,
@ -248,6 +259,9 @@ class ConsoleHandler(object):
self._got_connected()
def _got_connected(self):
if self.kickconnect:
self.kickconnect.cancel()
self.kickconnect = None
self.connectstate = 'connected'
self.log(
logdata='console connected', ltype=log.DataTypes.event,