RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-22 18:47:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

prometheus: Fix prometheus shutdown/restart

Browse Source 
The prometheus serve method uses block which it should not as it can
cause several problems when shutting down (restarting) ceph-mgr. As a
result the restart of the ceph-mgr module will block until time-out if
the prometheus module is enabled. This fixes the problem by using a
simple simple shutdown_event instead of block() and stopping the server
afterwards.

Signed-off-by: Boris Ranto <branto@redhat.com>
This commit is contained in:
Boris Ranto 2018-05-01 01:06:50 +02:00
parent 4dde3d355f
commit a662caa79b
1 changed files with 18 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/pybind/mgr/prometheus/module.py
View File

@ -4,6 +4,7 @@ import errno
import math import math
import os import os
import socket import socket
import threading
from collections import OrderedDict from collections import OrderedDict
from mgr_module import MgrModule, MgrStandbyModule from mgr_module import MgrModule, MgrStandbyModule
@ -341,6 +342,7 @@ class Module(MgrModule):
super(Module, self).__init__(*args, **kwargs) super(Module, self).__init__(*args, **kwargs)
self.metrics = Metrics() self.metrics = Metrics()
self.schema = OrderedDict() self.schema = OrderedDict()
self.shutdown_event = threading.Event()
_global_instance['plugin'] = self _global_instance['plugin'] = self
def get_health(self): def get_health(self):
@ -627,16 +629,22 @@ class Module(MgrModule):
self.log.info('Starting engine...') self.log.info('Starting engine...')
cherrypy.engine.start() cherrypy.engine.start()
self.log.info('Engine started.') self.log.info('Engine started.')
cherrypy.engine.block() # wait for the shutdown event
self.shutdown_event.wait()
self.shutdown_event.clear()
cherrypy.engine.stop()
self.log.info('Engine stopped.')
def shutdown(self): def shutdown(self):
self.log.info('Stopping engine...') self.log.info('Stopping engine...')
cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) self.shutdown_event.set()
cherrypy.engine.exit()
self.log.info('Stopped engine')
class StandbyModule(MgrStandbyModule): class StandbyModule(MgrStandbyModule):
def __init__(self, *args, **kwargs):
super(StandbyModule, self).__init__(*args, **kwargs)
self.shutdown_event = threading.Event()
def serve(self): def serve(self):
server_addr = self.get_localized_config('server_addr', '::') server_addr = self.get_localized_config('server_addr', '::')
server_port = self.get_localized_config('server_port', DEFAULT_PORT) server_port = self.get_localized_config('server_port', DEFAULT_PORT)
@ -671,12 +679,14 @@ class StandbyModule(MgrStandbyModule):
cherrypy.tree.mount(Root(), '/', {}) cherrypy.tree.mount(Root(), '/', {})
self.log.info('Starting engine...') self.log.info('Starting engine...')
cherrypy.engine.start() cherrypy.engine.start()
self.log.info("Waiting for engine...")
cherrypy.engine.wait(state=cherrypy.engine.states.STOPPED)
self.log.info('Engine started.') self.log.info('Engine started.')
# Wait for shutdown event
self.shutdown_event.wait()
self.shutdown_event.clear()
cherrypy.engine.stop()
self.log.info('Engine stopped.')
def shutdown(self): def shutdown(self):
self.log.info("Stopping engine...") self.log.info("Stopping engine...")
cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) self.shutdown_event.set()
cherrypy.engine.stop()
self.log.info("Stopped engine") self.log.info("Stopped engine")