Allow aborting when the handlers process exit

jhannus · Apr 3, 2017 · 854cadc · 854cadc
1 parent 1c0935b
commit 854cadc
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/conf/diamond.conf.example b/conf/diamond.conf.example
@@ -40,6 +40,8 @@ handlers_path = /usr/share/diamond/handlers/
 # When metric queue is full, new metrics are dropped.
 metric_queue_size = 16384
 
+# Abort the diamond process if the handler process exits
+abort_on_handlers_process_exit = False
 
 ################################################################################
 ### Options for handlers

diff --git a/src/diamond/server.py b/src/diamond/server.py
@@ -25,6 +25,7 @@
 from diamond.utils.classes import load_include_path
 
 from diamond.utils.config import load_config
+from diamond.utils.config import str_to_bool
 
 from diamond.utils.scheduler import collector_process
 from diamond.utils.scheduler import handler_process
@@ -115,14 +116,14 @@ def run(self):
         self.handler_queue = QueueHandler(
             config=self.config, queue=self.metric_queue, log=self.log)
 
-        process = multiprocessing.Process(
+        handlers_process = multiprocessing.Process(
             name="Handlers",
             target=handler_process,
             args=(self.handlers, self.metric_queue, self.log),
         )
 
-        process.daemon = True
-        process.start()
+        handlers_process.daemon = True
+        handlers_process.start()
 
         #######################################################################
         # Signals
@@ -203,6 +204,12 @@ def run(self):
                     process.daemon = True
                     process.start()
 
+                if not handlers_process.is_alive():
+                    self.log.error('Handlers process exited')
+                    if (str_to_bool(self.config['server'].get(
+                            'abort_on_handlers_process_exit', 'False'))):
+                        raise Exception('Handlers process exited')
+
                 ##############################################################
 
                 time.sleep(1)