[AIRFLOW-721] Descendant process can disappear before termination

There is a race condition in helpers.py's
kill_descendant_processes
that checks for running processes and then tries
to terminate them.
This is not done atomically allowing for a small
window where a PID
can disappear before termination.

Closes apache#1963 from bolkedebruin/AIRFLOW-721

airflow/utils/helpers.py

@@ -205,10 +205,16 @@ def kill_descendant_processes(logger, pids_to_kill=None):
     logger.warn("Terminating descendant processes of {} PID: {}"
                 .format(this_process.cmdline(),
                         this_process.pid))
-    for descendant in descendant_processes:
+
+    temp_processes = descendant_processes[:]
+    for descendant in temp_processes:
         logger.warn("Terminating descendant process {} PID: {}"
                     .format(descendant.cmdline(), descendant.pid))
-        descendant.terminate()
+        try:
+            descendant.terminate()
+        except psutil.NoSuchProcess:
+            descendant_processes.remove(descendant)
+
     logger.warn("Waiting up to {}s for processes to exit..."
                 .format(TIME_TO_WAIT_AFTER_SIGTERM))
     try:
@@ -228,8 +234,11 @@ def kill_descendant_processes(logger, pids_to_kill=None):
         for descendant in descendant_processes:
             logger.warn("Killing descendant process {} PID: {}"
                         .format(descendant.cmdline(), descendant.pid))
-            descendant.kill()
-            descendant.wait()
+            try:
+                descendant.kill()
+                descendant.wait()
+            except psutil.NoSuchProcess:
+                pass
         logger.warn("Killed all descendant processes of {} PID: {}"
                     .format(this_process.cmdline(),
                             this_process.pid))