From f048e94c8019564e3ce2d24c97e3a7b41c5e54ba Mon Sep 17 00:00:00 2001 From: Bolke de Bruin Date: Thu, 29 Dec 2016 16:08:03 +0100 Subject: [PATCH] [AIRFLOW-721] Descendant process can disappear before termination There is a race condition in helpers.py's kill_descendant_processes that checks for running processes and then tries to terminate them. This is not done atomically allowing for a small window where a PID can disappear before termination. Closes #1963 from bolkedebruin/AIRFLOW-721 --- airflow/utils/helpers.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 24fc31020d5c4..23fda0335685c 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -205,10 +205,16 @@ def kill_descendant_processes(logger, pids_to_kill=None): logger.warn("Terminating descendant processes of {} PID: {}" .format(this_process.cmdline(), this_process.pid)) - for descendant in descendant_processes: + + temp_processes = descendant_processes[:] + for descendant in temp_processes: logger.warn("Terminating descendant process {} PID: {}" .format(descendant.cmdline(), descendant.pid)) - descendant.terminate() + try: + descendant.terminate() + except psutil.NoSuchProcess: + descendant_processes.remove(descendant) + logger.warn("Waiting up to {}s for processes to exit..." .format(TIME_TO_WAIT_AFTER_SIGTERM)) try: @@ -228,8 +234,11 @@ def kill_descendant_processes(logger, pids_to_kill=None): for descendant in descendant_processes: logger.warn("Killing descendant process {} PID: {}" .format(descendant.cmdline(), descendant.pid)) - descendant.kill() - descendant.wait() + try: + descendant.kill() + descendant.wait() + except psutil.NoSuchProcess: + pass logger.warn("Killed all descendant processes of {} PID: {}" .format(this_process.cmdline(), this_process.pid))