Bug 1539631 - ignore 409 errors cancelling tasks; r=tomprince

Differential Revision: https://phabricator.services.mozilla.com/D28899

--HG--
extra : moz-landing-system : lando

taskcluster/taskgraph/actions/cancel.py

@@ -6,9 +6,14 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import logging
+import requests
+
 from taskgraph.util.taskcluster import cancel_task
 from .registry import register_callback_action
 
+logger = logging.getLogger(__name__)
+
 
 @register_callback_action(
     title='Cancel Task',
@@ -24,4 +29,13 @@
 def cancel_action(parameters, graph_config, input, task_group_id, task_id):
     # Note that this is limited by the scopes afforded to generic actions to
     # only cancel tasks with the level-specific schedulerId.
-    cancel_task(task_id, use_proxy=True)
+    try:
+        cancel_task(task_id, use_proxy=True)
+    except requests.HTTPError as e:
+        if e.response.status_code == 409:
+            # A 409 response indicates that this task is past its deadline.  It
+            # cannot be cancelled at this time, but it's also not running
+            # anymore, so we can ignore this error.
+            logger.info('Task is past its deadline and cannot be cancelled.'.format(task_id))
+            return
+        raise

taskcluster/taskgraph/actions/cancel_all.py

@@ -9,6 +9,7 @@
 import concurrent.futures as futures
 import logging
 import os
+import requests
 
 from taskgraph.util.taskcluster import (
     list_task_group_incomplete_tasks,
@@ -35,7 +36,17 @@
 def cancel_all_action(parameters, graph_config, input, task_group_id, task_id):
     def do_cancel_task(task_id):
         logger.info('Cancelling task {}'.format(task_id))
-        cancel_task(task_id, use_proxy=True)
+        try:
+            cancel_task(task_id, use_proxy=True)
+        except requests.HTTPError as e:
+            if e.response.status_code == 409:
+                # A 409 response indicates that this task is past its deadline.  It
+                # cannot be cancelled at this time, but it's also not running
+                # anymore, so we can ignore this error.
+                logger.info(
+                    'Task {} is past its deadline and cannot be cancelled.'.format(task_id))
+                return
+            raise
 
     own_task_id = os.environ.get('TASK_ID', '')
     to_cancel = [t for t in list_task_group_incomplete_tasks(task_group_id) if t != own_task_id]