Bug 1572102 - deprioritize partner-repack tasks as they don't block Q…

…E, r=aki

Lower the priority of partner-repack jobs and downstreams to medium, so that tasks which do block QE work are not impeded.

Differential Revision: https://phabricator.services.mozilla.com/D43399

--HG--
extra : moz-landing-system : lando

taskcluster/ci/release-partner-repack-beetmover/kind.yml

@@ -5,6 +5,7 @@
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
+    - taskgraph.transforms.chunk_partners:transforms
     - taskgraph.transforms.name_sanity:transforms
     - taskgraph.transforms.beetmover_repackage_partner:transforms
     - taskgraph.transforms.task:transforms

taskcluster/ci/release-partner-repack-repackage-signing/kind.yml

@@ -5,6 +5,7 @@
 loader: taskgraph.loader.single_dep:loader
 
 transforms:
+    - taskgraph.transforms.chunk_partners:transforms
     - taskgraph.transforms.name_sanity:transforms
     - taskgraph.transforms.repackage_signing_partner:transforms
     - taskgraph.transforms.task:transforms

taskcluster/taskgraph/transforms/beetmover_repackage_partner.py

@@ -49,6 +49,7 @@
     Optional('extra'): object,
     Required('shipping-phase'): task_description_schema['shipping-phase'],
     Optional('shipping-product'): task_description_schema['shipping-product'],
+    Optional('priority'): task_description_schema['priority'],
 })
 
 transforms = TransformSequence()
@@ -123,6 +124,10 @@ def make_task_description(config, jobs):
                 'repack_id': repack_id,
             },
         }
+        # we may have reduced the priority for partner jobs, otherwise task.py will set it
+        if job.get('priority'):
+            task['priority'] = job['priority']
+
         yield task
 
 

taskcluster/taskgraph/transforms/chunk_partners.py

@@ -11,9 +11,14 @@
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.partners import get_partner_config_by_kind, locales_per_build_platform
+from taskgraph.util.partners import (
+    get_partner_config_by_kind,
+    locales_per_build_platform,
+    apply_partner_priority,
+)
 
 transforms = TransformSequence()
+transforms.add(apply_partner_priority)
 
 
 used_repack_ids_by_platform = {}

taskcluster/taskgraph/transforms/job/__init__.py

@@ -67,6 +67,7 @@
     Exclusive('optimization', 'optimization'): task_description_schema['optimization'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
     Optional('release-artifacts'): task_description_schema['release-artifacts'],
+    Optional('priority'): task_description_schema['priority'],
 
     # The "when" section contains descriptions of the circumstances under which
     # this task should be included in the task graph.  This will be converted

taskcluster/taskgraph/transforms/partner_repack.py

@@ -13,10 +13,12 @@
 from taskgraph.util.partners import (
     check_if_partners_enabled,
     get_partner_url_config,
+    apply_partner_priority,
 )
 
 
 transforms = TransformSequence()
+transforms.add(apply_partner_priority)
 
 
 @transforms.add

taskcluster/taskgraph/transforms/repackage_partner.py

@@ -66,7 +66,10 @@ def _by_platform(arg):
         # if true, perform a checkout of a comm-central based branch inside the
         # gecko checkout
         Required('comm-checkout', default=False): bool,
-    }
+    },
+
+    # Override the default priority for the project
+    Optional('priority'): task_description_schema['priority'],
 })
 
 transforms = TransformSequence()
@@ -213,6 +216,9 @@ def make_job_description(config, jobs):
                                                  repack_stub_installer=repack_stub_installer),
         }
 
+        # we may have reduced the priority for partner jobs, otherwise task.py will set it
+        if job.get('priority'):
+            task['priority'] = job['priority']
         if build_platform.startswith('macosx'):
             task.setdefault('fetches', {}).setdefault('toolchain', []).extend([
                 'linux64-libdmg',

taskcluster/taskgraph/transforms/repackage_signing_partner.py

@@ -27,6 +27,7 @@
     Optional('extra'): object,
     Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
+    Optional('priority'): task_description_schema['priority'],
 })
 
 transforms.add(check_if_partners_enabled)
@@ -131,5 +132,8 @@ def make_repackage_signing_description(config, jobs):
                 'repack_id': repack_id,
             }
         }
+        # we may have reduced the priority for partner jobs, otherwise task.py will set it
+        if job.get('priority'):
+            task['priority'] = job['priority']
 
         yield task

taskcluster/taskgraph/transforms/signing.py

@@ -65,6 +65,9 @@
 
     # Max number of partner repacks per chunk
     Optional('repacks-per-chunk'): int,
+
+    # Override the default priority for the project
+    Optional('priority'): task_description_schema['priority'],
 })
 
 
@@ -196,6 +199,9 @@ def make_task_description(config, jobs):
             task['treeherder'] = treeherder
         if job.get('extra'):
             task['extra'] = job['extra']
+        # we may have reduced the priority for partner jobs, otherwise task.py will set it
+        if job.get('priority'):
+            task['priority'] = job['priority']
 
         yield task
 

taskcluster/taskgraph/transforms/task.py

@@ -236,7 +236,10 @@ def _compute_geckoview_version(app_version, moz_build_date):
     Optional('worker'): {
         Required('implementation'): basestring,
         Extra: object,
-    }
+    },
+
+    # Override the default priority for the project
+    Optional('priority'): basestring,
 })
 
 TC_TREEHERDER_SCHEMA_URL = 'https://github.com/taskcluster/taskcluster-treeherder/' \

taskcluster/taskgraph/util/partners.py

@@ -415,3 +415,20 @@ def get_partners_to_be_published(config):
             if sub_config.get("publish_to_releases"):
                 partners.append((partner, sub_config_name, sub_config['platforms']))
     return partners
+
+
+def apply_partner_priority(config, jobs):
+    priority = None
+    # Reduce the priority of the partner repack jobs because they don't block QE. Meanwhile
+    # leave EME-free jobs alone because they do, and they'll get the branch priority like the rest
+    # of the release. Only bother with this in production, not on staging releases on try.
+    # medium is the same as mozilla-central, see taskcluster/ci/config.yml. ie higher than
+    # integration branches because we don't want to wait a lot for the graph to be done, but
+    # for multiple releases the partner tasks always wait for non-partner.
+    if (config.kind.startswith('release-partner-repack') and
+            config.params.release_level() == "production"):
+        priority = 'medium'
+    for job in jobs:
+        if priority:
+            job['priority'] = priority
+        yield job