From 1b90c899f3f4d62d4c9e04652310d951eca67af4 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Fri, 14 Oct 2022 14:19:15 +0000 Subject: [PATCH] Bug 1759030 - [gecko_taskgraph] Move 'always_target' implementation outside of generator.py, r=releng-reviewers,gbrown The `always_target` attribute is a feature which I believe I previously mis-implemented. It doesn't belong in `generator.py` and should instead be implemented somewhere in `target_tasks.py`. I chose to wrap the registration decorator to guarantee backwards compatibility within Gecko. Though we may wish to move it out to a filter in the future. I'm making this change now to facilitate merging standalone and Gecko taskgraphs. The `always_target` feature will be removed from standalone (as it isn't being used and should consumers need something like it, they can roll their own implementations). With the feature removed from both Gecko and standalone in generator.py, this file will now be identical across both Taskgraphs and can therefore be removed from Gecko. Differential Revision: https://phabricator.services.mozilla.com/D159181 --- taskcluster/gecko_taskgraph/generator.py | 15 +- taskcluster/gecko_taskgraph/target_tasks.py | 131 +++++++++++------- .../gecko_taskgraph/test/test_generator.py | 30 ---- .../gecko_taskgraph/test/test_target_tasks.py | 16 +++ 4 files changed, 97 insertions(+), 95 deletions(-) diff --git a/taskcluster/gecko_taskgraph/generator.py b/taskcluster/gecko_taskgraph/generator.py index 3ccb91ee2e4d0..8ae6ed79c7e44 100644 --- a/taskcluster/gecko_taskgraph/generator.py +++ b/taskcluster/gecko_taskgraph/generator.py @@ -358,20 +358,7 @@ def _run(self): for t in full_task_graph.tasks.values() if t.attributes["kind"] == "docker-image" } - # include all tasks with `always_target` set - if parameters["tasks_for"] == "hg-push": - always_target_tasks = { - t.label - for t in full_task_graph.tasks.values() - if t.attributes.get("always_target") - } - else: - always_target_tasks = set() - logger.info( - "Adding %d tasks with `always_target` attribute" - % (len(always_target_tasks) - len(always_target_tasks & target_tasks)) - ) - requested_tasks = target_tasks | docker_image_tasks | always_target_tasks + requested_tasks = target_tasks | docker_image_tasks target_graph = full_task_graph.graph.transitive_closure(requested_tasks) target_task_graph = TaskGraph( {l: all_tasks[l] for l in target_graph.nodes}, target_graph diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py index 1da1a9df91dbb..30ee77411e0c5 100644 --- a/taskcluster/gecko_taskgraph/target_tasks.py +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -4,9 +4,10 @@ import copy -from datetime import datetime, timedelta +import logging import os import re +from datetime import datetime, timedelta from redo import retry from taskgraph.parameters import Parameters @@ -21,6 +22,34 @@ from gecko_taskgraph.util.platforms import platform_family from gecko_taskgraph.util.hg import find_hg_revision_push_info, get_hg_commit_message +logger = logging.getLogger(__name__) + + +def register_target_task(name, enable_always_target=True): + """Wrapper around `taskgraph._target_task` which supports the `always_target` attribute.""" + + def wrap(func): + def always_target_wrapper(full_task_graph, parameters, graph_config): + target_tasks = func(full_task_graph, parameters, graph_config) + if not enable_always_target or parameters["tasks_for"] != "hg-push": + return target_tasks + + target_tasks = set(target_tasks) + always_target_tasks = { + t.label + for t in full_task_graph.tasks.values() + if t.attributes.get("always_target") + } + logger.info( + "Adding %d tasks with `always_target` attribute" + % (len(always_target_tasks) - len(always_target_tasks & target_tasks)) + ) + return list(target_tasks | always_target_tasks) + + return _target_task(name)(always_target_wrapper) + + return wrap + # Some tasks show up in the target task set, but are possibly special cases, # uncommon tasks, or tasks running against limited hardware set that they @@ -333,7 +362,7 @@ def _try_option_syntax(full_task_graph, parameters, graph_config): return target_tasks_labels -@_target_task("try_tasks") +@register_target_task("try_tasks") def target_tasks_try(full_task_graph, parameters, graph_config): try_mode = parameters["try_mode"] if try_mode == "try_task_config": @@ -346,13 +375,13 @@ def target_tasks_try(full_task_graph, parameters, graph_config): return [] -@_target_task("try_select_tasks") +@register_target_task("try_select_tasks") def target_tasks_try_select(full_task_graph, parameters, graph_config): tasks = target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config) return [l for l in tasks if filter_by_uncommon_try_tasks(l)] -@_target_task("try_select_tasks_uncommon") +@register_target_task("try_select_tasks_uncommon") def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): from gecko_taskgraph.decision import PER_PROJECT_PARAMETERS @@ -378,7 +407,7 @@ def target_tasks_try_select_uncommon(full_task_graph, parameters, graph_config): return sorted(tasks) -@_target_task("try_auto") +@register_target_task("try_auto") def target_tasks_try_auto(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on autoland (rather than try) via the `run_on_projects` attributes. @@ -409,7 +438,7 @@ def target_tasks_try_auto(full_task_graph, parameters, graph_config): ] -@_target_task("default") +@register_target_task("default") def target_tasks_default(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on this project via the `run_on_projects` attributes.""" @@ -422,7 +451,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config): ] -@_target_task("autoland_tasks") +@register_target_task("autoland_tasks") def target_tasks_autoland(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove any tests running against shippable builds @@ -448,7 +477,7 @@ def filter(task): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("mozilla_central_tasks") +@register_target_task("mozilla_central_tasks") def target_tasks_mozilla_central(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove any tests running against regular (aka not shippable, @@ -488,7 +517,7 @@ def filter(task): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("graphics_tasks") +@register_target_task("graphics_tasks") def target_tasks_graphics(full_task_graph, parameters, graph_config): """In addition to doing the filtering by project that the 'default' filter does, also remove artifact builds because we have csets on @@ -506,7 +535,7 @@ def filter(task): return [l for l in filtered_for_project if filter(full_task_graph[l])] -@_target_task("mozilla_beta_tasks") +@register_target_task("mozilla_beta_tasks") def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, plus android CI. The candidates build process involves a pipeline @@ -519,7 +548,7 @@ def target_tasks_mozilla_beta(full_task_graph, parameters, graph_config): ] -@_target_task("mozilla_release_tasks") +@register_target_task("mozilla_release_tasks") def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, plus android CI. The candidates build process involves a pipeline @@ -532,7 +561,7 @@ def target_tasks_mozilla_release(full_task_graph, parameters, graph_config): ] -@_target_task("mozilla_esr102_tasks") +@register_target_task("mozilla_esr102_tasks") def target_tasks_mozilla_esr102(full_task_graph, parameters, graph_config): """Select the set of tasks required for a promotable beta or release build of desktop, without android CI. The candidates build process involves a pipeline @@ -556,7 +585,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("promote_desktop") +@register_target_task("promote_desktop") def target_tasks_promote_desktop(full_task_graph, parameters, graph_config): """Select the superset of tasks required to promote a beta or release build of a desktop product. This should include all non-android @@ -594,7 +623,7 @@ def is_geckoview(task, parameters): ) -@_target_task("push_desktop") +@register_target_task("push_desktop") def target_tasks_push_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required to push a build of desktop to cdns. Previous build deps will be optimized out via action task.""" @@ -620,7 +649,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("ship_desktop") +@register_target_task("ship_desktop") def target_tasks_ship_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required to ship desktop. Previous build deps will be optimized out via action task.""" @@ -662,7 +691,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("pine_tasks") +@register_target_task("pine_tasks") def target_tasks_pine(full_task_graph, parameters, graph_config): """Bug 1339179 - no mobile automation needed on pine""" @@ -683,7 +712,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("kaios_tasks") +@register_target_task("kaios_tasks") def target_tasks_kaios(full_task_graph, parameters, graph_config): """The set of tasks to run for kaios integration""" @@ -694,7 +723,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("ship_geckoview") +@register_target_task("ship_geckoview") def target_tasks_ship_geckoview(full_task_graph, parameters, graph_config): """Select the set of tasks required to ship geckoview nightly. The nightly build process involves a pipeline of builds and an upload to @@ -723,7 +752,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("general_perf_testing") +@register_target_task("general_perf_testing") def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config): """ Select tasks required for running performance tests 3 times a week. @@ -826,7 +855,7 @@ def filter(task, parameters): return filter -@_target_task("nightly_linux") +@register_target_task("nightly_linux") def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux. The nightly build process involves a pipeline of builds, signing, @@ -835,7 +864,7 @@ def target_tasks_nightly_linux(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_macosx") +@register_target_task("nightly_macosx") def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of macosx. The nightly build process involves a pipeline of builds, signing, @@ -844,7 +873,7 @@ def target_tasks_nightly_macosx(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_win32") +@register_target_task("nightly_win32") def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -853,7 +882,7 @@ def target_tasks_nightly_win32(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_win64") +@register_target_task("nightly_win64") def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -862,7 +891,7 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_win64_aarch64") +@register_target_task("nightly_win64_aarch64") def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of win32 and win64. The nightly build process involves a pipeline of builds, signing, @@ -871,7 +900,7 @@ def target_tasks_nightly_win64_aarch64(full_task_graph, parameters, graph_config return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("nightly_asan") +@register_target_task("nightly_asan") def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of asan. The nightly build process involves a pipeline of builds, signing, @@ -882,7 +911,7 @@ def target_tasks_nightly_asan(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.items() if filter(t, parameters)] -@_target_task("daily_releases") +@register_target_task("daily_releases") def target_tasks_daily_releases(full_task_graph, parameters, graph_config): """Select the set of tasks required to identify if we should release. If we determine that we should the task will communicate to ship-it to @@ -894,7 +923,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("nightly_desktop") +@register_target_task("nightly_desktop") def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build of linux, mac, windows.""" @@ -933,7 +962,7 @@ def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config): # Run Searchfox analysis once daily. -@_target_task("searchfox_index") +@register_target_task("searchfox_index") def target_tasks_searchfox(full_task_graph, parameters, graph_config): """Select tasks required for indexing Firefox for Searchfox web site each day""" return [ @@ -948,7 +977,7 @@ def target_tasks_searchfox(full_task_graph, parameters, graph_config): # Run build linux64-plain-clang-trunk/opt on mozilla-central/beta with perf tests -@_target_task("linux64_clang_trunk_perf") +@register_target_task("linux64_clang_trunk_perf") def target_tasks_build_linux64_clang_trunk_perf( full_task_graph, parameters, graph_config ): @@ -964,19 +993,19 @@ def filter(task_label): # Run Updatebot's cron job 4 times daily. -@_target_task("updatebot_cron") +@register_target_task("updatebot_cron") def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config): """Select tasks required to run Updatebot's cron job""" return ["updatebot-cron"] -@_target_task("customv8_update") +@register_target_task("customv8_update") def target_tasks_customv8_update(full_task_graph, parameters, graph_config): """Select tasks required for building latest d8/v8 version.""" return ["toolchain-linux64-custom-v8"] -@_target_task("chromium_update") +@register_target_task("chromium_update") def target_tasks_chromium_update(full_task_graph, parameters, graph_config): """Select tasks required for building latest chromium versions.""" return [ @@ -987,7 +1016,7 @@ def target_tasks_chromium_update(full_task_graph, parameters, graph_config): ] -@_target_task("file_update") +@register_target_task("file_update") def target_tasks_file_update(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform nightly in-tree file updates""" @@ -998,7 +1027,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("l10n_bump") +@register_target_task("l10n_bump") def target_tasks_l10n_bump(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform l10n bumping.""" @@ -1009,7 +1038,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("merge_automation") +@register_target_task("merge_automation") def target_tasks_merge_automation(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform repository merges.""" @@ -1020,7 +1049,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("scriptworker_canary") +@register_target_task("scriptworker_canary") def target_tasks_scriptworker_canary(full_task_graph, parameters, graph_config): """Select the set of tasks required to run scriptworker canaries.""" @@ -1031,7 +1060,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("cron_bouncer_check") +@register_target_task("cron_bouncer_check") def target_tasks_bouncer_check(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform bouncer version verification.""" @@ -1044,7 +1073,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("staging_release_builds") +@register_target_task("staging_release_builds") def target_tasks_staging_release(full_task_graph, parameters, graph_config): """ Select all builds that are part of releases. @@ -1068,7 +1097,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("release_simulation") +@register_target_task("release_simulation") def target_tasks_release_simulation(full_task_graph, parameters, graph_config): """ Select builds that would run on push on a release branch. @@ -1105,7 +1134,7 @@ def filter_out_android_on_esr(task): ] -@_target_task("codereview") +@register_target_task("codereview") def target_tasks_codereview(full_task_graph, parameters, graph_config): """Select all code review tasks needed to produce a report""" @@ -1123,13 +1152,13 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("nothing") +@register_target_task("nothing") def target_tasks_nothing(full_task_graph, parameters, graph_config): """Select nothing, for DONTBUILD pushes""" return [] -@_target_task("daily_beta_perf") +@register_target_task("daily_beta_perf") def target_tasks_daily_beta_perf(full_task_graph, parameters, graph_config): """ Select performance tests on the beta branch to be run daily @@ -1180,7 +1209,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("weekly_release_perf") +@register_target_task("weekly_release_perf") def target_tasks_weekly_release_perf(full_task_graph, parameters, graph_config): """ Select performance tests on the release branch to be run weekly @@ -1221,7 +1250,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("raptor_tp6m") +@register_target_task("raptor_tp6m") def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): """ Select tasks required for running raptor cold page-load tests on fenix and refbrow @@ -1249,7 +1278,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("backfill_all_browsertime") +@register_target_task("backfill_all_browsertime") def target_tasks_backfill_all_browsertime(full_task_graph, parameters, graph_config): """ Search for revisions that contains patches that were reviewed by perftest reviewers @@ -1320,7 +1349,7 @@ def reviewed_by_perftest(push): return [] -@_target_task("condprof") +@register_target_task("condprof") def target_tasks_condprof(full_task_graph, parameters, graph_config): """ Select tasks required for building conditioned profiles. @@ -1331,7 +1360,7 @@ def target_tasks_condprof(full_task_graph, parameters, graph_config): yield name -@_target_task("system_symbols") +@register_target_task("system_symbols") def target_tasks_system_symbols(full_task_graph, parameters, graph_config): """ Select tasks for scraping and uploading system symbols. @@ -1345,7 +1374,7 @@ def target_tasks_system_symbols(full_task_graph, parameters, graph_config): yield name -@_target_task("perftest") +@register_target_task("perftest") def target_tasks_perftest(full_task_graph, parameters, graph_config): """ Select perftest tasks we want to run daily @@ -1357,7 +1386,7 @@ def target_tasks_perftest(full_task_graph, parameters, graph_config): yield name -@_target_task("perftest-on-autoland") +@register_target_task("perftest-on-autoland") def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): """ Select perftest tasks we want to run daily @@ -1371,7 +1400,7 @@ def target_tasks_perftest_autoland(full_task_graph, parameters, graph_config): yield name -@_target_task("l10n-cross-channel") +@register_target_task("l10n-cross-channel") def target_tasks_l10n_cross_channel(full_task_graph, parameters, graph_config): """Select the set of tasks required to run l10n cross-channel.""" @@ -1381,7 +1410,7 @@ def filter(task): return [l for l, t in full_task_graph.tasks.items() if filter(t)] -@_target_task("are-we-esmified-yet") +@register_target_task("are-we-esmified-yet") def target_tasks_are_we_esmified_yet(full_task_graph, parameters, graph_config): """ select the task to track the progress of the esmification project diff --git a/taskcluster/gecko_taskgraph/test/test_generator.py b/taskcluster/gecko_taskgraph/test/test_generator.py index 35c62e2479c14..e74860e2b87f9 100644 --- a/taskcluster/gecko_taskgraph/test/test_generator.py +++ b/taskcluster/gecko_taskgraph/test/test_generator.py @@ -74,36 +74,6 @@ def test_target_task_graph(maketgg): ) -def test_always_target_tasks(maketgg): - "The target_task_graph includes tasks with 'always_target'" - tgg_args = { - "target_tasks": ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1"], - "kinds": [ - ("_fake", {"job-defaults": {"optimization": {"odd": None}}}), - ( - "_ignore", - { - "job-defaults": { - "attributes": {"always_target": True}, - "optimization": {"even": None}, - } - }, - ), - ], - "params": {"optimize_target_tasks": False}, - } - tgg = maketgg(**tgg_args) - assert sorted(tgg.target_task_set.tasks.keys()) == sorted( - ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1"] - ) - assert sorted(tgg.target_task_graph.tasks.keys()) == sorted( - ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1", "_ignore-t-2"] - ) - assert sorted(t.label for t in tgg.optimized_task_graph.tasks.values()) == sorted( - ["_fake-t-0", "_fake-t-1", "_ignore-t-0", "_ignore-t-1"] - ) - - def test_optimized_task_graph(maketgg): "The optimized task graph contains task ids" tgg = maketgg(["_fake-t-2"]) diff --git a/taskcluster/gecko_taskgraph/test/test_target_tasks.py b/taskcluster/gecko_taskgraph/test/test_target_tasks.py index 80fa380a907ce..a9edb3aeca3ac 100644 --- a/taskcluster/gecko_taskgraph/test/test_target_tasks.py +++ b/taskcluster/gecko_taskgraph/test/test_target_tasks.py @@ -41,6 +41,7 @@ def default_matches_project(self, run_on_projects, project): parameters={ "project": project, "hg_branch": "default", + "tasks_for": "hg-push", }, ) @@ -54,6 +55,7 @@ def default_matches_hg_branch(self, run_on_hg_branches, hg_branch): parameters={ "project": "mozilla-central", "hg_branch": hg_branch, + "tasks_for": "hg-push", }, ) @@ -154,6 +156,7 @@ def test_empty_try(self): "try_mode": None, "project": "try", "message": "", + "tasks_for": "hg-push", } # only runs the task with run_on_projects: try self.assertEqual(method(tg, params, {}), []) @@ -166,6 +169,7 @@ def test_try_option_syntax(self): params = { "try_mode": "try_option_syntax", "message": "try: -p all", + "tasks_for": "hg-push", } self.assertEqual(method(tg, params, {}), ["b"]) @@ -176,9 +180,21 @@ def test_try_task_config(self): params = { "try_mode": "try_task_config", "try_task_config": {"tasks": ["a"]}, + "tasks_for": "hg-push", } self.assertEqual(method(tg, params, {}), ["a"]) + def test_always_target(self): + "The target_task_graph includes tasks with 'always_target'" + tg = self.make_task_graph() + tg.tasks["a"].attributes["always_target"] = True + method = target_tasks.get_method("nothing") + params = {"tasks_for": "hg-push"} + self.assertEqual(method(tg, params, {}), ["a"]) + + params = {"tasks_for": "release"} + self.assertEqual(method(tg, params, {}), []) + # tests for specific filters