Skip to content

Commit

Permalink
bug 1340609 - toggle nightly scopes on-demand. r=dustin a=release
Browse files Browse the repository at this point in the history
MozReview-Commit-ID: JiBpEWfGefh

--HG--
extra : rebase_source : f78b4bf97a90890d941141cd147f6ab8541ae0ef
  • Loading branch information
escapewindow committed Feb 22, 2017
1 parent 5c520e7 commit 344d405
Show file tree
Hide file tree
Showing 6 changed files with 148 additions and 6 deletions.
5 changes: 3 additions & 2 deletions taskcluster/taskgraph/transforms/balrog.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import get_balrog_server_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional

Expand Down Expand Up @@ -90,6 +91,7 @@ def make_task_description(config, jobs):
],
}]

server_scope = get_balrog_server_scope(config)
task = {
'label': label,
'description': "{} Balrog".format(
Expand All @@ -100,8 +102,7 @@ def make_task_description(config, jobs):
'implementation': 'balrog',
'upstream-artifacts': upstream_artifacts,
},
# bump this to nightly / release when applicable+permitted
'scopes': ["project:releng:balrog:nightly"],
'scopes': [server_scope],
'dependencies': {'beetmover': dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
Expand Down
5 changes: 4 additions & 1 deletion taskcluster/taskgraph/transforms/beetmover.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import get_beetmover_bucket_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional

Expand Down Expand Up @@ -201,12 +202,14 @@ def make_task_description(config, jobs):
if job.get('locale'):
attributes['locale'] = job['locale']

bucket_scope = get_beetmover_bucket_scope(config)

task = {
'label': label,
'description': "{} Beetmover".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': ["project:releng:beetmover:nightly"],
'scopes': [bucket_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
Expand Down
4 changes: 3 additions & 1 deletion taskcluster/taskgraph/transforms/beetmover_checksums.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import get_beetmover_bucket_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional

Expand Down Expand Up @@ -70,12 +71,13 @@ def make_beetmover_checksums_description(config, jobs):
treeherder['symbol'] = 'tc-BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')

bucket_scope = get_beetmover_bucket_scope(config)
task = {
'label': label,
'description': "Beetmover {} ".format(
dep_job.task["metadata"]["description"]),
'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
'scopes': ["project:releng:beetmover:nightly"],
'scopes': [bucket_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
Expand Down
4 changes: 3 additions & 1 deletion taskcluster/taskgraph/transforms/checksums_signing.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import get_signing_cert_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional

Expand Down Expand Up @@ -74,6 +75,7 @@ def make_checksums_signing_description(config, jobs):
"formats": ["gpg"]
}]

signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': "Checksum signing {} ".format(
Expand All @@ -83,7 +85,7 @@ def make_checksums_signing_description(config, jobs):
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': [
"project:releng:signing:cert:nightly-signing",
signing_cert_scope,
"project:releng:signing:format:gpg"
],
'dependencies': dependencies,
Expand Down
4 changes: 3 additions & 1 deletion taskcluster/taskgraph/transforms/signing.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import validate_schema
from taskgraph.util.scriptworker import get_signing_cert_scope
from taskgraph.transforms.task import task_description_schema
from voluptuous import Schema, Any, Required, Optional

Expand Down Expand Up @@ -105,6 +106,7 @@ def make_task_description(config, jobs):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')

signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': "{} Signing".format(
Expand All @@ -113,7 +115,7 @@ def make_task_description(config, jobs):
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': job['upstream-artifacts'],
'max-run-time': 3600},
'scopes': ["project:releng:signing:cert:nightly-signing"] + signing_format_scopes,
'scopes': [signing_cert_scope] + signing_format_scopes,
'dependencies': {job['depname']: dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
Expand Down
132 changes: 132 additions & 0 deletions taskcluster/taskgraph/util/scriptworker.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""Make scriptworker.cot.verify more user friendly by making scopes dynamic.
Scriptworker uses certain scopes to determine which sets of credentials to use.
Certain scopes are restricted by branch in chain of trust verification, and are
checked again at the script level. This file provides functions to adjust
these scopes automatically by project; this makes pushing to try, forking a
project branch, and merge day uplifts more user friendly.
In the future, we may adjust scopes by other settings as well, e.g. different
scopes for `push-to-candidates` rather than `push-to-releases`, even if both
happen on mozilla-beta and mozilla-release.
"""
from __future__ import absolute_import, print_function, unicode_literals
from copy import deepcopy
import functools


"""Map signing scope aliases to sets of projects.
Currently m-c and m-a use nightly signing; m-b and m-r use release signing.
We will need to add esr support at some point. Eventually we want to add
nuance so certain m-b and m-r tasks use dep or nightly signing, and we only
release sign when we have a signed-off set of candidate builds. This current
approach works for now, though.
This is a list of list-pairs, for ordering.
"""
SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
'all-nightly-branches', set([
'mozilla-central',
'mozilla-aurora',
])
], [
'all-release-branches', set([
'mozilla-beta',
'mozilla-release',
])
]]

"""Map the signing scope aliases to the actual scopes.
"""
SIGNING_CERT_SCOPES = {
'all-release-branches': 'project:releng:signing:cert:release-signing',
'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
'default': 'project:releng:signing:cert:dep-signing',
}

"""Map beetmover scope aliases to sets of projects.
Currently this mirrors the signing scope alias behavior.
"""
BEETMOVER_SCOPE_ALIAS_TO_PROJECT = deepcopy(SIGNING_SCOPE_ALIAS_TO_PROJECT)

"""Map the beetmover scope aliases to the actual scopes.
"""
BEETMOVER_BUCKET_SCOPES = {
'all-release-branches': 'project:releng:beetmover:bucket:release',
'all-nightly-branches': 'project:releng:beetmover:bucket:nightly',
'default': 'project:releng:beetmover:bucket:dep',
}

"""Map balrog scope aliases to sets of projects.
This is a list of list-pairs, for ordering.
"""
BALROG_SCOPE_ALIAS_TO_PROJECT = [[
'nightly', set([
'mozilla-central',
])
], [
'aurora', set([
'mozilla-aurora',
])
], [
'beta', set([
'mozilla-beta',
])
], [
'release', set([
'mozilla-release',
])
]]

"""Map the balrog scope aliases to the actual scopes.
"""
BALROG_SERVER_SCOPES = {
'nightly': 'project:releng:balrog:server:nightly',
'aurora': 'project:releng:balrog:server:aurora',
'beta': 'project:releng:balrog:server:beta',
'release': 'project:releng:balrog:server:release',
'default': 'project:releng:balrog:server:dep',
}


def get_scope_from_project(alias_to_project_map, alias_to_scope_map, config):
"""Determine the restricted scope from `config.params['project']`.
Args:
alias_to_project_map (list of lists): each list pair contains the
alias alias and the set of projects that match. This is ordered.
alias_to_scope_map (dict): the alias alias to scope
config (dict): the task config that defines the project.
Returns:
string: the scope to use.
"""
for alias, projects in alias_to_project_map:
if config.params['project'] in projects and alias in alias_to_scope_map:
return alias_to_scope_map[alias]
return alias_to_scope_map['default']


get_signing_cert_scope = functools.partial(
get_scope_from_project,
SIGNING_SCOPE_ALIAS_TO_PROJECT,
SIGNING_CERT_SCOPES
)

get_beetmover_bucket_scope = functools.partial(
get_scope_from_project,
BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
BEETMOVER_BUCKET_SCOPES
)

get_balrog_server_scope = functools.partial(
get_scope_from_project,
BALROG_SCOPE_ALIAS_TO_PROJECT,
BALROG_SERVER_SCOPES
)

0 comments on commit 344d405

Please sign in to comment.