From 1deb8dd0e580d84d2c74e581103a6104e46e9deb Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Thu, 20 Oct 2016 12:56:43 -0700 Subject: [PATCH] Bug 1311791 - Explicitly declare path to shared hg store; r=dustin 332a08725ed0 changed the store path logic in a quick and crude manner. The code could lead to multiple shared stores existing on a cache if checkouts were in different parent directories. This commit refactors the code to explicitly declare a path to the shared hg store. This restores the behavior of ensuring there is only a single shared store per cache. MozReview-Commit-ID: 19Aa1QVrVQ8 --HG-- extra : rebase_source : f74c0ab9cba2bf18e335f24c86dd095335961723 --- .taskcluster.yml | 4 ++-- taskcluster/scripts/tester/run-wizard | 6 +---- taskcluster/taskgraph/action.yml | 3 ++- .../taskgraph/transforms/job/common.py | 1 + testing/docker/decision/VERSION | 2 +- testing/docker/recipes/run-task | 24 ++++++++++--------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 4db808caf8701..bce579bf01b73 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -75,9 +75,9 @@ tasks: GECKO_HEAD_REPOSITORY: '{{{url}}}' GECKO_HEAD_REF: '{{revision}}' GECKO_HEAD_REV: '{{revision}}' + HG_STORE_PATH: /home/worker/checkouts/hg-store cache: - level-{{level}}-hg-shared: /home/worker/hg-shared level-{{level}}-checkouts: /home/worker/checkouts features: @@ -86,7 +86,7 @@ tasks: # Note: This task is built server side without the context or tooling that # exist in tree so we must hard code the version - image: 'taskcluster/decision:0.1.6' + image: 'taskcluster/decision:0.1.7' maxRunTime: 1800 diff --git a/taskcluster/scripts/tester/run-wizard b/taskcluster/scripts/tester/run-wizard index 29332512cd539..5dafb0b62d609 100755 --- a/taskcluster/scripts/tester/run-wizard +++ b/taskcluster/scripts/tester/run-wizard @@ -93,14 +93,10 @@ def clone(): print('revision is not specified for checkout') return 1 - # Put the shared stores as a sibling of the checkout directory. This ensures - # they are on the same cache. - share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared')) - # TODO Bug 1301382 - pin hg.mozilla.org fingerprint. call([ b'/usr/bin/hg', b'robustcheckout', - b'--sharebase', share_base + b'--sharebase', os.environ['HG_STORE_PATH'], b'--purge', b'--upstream', b'https://hg.mozilla.org/mozilla-unified', revision_flag, revision, diff --git a/taskcluster/taskgraph/action.yml b/taskcluster/taskgraph/action.yml index 288b122e2a0d8..c816f4d5c83fd 100644 --- a/taskcluster/taskgraph/action.yml +++ b/taskcluster/taskgraph/action.yml @@ -32,6 +32,7 @@ payload: GECKO_HEAD_REPOSITORY: '{{{head_repository}}}' GECKO_HEAD_REF: '{{head_ref}}' GECKO_HEAD_REV: '{{head_rev}}' + HG_STORE_PATH: /home/worker/checkouts/hg-store cache: level-{{level}}-checkouts: /home/worker/checkouts @@ -41,7 +42,7 @@ payload: # Note: This task is built server side without the context or tooling that # exist in tree so we must hard code the version - image: 'taskcluster/decision:0.1.6' + image: 'taskcluster/decision:0.1.7' # Virtually no network or other potentially risky operations happen as part # of the task timeout aside from the initial clone. We intentionally have diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py index 8c0789a659b9e..6fac440d5ffa2 100644 --- a/taskcluster/taskgraph/transforms/job/common.py +++ b/taskcluster/taskgraph/transforms/job/common.py @@ -75,6 +75,7 @@ def docker_worker_support_vcs_checkout(config, job, taskdesc): 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], + 'HG_STORE_PATH': '/home/worker/checkouts/hg-store', }) # Give task access to hgfingerprint secret so it can pin the certificate diff --git a/testing/docker/decision/VERSION b/testing/docker/decision/VERSION index c946ee6160c20..11808190d4b90 100644 --- a/testing/docker/decision/VERSION +++ b/testing/docker/decision/VERSION @@ -1 +1 @@ -0.1.6 +0.1.7 diff --git a/testing/docker/recipes/run-task b/testing/docker/recipes/run-task index 37f7ec0f064bb..a5dd75f38edd0 100755 --- a/testing/docker/recipes/run-task +++ b/testing/docker/recipes/run-task @@ -79,7 +79,8 @@ def run_and_prefix_output(prefix, args, extra_env=None): return p.wait() -def vcs_checkout(source_repo, dest, base_repo=None, revision=None, branch=None): +def vcs_checkout(source_repo, dest, store_path, + base_repo=None, revision=None, branch=None): # Specify method to checkout a revision. This defaults to revisions as # SHA-1 strings, but also supports symbolic revisions like `tip` via the # branch flag. @@ -113,15 +114,11 @@ def vcs_checkout(source_repo, dest, base_repo=None, revision=None, branch=None): hgmo_fingerprint = secret['secret']['fingerprints'].encode('ascii') - # Put the shared stores as a sibling of the checkout directory. This ensures - # they are on the same cache. - share_base = os.path.normpath(os.path.join(os.path.dirname(dest), b'hg-shared')) - args = [ b'/usr/bin/hg', b'--config', b'hostsecurity.hg.mozilla.org:fingerprints=%s' % hgmo_fingerprint, b'robustcheckout', - b'--sharebase', share_base, + b'--sharebase', store_path, b'--purge', ] @@ -268,16 +265,19 @@ def main(args): # And that it is owned by the appropriate user/group. os.chown(os.path.dirname(checkout), uid, gid) - # And ensure the shared store path (derived from the checkout path) - # exists and has proper permissions. - share_path = os.path.join(os.path.dirname(checkout), 'hg-shared') + # And ensure the shared store path exists and has proper permissions. + if 'HG_STORE_PATH' not in os.environ: + print('error: HG_STORE_PATH environment variable not set') + sys.exit(1) + + store_path = os.environ['HG_STORE_PATH'] try: - os.makedirs(share_path) + os.makedirs(store_path) except OSError as e: if e.errno != errno.EEXIST: raise - os.chown(share_path, uid, gid) + os.chown(store_path, uid, gid) prepare_checkout_dir(args.vcs_checkout) prepare_checkout_dir(args.tools_checkout) @@ -307,6 +307,7 @@ def main(args): os.environ['GECKO_HEAD_REV'] = vcs_checkout( os.environ['GECKO_HEAD_REPOSITORY'], args.vcs_checkout, + os.environ['HG_STORE_PATH'], base_repo=base_repo, revision=os.environ.get('GECKO_HEAD_REV'), branch=os.environ.get('GECKO_HEAD_REF')) @@ -319,6 +320,7 @@ def main(args): if args.tools_checkout: vcs_checkout(b'https://hg.mozilla.org/build/tools', args.tools_checkout, + os.environ['HG_STORE_PATH'], # Always check out the latest commit on default branch. # This is non-deterministic! branch=b'default')