Use S3 Sync for CI deploys to avoid recopying files (pantsbuild#7895)

We currently copy all of `dist/deploy` into S3, even if the file is an exact copy of something already in S3. This poses a real cost, both monetary and time.

Instead, here we use `aws s3 sync` to only transfer files that are different from what is currently in the bucket.

Will close pantsbuild#7258.

.travis.yml

@@ -31,6 +31,11 @@ env:
     - PYENV_ROOT="${PYENV_ROOT:-${HOME}/.pants_pyenv}"
     - PATH="${PYENV_ROOT}/shims:${PATH}"
     - AWS_CLI_ROOT="${HOME}/.aws_cli"
+    - AWS_ACCESS_KEY_ID=AKIAV6A6G7RQWPRUWIXR
+    # This stores the encrypted AWS secret access key with the env var AWS_SECRET_ACCESS_KEY.
+    # Travis converts it back into its original decrypted value when ran in CI, per
+    # https://docs.travis-ci.com/user/environment-variables#defining-encrypted-variables-in-travisyml.
+    - secure: hFVAQGLVkexzTd3f9NF+JoG1dE+CPICKqOcdvQYv8+YB2rwwqu0/J6MnqKUZSmec4AM4ZvyPUBIHnSw8aMJysYs+GZ6iG/8ZRRmdbmo2WBPbSZ+ThRZxx/F6AjmovUmf8Zt366ZAZXpc9NHKREkTUGl6UL7FFe9+ouVnb90asdw=
 
 # Stages are documented here: https://docs.travis-ci.com/user/build-stages
 stages:
@@ -1909,17 +1914,10 @@ matrix:
     - <<: *py36_deploy_unstable_multiplatform_pex
 
 deploy:
-  # Deploy whatever a previous stage has left in dist/deploy.
-  # See: https://docs.travis-ci.com/user/deployment/s3/
-  provider: s3
-  access_key_id: AKIAIWOKBXVU3JLY6EGQ
-  secret_access_key:
-    secure: "UBVbpdYJ81OsDGKlPRBw6FlPJGlxosnFQ4A1xBbU5GwEBfv90GoKc6J0UwF+I4CDwytj/BlAks1XbW0zYX0oeIlXDnl1Vfikm1k4hfIr6VCLHKppiU69FlEs+ph0Dktz8+aUWhrvJzICZs6Gu08kTBQ5++3ulDWDeTHqjr713YM="
-  bucket: binaries.pantsbuild.org
-  local_dir: dist/deploy
+  provider: script
+  script: ./build-support/bin/deploy_to_s3.py
   # Otherwise travis will stash dist/deploy and the deploy will fail.
   skip_cleanup: true
-  acl: public_read
   on:
     condition: $PREPARE_DEPLOY = 1
     # NB: We mainly want deploys for `master` commits; but we also need new binaries for stable

build-support/bin/BUILD

@@ -48,6 +48,16 @@ python_binary(
   tags = {'type_checked'},
 )
 
+python_binary(
+  name = 'deploy_to_s3',
+  sources = 'deploy_to_s3.py',
+  dependencies = [
+    ':common',
+  ],
+  compatibility = ['CPython>=3.6'],
+  tags = {'type_checked'},
+)
+
 python_binary(
   name = 'mypy',
   sources = 'mypy.py',

build-support/bin/common.py

@@ -22,7 +22,7 @@
 import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Iterator, Tuple
+from typing import Iterator, NoReturn, Tuple
 
 
 _SCRIPT_START_TIME = time.time()
@@ -34,7 +34,7 @@
 _COLOR_RESET = "\x1b[0m"
 
 
-def die(message: str) -> None:
+def die(message: str) -> NoReturn:
   raise SystemExit(f"{_COLOR_RED}{message}{_COLOR_RESET}")
 
 

build-support/bin/deploy_to_s3.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os
+import shutil
+import subprocess
+
+from common import die
+
+
+def main() -> None:
+  if shutil.which("aws") is None:
+    install_aws_cli()
+  check_authentication_setup()
+  deploy()
+
+
+def install_aws_cli() -> None:
+  subprocess.run(["./build-support/bin/install_aws_cli_for_ci.sh"], check=True)
+
+
+def check_authentication_setup() -> None:
+  access_key_id = "AWS_ACCESS_KEY_ID"
+  secret_access_key = "AWS_SECRET_ACCESS_KEY"
+  if access_key_id not in os.environ or secret_access_key not in os.environ:
+    die(f"Caller of the script must set both {access_key_id} and {secret_access_key}.")
+
+
+def deploy() -> None:
+  # NB: we use the sync command to avoid transferring files that have not changed. See
+  # https://github.com/pantsbuild/pants/issues/7258.
+  subprocess.run([
+    "aws",
+    "s3",
+    "sync",
+    # This instructs the sync command to ignore timestamps, which we must do to allow distinct
+    # shards—which may finish building their wheels at different times—to not overwrite
+    # otherwise-identical wheels.
+    "--size-only",
+    "--acl",
+    "public-read",
+    "dist/deploy",
+    "s3://binaries.pantsbuild.org"
+  ], check=True)
+
+
+if __name__ == '__main__':
+  main()

build-support/bin/install_aws_cli_for_ci.sh

@@ -14,6 +14,7 @@ source build-support/common.sh
 if [[ -z "${AWS_CLI_ROOT:+''}" ]]; then
   die "Caller of the script must set the env var AWS_CLI_ROOT."
 fi
+
 AWS_CLI_BIN="${AWS_CLI_ROOT}/bin/aws"
 
 # We first check if AWS CLI is already installed thanks to Travis's cache.
@@ -31,9 +32,11 @@ if [[ ! -x "${AWS_CLI_BIN}" ]]; then
 
 fi
 
-# Travis does not cache symlinks (https://docs.travis-ci.com/user/caching/), so we create
-# the symlink ourselves everytime to ensure `aws` is discoverable globally.
-sudo ln -s "${AWS_CLI_BIN}" /usr/local/bin/aws
+# We symlink so that `aws` is discoverable on the $PATH.
+symlink="/usr/local/bin/aws"
+if [[ ! -L "${symlink}" ]]; then
+  sudo ln -s "${AWS_CLI_BIN}" "${symlink}"
+fi
 
 # Multipart operations aren't supported for anonymous users, so we set the
 # threshold high to avoid them being used automatically by the aws cli.

build-support/travis/travis.yml.mustache

@@ -24,6 +24,11 @@ env:
     - PYENV_ROOT="${PYENV_ROOT:-${HOME}/.pants_pyenv}"
     - PATH="${PYENV_ROOT}/shims:${PATH}"
     - AWS_CLI_ROOT="${HOME}/.aws_cli"
+    - AWS_ACCESS_KEY_ID=AKIAV6A6G7RQWPRUWIXR
+    # This stores the encrypted AWS secret access key with the env var AWS_SECRET_ACCESS_KEY.
+    # Travis converts it back into its original decrypted value when ran in CI, per
+    # https://docs.travis-ci.com/user/environment-variables#defining-encrypted-variables-in-travisyml.
+    - secure: hFVAQGLVkexzTd3f9NF+JoG1dE+CPICKqOcdvQYv8+YB2rwwqu0/J6MnqKUZSmec4AM4ZvyPUBIHnSw8aMJysYs+GZ6iG/8ZRRmdbmo2WBPbSZ+ThRZxx/F6AjmovUmf8Zt366ZAZXpc9NHKREkTUGl6UL7FFe9+ouVnb90asdw=
 
 # Stages are documented here: https://docs.travis-ci.com/user/build-stages
 stages:
@@ -1022,17 +1027,10 @@ matrix:
     - <<: *py36_deploy_unstable_multiplatform_pex
 
 deploy:
-  # Deploy whatever a previous stage has left in dist/deploy.
-  # See: https://docs.travis-ci.com/user/deployment/s3/
-  provider: s3
-  access_key_id: AKIAIWOKBXVU3JLY6EGQ
-  secret_access_key:
-    secure: "UBVbpdYJ81OsDGKlPRBw6FlPJGlxosnFQ4A1xBbU5GwEBfv90GoKc6J0UwF+I4CDwytj/BlAks1XbW0zYX0oeIlXDnl1Vfikm1k4hfIr6VCLHKppiU69FlEs+ph0Dktz8+aUWhrvJzICZs6Gu08kTBQ5++3ulDWDeTHqjr713YM="
-  bucket: binaries.pantsbuild.org
-  local_dir: dist/deploy
+  provider: script
+  script: ./build-support/bin/deploy_to_s3.py
   # Otherwise travis will stash dist/deploy and the deploy will fail.
   skip_cleanup: true
-  acl: public_read
   on:
     condition: $PREPARE_DEPLOY = 1
     # NB: We mainly want deploys for `master` commits; but we also need new binaries for stable