diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3d260c5424c..4de19450a4e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,24 +4,286 @@ jobs: - publish-tag-to-commit-mapping: + build_wheels_linux_arm64: + container: + image: ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf + env: + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' if: github.repository_owner == 'pantsbuild' + name: Build wheels (Linux-ARM64) + runs-on: + - self-hosted + - Linux + - ARM64 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Configure Git + run: git config --global safe.directory "$GITHUB_WORKSPACE" + - name: Install rustup + run: 'curl --proto ''=https'' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- + -v -y --default-toolchain none + + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + + ' + - name: Expose Pythons + run: 'echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH + + echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH + + echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH + + ' + - if: github.event_name != 'pull_request' + name: Setup toolchain auth + run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV + + ' + - env: + PANTS_CONFIG_FILES: +['pants.ci.toml','pants.ci.aarch64.toml'] + name: Build wheels + run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex + + + USE_PY39=true ./build-support/bin/release.sh build-wheels + + ./build-support/bin/release.sh build-wheels + + USE_PY38=true ./build-support/bin/release.sh build-wheels' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-wheels-Linux-ARM64 + path: .pants.d/pants.log + - name + - run + - env + timeout-minutes: 90 + build_wheels_linux_x86_64: + container: + image: quay.io/pypa/manylinux2014_x86_64:latest + env: + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + if: github.repository_owner == 'pantsbuild' + name: Build wheels (Linux-x86_64) + runs-on: + - ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + - name: Configure Git + run: git config --global safe.directory "$GITHUB_WORKSPACE" + - name: Install rustup + run: 'curl --proto ''=https'' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- + -v -y --default-toolchain none + + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + + ' + - name: Expose Pythons + run: 'echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH + + echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH + + echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH + + ' + - if: github.event_name != 'pull_request' + name: Setup toolchain auth + run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV + + ' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - env: {} + name: Build wheels + run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex + + + USE_PY39=true ./build-support/bin/release.sh build-wheels + + ./build-support/bin/release.sh build-wheels + + USE_PY38=true ./build-support/bin/release.sh build-wheels' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-wheels-Linux-x86_64 + path: .pants.d/pants.log + - name + - run + - env + timeout-minutes: 90 + build_wheels_macos10_15_x86_64: + env: + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + if: github.repository_owner == 'pantsbuild' + name: Build wheels (macOS10-15-x86_64) + runs-on: + - self-hosted + - macOS-10.15-X64 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + ref: ${{ needs.determine_ref.outputs.build-ref }} + - name: Cache Rust toolchain + uses: actions/cache@v3 + with: + key: macOS10-15-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 + path: '~/.rustup/toolchains/1.69.0-* + + ~/.rustup/update-hashes + + ~/.rustup/settings.toml + + ' + - name: Cache Cargo + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 + with: + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine + - if: github.event_name != 'pull_request' + name: Setup toolchain auth + run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV + + ' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - env: + ARCHFLAGS: -arch x86_64 + name: Build wheels + run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex + + + USE_PY39=true ./build-support/bin/release.sh build-wheels + + ./build-support/bin/release.sh build-wheels + + USE_PY38=true ./build-support/bin/release.sh build-wheels' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-wheels-macOS10-15-x86_64 + path: .pants.d/pants.log + - name + - run + - env + timeout-minutes: 90 + build_wheels_macos11_arm64: + env: + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + if: github.repository_owner == 'pantsbuild' + name: Build wheels (macOS11-ARM64) + runs-on: + - self-hosted + - macOS-11-ARM64 + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 10 + ref: ${{ needs.determine_ref.outputs.build-ref }} + - name: Cache Rust toolchain + uses: actions/cache@v3 + with: + key: macOS11-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v2 + path: '~/.rustup/toolchains/1.69.0-* + + ~/.rustup/update-hashes + + ~/.rustup/settings.toml + + ' + - name: Cache Cargo + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 + with: + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine + - if: github.event_name != 'pull_request' + name: Setup toolchain auth + run: 'echo TOOLCHAIN_AUTH_TOKEN="${{ secrets.TOOLCHAIN_AUTH_TOKEN }}" >> $GITHUB_ENV + + ' + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.5 + - env: + ARCHFLAGS: -arch arm64 + name: Build wheels + run: 'USE_PY39=true ./build-support/bin/release.sh build-local-pex + + + USE_PY39=true ./build-support/bin/release.sh build-wheels' + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@v3 + with: + name: pants-log-wheels-macOS11-ARM64 + path: .pants.d/pants.log + - name + - run + - env + timeout-minutes: 90 + determine_ref: + if: github.repository_owner == 'pantsbuild' + outputs: + build-ref: ${{ steps.determine_ref.outputs.build-ref }} + is-release: ${{ steps.determine_ref.outputs.is-release }} runs-on: ubuntu-latest steps: - env: - TAG: ${{ github.event.inputs.tag }} - id: determine-tag - name: Determine Release Tag - run: "if [[ -n \"$TAG\" ]]; then\n tag=\"$TAG\"\nelse\n tag=\"${GITHUB_REF#refs/tags/}\"\ - \nfi\nif [[ \"${tag}\" =~ ^release_.+$ ]]; then\n echo \"release-tag=${tag}\"\ - \ >> $GITHUB_OUTPUT\nelse\n echo \"::error::Release tag '${tag}' must match\ - \ 'release_.+'.\"\n exit 1\nfi\n" + REF: ${{ github.event.inputs.ref }} + id: determine_ref + name: Determine ref to build + run: "if [[ -n \"$REF\" ]]; then\n ref=\"$REF\"\nelse\n ref=\"${GITHUB_REF#refs/tags/}\"\ + \nfi\necho \"build-ref=${ref}\" >> $GITHUB_OUTPUT\nif [[ \"${ref}\" =~ ^release_.+$\ + \ ]]; then\n echo \"is-release=true\" >> $GITHUB_OUTPUT\nfi\n" + publish: + if: github.repository_owner == 'pantsbuild' && needs.determine_ref.outputs.is-release + == 'true' + needs: + - build_wheels_linux_x86_64 + - build_wheels_linux_arm64 + - build_wheels_macos10_15_x86_64 + - build_wheels_macos11_arm64 + - determine_ref + runs-on: ubuntu-latest + steps: - name: Checkout Pants at Release Tag uses: actions/checkout@v3 with: - ref: ${{ steps.determine-tag.outputs.release-tag }} + ref: ${{ needs.determine_ref.outputs.build-ref }} + - env: + MODE: debug + name: Fetch and stabilize wheels + run: ./build-support/bin/release.sh fetch-and-stabilize - name: Create Release -> Commit Mapping - run: 'tag="${{ steps.determine-tag.outputs.release-tag }}" + run: 'tag="${{ needs.determine_ref.outputs.build-ref }}" commit="$(git rev-parse ${tag}^{commit})" @@ -33,19 +295,22 @@ jobs: echo "${commit}" > "dist/deploy/tags/pantsbuild.pants/${tag}" ' + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PANTSBUILD_PYPI_API_TOKEN }} - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - name: Deploy to S3 + name: Deploy commit mapping to S3 run: ./build-support/bin/deploy_to_s3.py --scope tags/pantsbuild.pants -name: Record Release Commit +name: Release 'on': push: tags: - release_* workflow_dispatch: inputs: - tag: + ref: required: true type: string diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6ab5226b2ca..d19ba87f0e8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -324,6 +324,7 @@ jobs: container: image: ghcr.io/pantsbuild/wheel_build_aarch64:v3-8384c5cf env: + MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release @@ -379,17 +380,12 @@ jobs: with: name: pants-log-wheels-Linux-ARM64 path: .pants.d/pants.log - - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.event_name == 'push' - name: Deploy to S3 - run: ./build-support/bin/deploy_to_s3.py timeout-minutes: 90 build_wheels_linux_x86_64: container: image: quay.io/pypa/manylinux2014_x86_64:latest env: + MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release @@ -446,15 +442,10 @@ jobs: with: name: pants-log-wheels-Linux-x86_64 path: .pants.d/pants.log - - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.event_name == 'push' - name: Deploy to S3 - run: ./build-support/bin/deploy_to_s3.py timeout-minutes: 90 build_wheels_macos10_15_x86_64: env: + MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release @@ -514,15 +505,10 @@ jobs: with: name: pants-log-wheels-macOS10-15-x86_64 path: .pants.d/pants.log - - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.event_name == 'push' - name: Deploy to S3 - run: ./build-support/bin/deploy_to_s3.py timeout-minutes: 90 build_wheels_macos11_arm64: env: + MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' if: ((github.repository_owner == 'pantsbuild') && (github.event_name == 'push' || needs.classify_changes.outputs.release @@ -578,12 +564,6 @@ jobs: with: name: pants-log-wheels-macOS11-ARM64 path: .pants.d/pants.log - - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.event_name == 'push' - name: Deploy to S3 - run: ./build-support/bin/deploy_to_s3.py timeout-minutes: 90 check_labels: if: github.repository_owner == 'pantsbuild' diff --git a/build-support/bin/_release_helper.py b/build-support/bin/_release_helper.py index 15dce9edc0b..b637e5e69c2 100644 --- a/build-support/bin/_release_helper.py +++ b/build-support/bin/_release_helper.py @@ -12,7 +12,6 @@ import subprocess import sys import venv -import xmlrpc.client from configparser import ConfigParser from contextlib import contextmanager from dataclasses import dataclass @@ -21,7 +20,6 @@ from functools import total_ordering from math import ceil from pathlib import Path -from time import sleep from typing import Any, Callable, Iterable, Iterator, NamedTuple, Sequence, cast from urllib.parse import quote_plus from xml.etree import ElementTree @@ -83,57 +81,6 @@ } -class PackageAccessValidator: - @classmethod - def validate_all(cls): - instance = cls() - for pkg_name in _known_packages: - instance.validate_package_access(pkg_name) - - def __init__(self): - self._client = xmlrpc.client.ServerProxy("https://pypi.org/pypi") - - @property - def client(self): - # The PyPI XML-RPC API requires at least 1 second between requests, or it rejects them - # with HTTPTooManyRequests. - sleep(1.0) - return self._client - - @staticmethod - def validate_role_sets(role: str, actual: set[str], expected: set[str]) -> str: - err_msg = "" - if actual != expected: - expected_not_actual = sorted(expected - actual) - actual_not_expected = sorted(actual - expected) - if expected_not_actual: - err_msg += f"Missing expected {role}s: {','.join(expected_not_actual)}." - if actual_not_expected: - err_msg += f"Found unexpected {role}s: {','.join(actual_not_expected)}" - return err_msg - - def validate_package_access(self, pkg_name: str) -> None: - actual_owners = set() - actual_maintainers = set() - for role_assignment in self.client.package_roles(pkg_name): - role, username = role_assignment - if role == "Owner": - actual_owners.add(username) - elif role == "Maintainer": - actual_maintainers.add(username) - else: - raise ValueError(f"Unrecognized role {role} for user {username}") - - err_msg = "" - err_msg += self.validate_role_sets("owner", actual_owners, _expected_owners) - err_msg += self.validate_role_sets("maintainer", actual_maintainers, _expected_maintainers) - - if err_msg: - die(f"Role discrepancies for {pkg_name}: {err_msg}") - - print(f"Roles for package {pkg_name} as expected.") - - @total_ordering class PackageVersionType(Enum): DEV = 0 @@ -871,32 +818,40 @@ def build_pex(fetch: bool) -> None: # ----------------------------------------------------------------------------------------------- -# Publish +# Fetch and stabilize the versions of wheels for publishing # ----------------------------------------------------------------------------------------------- -def publish() -> None: - banner("Releasing to PyPI and GitHub") - # Check prereqs. - check_clean_git_branch() - prompt_artifact_freshness() - check_pgp() - check_roles() - +def fetch_and_stabilize() -> None: + # TODO: Because wheels are now built specifically for a particular tag, we could likely remove + # "reversioning". + banner("Fetching and stabilizing wheels.") # Fetch and validate prebuilt wheels. if CONSTANTS.deploy_pants_wheel_dir.exists(): shutil.rmtree(CONSTANTS.deploy_pants_wheel_dir) fetch_prebuilt_wheels(CONSTANTS.deploy_dir, include_3rdparty=False) check_pants_wheels_present(CONSTANTS.deploy_dir) reversion_prebuilt_wheels() + banner("Successfully fetched and stabilized wheels") - # Release. - create_twine_venv() - upload_wheels_via_twine() - tag_release() - banner("Successfully released to PyPI and GitHub") + +# ----------------------------------------------------------------------------------------------- +# Begin a release by pushing a release tag +# ----------------------------------------------------------------------------------------------- + + +def tag_release() -> None: + banner("Tagging release") + + check_clean_git_branch() + check_pgp() + + prompt_artifact_freshness() prompt_to_generate_docs() + run_tag_release() + banner("Successfully tagged release") + def check_clean_git_branch() -> None: banner("Checking for a clean Git branch") @@ -946,24 +901,6 @@ def check_pgp() -> None: ) -def check_roles() -> None: - # Check that the packages we plan to publish are correctly owned. - banner("Checking current user.") - username = get_pypi_config("server-login", "username") - if ( - username != "__token__" # See: https://pypi.org/help/#apitoken - and username not in _expected_owners - and username not in _expected_maintainers - ): - die(f"User {username} not authorized to publish.") - banner("Checking package roles.") - validator = PackageAccessValidator() - for pkg in PACKAGES: - if pkg.name not in _known_packages: - die(f"Unknown package {pkg}") - validator.validate_package_access(pkg.name) - - def reversion_prebuilt_wheels() -> None: # First, rewrite to manylinux. See https://www.python.org/dev/peps/pep-0599/. We use # manylinux2014 images. @@ -985,7 +922,7 @@ def reversion_prebuilt_wheels() -> None: ) -def tag_release() -> None: +def run_tag_release() -> None: tag_name = f"release_{CONSTANTS.pants_stable_version}" subprocess.run( [ @@ -1202,13 +1139,13 @@ def check_pants_wheels_present(check_dir: str | Path) -> None: def create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) - subparsers.add_parser("publish") + subparsers.add_parser("tag-release") + subparsers.add_parser("fetch-and-stabilize") subparsers.add_parser("test-release") subparsers.add_parser("build-wheels") subparsers.add_parser("build-fs-util") subparsers.add_parser("build-local-pex") subparsers.add_parser("build-universal-pex") - subparsers.add_parser("validate-roles") subparsers.add_parser("validate-freshness") subparsers.add_parser("list-prebuilt-wheels") subparsers.add_parser("check-pants-wheels") @@ -1217,8 +1154,10 @@ def create_parser() -> argparse.ArgumentParser: def main() -> None: args = create_parser().parse_args() - if args.command == "publish": - publish() + if args.command == "tag-release": + tag_release() + if args.command == "fetch-and-stabilize": + fetch_and_stabilize() if args.command == "test-release": test_release() if args.command == "build-wheels": @@ -1229,8 +1168,6 @@ def main() -> None: build_pex(fetch=False) if args.command == "build-universal-pex": build_pex(fetch=True) - if args.command == "validate-roles": - PackageAccessValidator.validate_all() if args.command == "validate-freshness": prompt_artifact_freshness() if args.command == "list-prebuilt-wheels": diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index 132a9a39162..bb8c474429f 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -167,8 +167,11 @@ def ensure_category_label() -> Sequence[Step]: ] -def checkout(*, fetch_depth: int = 10, containerized: bool = False) -> Sequence[Step]: +def checkout( + *, fetch_depth: int = 10, containerized: bool = False, ref: str | None = None +) -> Sequence[Step]: """Get prior commits and the commit message.""" + fetch_depth_opt: dict[str, Any] = {"fetch-depth": fetch_depth} steps = [ # See https://github.community/t/accessing-commit-message-in-pull-request-event/17158/8 # for details on how we get the commit message here. @@ -176,7 +179,10 @@ def checkout(*, fetch_depth: int = 10, containerized: bool = False) -> Sequence[ { "name": "Check out code", "uses": "actions/checkout@v3", - "with": {"fetch-depth": fetch_depth}, + "with": { + **fetch_depth_opt, + **({"ref": ref} if ref else {}), + }, }, ] if containerized: @@ -249,14 +255,17 @@ def install_go() -> Step: } -def deploy_to_s3(when: str = "github.event_name == 'push'", scope: str | None = None) -> Step: +def deploy_to_s3( + name: str, + *, + scope: str | None = None, +) -> Step: run = "./build-support/bin/deploy_to_s3.py" if scope: run = f"{run} --scope {scope}" return { - "name": "Deploy to S3", + "name": name, "run": run, - "if": when, "env": { "AWS_SECRET_ACCESS_KEY": f"{gha_expr('secrets.AWS_SECRET_ACCESS_KEY')}", "AWS_ACCESS_KEY_ID": f"{gha_expr('secrets.AWS_ACCESS_KEY_ID')}", @@ -719,7 +728,12 @@ def macos11_x86_64_test_jobs(python_versions: list[str]) -> Jobs: return jobs -def build_wheels_job(platform: Platform, python_versions: list[str]) -> Jobs: +def build_wheels_job( + platform: Platform, + python_versions: list[str], + for_deploy_ref: str | None, + needs: list[str] | None, +) -> Jobs: helper = Helper(platform) # For manylinux compatibility, we build Linux wheels in a container rather than directly # on the Ubuntu runner. As a result, we have custom steps here to check out @@ -753,7 +767,7 @@ def build_wheels_job(platform: Platform, python_versions: list[str]) -> Jobs: ] else: initial_steps = [ - *checkout(), + *checkout(ref=for_deploy_ref), *helper.expose_all_pythons(), # NB: We only cache Rust, but not `native_engine.so` and the Pants # virtualenv. This is because we must build both these things with @@ -762,35 +776,45 @@ def build_wheels_job(platform: Platform, python_versions: list[str]) -> Jobs: *helper.rust_caches(), ] + if_condition = ( + IS_PANTS_OWNER if for_deploy_ref else f"({IS_PANTS_OWNER}) && ({DONT_SKIP_WHEELS})" + ) return { helper.job_name("build_wheels"): { - "if": f"({IS_PANTS_OWNER}) && ({DONT_SKIP_WHEELS})", + "if": if_condition, "name": f"Build wheels ({str(platform.value)})", "runs-on": helper.runs_on(), **({"container": container} if container else {}), "timeout-minutes": 90, - "env": DISABLE_REMOTE_CACHE_ENV, + "env": { + **DISABLE_REMOTE_CACHE_ENV, + # If we're not deploying these wheels, build in debug mode, which allows for + # incremental compilation across wheels. If this becomes too slow in CI, most likely + # the answer will be to adjust the `opt-level` for the relevant Cargo profile rather + # than to not use debug mode. + **({} if for_deploy_ref else {"MODE": "debug"}), + }, "steps": initial_steps + [ setup_toolchain_auth(), *([] if platform == Platform.LINUX_ARM64 else [install_go()]), *helper.build_wheels(python_versions), helper.upload_log_artifacts(name="wheels"), - deploy_to_s3(), + *(deploy_to_s3("Deploy wheels to S3") if for_deploy_ref else []), ], }, } -def build_wheels_jobs() -> Jobs: +def build_wheels_jobs(*, for_deploy_ref: str | None = None, needs: list[str] | None = None) -> Jobs: # N.B.: When altering the number of total wheels built (currently 10), please edit the expected # total in the _release_helper script. Currently here: # https://github.com/pantsbuild/pants/blob/8c83e4db33d5fe577918ce073f6d89957cb6eef1/build-support/bin/_release_helper.py#L1182-L1192 return { - **build_wheels_job(Platform.LINUX_X86_64, ALL_PYTHON_VERSIONS), - **build_wheels_job(Platform.LINUX_ARM64, ALL_PYTHON_VERSIONS), - **build_wheels_job(Platform.MACOS10_15_X86_64, ALL_PYTHON_VERSIONS), - **build_wheels_job(Platform.MACOS11_ARM64, [PYTHON39_VERSION]), + **build_wheels_job(Platform.LINUX_X86_64, ALL_PYTHON_VERSIONS, for_deploy_ref, needs), + **build_wheels_job(Platform.LINUX_ARM64, ALL_PYTHON_VERSIONS, for_deploy_ref, needs), + **build_wheels_job(Platform.MACOS10_15_X86_64, ALL_PYTHON_VERSIONS, for_deploy_ref, needs), + **build_wheels_job(Platform.MACOS11_ARM64, [PYTHON39_VERSION], for_deploy_ref, needs), } @@ -934,37 +958,60 @@ def cache_comparison_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: - inputs, env = workflow_dispatch_inputs([WorkflowInput("TAG", "string")]) + """Builds and releases a git ref to S3, and (if the ref is a release tag) to PyPI.""" + inputs, env = workflow_dispatch_inputs([WorkflowInput("REF", "string")]) + wheels_jobs = build_wheels_jobs( + needs=["determine_ref"], for_deploy_ref=gha_expr("needs.determine_ref.outputs.build-ref") + ) + wheels_job_names = tuple(wheels_jobs.keys()) jobs = { - "publish-tag-to-commit-mapping": { + "determine_ref": { "runs-on": "ubuntu-latest", "if": IS_PANTS_OWNER, "steps": [ { - "name": "Determine Release Tag", - "id": "determine-tag", + "name": "Determine ref to build", "env": env, + "id": "determine_ref", "run": dedent( """\ - if [[ -n "$TAG" ]]; then - tag="$TAG" + if [[ -n "$REF" ]]; then + ref="$REF" else - tag="${GITHUB_REF#refs/tags/}" + ref="${GITHUB_REF#refs/tags/}" fi - if [[ "${tag}" =~ ^release_.+$ ]]; then - echo "release-tag=${tag}" >> $GITHUB_OUTPUT - else - echo "::error::Release tag '${tag}' must match 'release_.+'." - exit 1 + echo "build-ref=${ref}" >> $GITHUB_OUTPUT + if [[ "${ref}" =~ ^release_.+$ ]]; then + echo "is-release=true" >> $GITHUB_OUTPUT fi """ ), }, + ], + "outputs": { + "build-ref": gha_expr("steps.determine_ref.outputs.build-ref"), + "is-release": gha_expr("steps.determine_ref.outputs.is-release"), + }, + }, + **wheels_jobs, + "publish": { + "runs-on": "ubuntu-latest", + "needs": [*wheels_job_names, "determine_ref"], + "if": f"{IS_PANTS_OWNER} && needs.determine_ref.outputs.is-release == 'true'", + "steps": [ { "name": "Checkout Pants at Release Tag", "uses": "actions/checkout@v3", - "with": {"ref": f"{gha_expr('steps.determine-tag.outputs.release-tag')}"}, + "with": {"ref": f"{gha_expr('needs.determine_ref.outputs.build-ref')}"}, + }, + { + "name": "Fetch and stabilize wheels", + "run": "./build-support/bin/release.sh fetch-and-stabilize", + "env": { + # This step does not actually build anything: only download wheels from S3. + "MODE": "debug", + }, }, { "name": "Create Release -> Commit Mapping", @@ -975,7 +1022,7 @@ def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: # ${VAR} syntax to it and the ${{ github }} syntax ... this is a confusing read. "run": dedent( f"""\ - tag="{gha_expr("steps.determine-tag.outputs.release-tag")}" + tag="{gha_expr("needs.determine_ref.outputs.build-ref")}" commit="$(git rev-parse ${{tag}}^{{commit}})" echo "Recording tag ${{tag}} is of commit ${{commit}}" @@ -984,12 +1031,19 @@ def release_jobs_and_inputs() -> tuple[Jobs, dict[str, Any]]: """ ), }, + { + "name": "Publish to PyPI", + "uses": "pypa/gh-action-pypi-publish@release/v1", + "with": { + "password": gha_expr("secrets.PANTSBUILD_PYPI_API_TOKEN"), + }, + }, deploy_to_s3( - when="github.event_name == 'push' || github.event_name == 'workflow_dispatch'", + "Deploy commit mapping to S3", scope="tags/pantsbuild.pants", ), ], - } + }, } return jobs, inputs @@ -1154,7 +1208,7 @@ def generate() -> dict[Path, str]: release_jobs, release_inputs = release_jobs_and_inputs() release_yaml = yaml.dump( { - "name": "Record Release Commit", + "name": "Release", "on": { "push": {"tags": ["release_*"]}, "workflow_dispatch": {"inputs": release_inputs}, diff --git a/docs/markdown/Contributions/releases/release-process.md b/docs/markdown/Contributions/releases/release-process.md index b1e9efae472..cc458ed6950 100644 --- a/docs/markdown/Contributions/releases/release-process.md +++ b/docs/markdown/Contributions/releases/release-process.md @@ -213,15 +213,10 @@ Also, update the [Changelog](doc:changelog)'s "highlights" column with a link to > > Ping someone in the `#maintainers-confidential` channel in Slack to be added. Alternatively, you can "Suggest edits" in the top right corner. -Step 3: Wait for CI to build the wheels +Step 3: Tag the release to build wheels --------------------------------------- -Once you have merged the `VERSION` bump—which will be on `main` for `dev` and `a0` releases and the release branch for release candidates—CI will start building the wheels you need to finish the release. - -Head to and find your relevant build. You need the "Build wheels and fs_util" jobs to pass. - -Step 4: Run `release.sh` ------------------------- +Once you have merged the `VERSION` bump — which will be on `main` for `dev` and `a0` releases and the release branch for release candidates — you should tag the release commit to trigger wheel building and PyPI publishing. First, ensure that you are on your release branch at your version bump commit. @@ -232,12 +227,15 @@ First, ensure that you are on your release branch at your version bump commit. Then, run: ```bash -./build-support/bin/release.sh publish +./build-support/bin/release.sh tag-release ``` -This will first download the pre-built wheels built in CI and will publish them to PyPI. About 2-3 minutes in, the script will prompt you for your PGP password. +This will tag the release with your PGP key, and push the tag to origin, which will kick off a [`Release` job](https://github.com/pantsbuild/pants/actions/workflows/release.yaml) to build the wheels and publish them to PyPI. + +Step 4: Release a Pants PEX +--------------------------- -We also release a Pants Pex via GitHub releases. Run this: +After the [`Release` job](https://github.com/pantsbuild/pants/actions/workflows/release.yaml) for your tag has completed, you should additionally build and publish the "universal" PEX to Github. ```bash PANTS_PEX_RELEASE=STABLE ./build-support/bin/release.sh build-universal-pex diff --git a/docs/markdown/Getting Started/getting-started/installation.md b/docs/markdown/Getting Started/getting-started/installation.md index 1aed4145b43..290b0dd370e 100644 --- a/docs/markdown/Getting Started/getting-started/installation.md +++ b/docs/markdown/Getting Started/getting-started/installation.md @@ -53,6 +53,8 @@ To use an unreleased build of Pants from the [pantsbuild/pants](https://github.c PANTS_SHA=8553e8cbc5a1d9da3f84dcfc5e7bf3139847fb5f pants --version ``` +If a particular SHA does not have built wheels, you can either wait for [the next release from the relevant branch](doc:release-strategy), ping a maintainer [in Slack](doc:getting-help), or file a Github issue mentioning the SHA that you would like to test. + Running Pants from sources -------------------------- diff --git a/docs/markdown/Getting Started/getting-started/manual-installation.md b/docs/markdown/Getting Started/getting-started/manual-installation.md index cc636460e97..2d01f502a32 100644 --- a/docs/markdown/Getting Started/getting-started/manual-installation.md +++ b/docs/markdown/Getting Started/getting-started/manual-installation.md @@ -43,15 +43,6 @@ Now, run this to bootstrap Pants and to verify the version it installs: > > The `./pants` script will automatically install and use the Pants version specified in `pants.toml`, so upgrading Pants is as simple as editing `pants_version` in that file. -Running Pants from unreleased builds ------------------------------------- - -To use an unreleased build of Pants from the [pantsbuild/pants](https://github.com/pantsbuild/pants) main branch, locate the main branch SHA, set `PANTS_SHA=` in the environment, and run `./pants` as usual: - -``` -PANTS_SHA=8553e8cbc5a1d9da3f84dcfc5e7bf3139847fb5f ./pants --version -``` - Building Pants from sources ---------------------------