Skip to content

Commit

Permalink
Simplify CI configs. (pantsbuild#11744)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsirois
jsirois authored Mar 19, 2021
1 parent d46197d commit 306d185
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 78 deletions.
39 changes: 10 additions & 29 deletions .github/workflows/test-cron.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
jobs:
bootstrap_pants_linux:
env:
rust_version: 1.49.0
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Bootstrap Pants, test+lint Rust (Linux)
runs-on: ubuntu-20.04
steps:
Expand Down Expand Up @@ -40,7 +40,7 @@ jobs:
uses: actions/cache@v2
with:
key: ${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}
path: '~/.rustup/toolchains/${{ env.rust_version }}-*
path: '~/.rustup/toolchains/1.49.0-*
~/.rustup/update-hashes
Expand Down Expand Up @@ -89,11 +89,6 @@ jobs:
src/python/pants/engine/internals/native_engine.so.metadata
'
- name: Set env vars
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Bootstrap Pants
run: './pants --version
Expand Down Expand Up @@ -136,7 +131,7 @@ jobs:
- 3.8.3
bootstrap_pants_macos:
env:
rust_version: 1.49.0
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Bootstrap Pants, test Rust (MacOS)
runs-on: macos-10.15
steps:
Expand Down Expand Up @@ -170,7 +165,7 @@ jobs:
uses: actions/cache@v2
with:
key: ${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}
path: '~/.rustup/toolchains/${{ env.rust_version }}-*
path: '~/.rustup/toolchains/1.49.0-*
~/.rustup/update-hashes
Expand Down Expand Up @@ -219,11 +214,6 @@ jobs:
src/python/pants/engine/internals/native_engine.so.metadata
'
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Bootstrap Pants
run: './pants --version
Expand Down Expand Up @@ -256,6 +246,8 @@ jobs:
python-version:
- 3.8.3
lint_python_linux:
env:
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Lint Python (Linux)
needs: bootstrap_pants_linux
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -301,11 +293,6 @@ jobs:
with:
name: native_engine.so.${{ matrix.python-version }}.${{ runner.os }}
path: src/python/pants/engine/internals/
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Lint
run: './pants validate ''**''
Expand All @@ -317,6 +304,8 @@ jobs:
python-version:
- 3.8.3
test_python_linux:
env:
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Test Python (Linux)
needs: bootstrap_pants_linux
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -364,11 +353,6 @@ jobs:
with:
name: native_engine.so.${{ matrix.python-version }}.${{ runner.os }}
path: src/python/pants/engine/internals/
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Run Python tests
run: './pants test ::
Expand All @@ -378,6 +362,8 @@ jobs:
python-version:
- 3.8.3
test_python_macos:
env:
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Test Python (MacOS)
needs: bootstrap_pants_macos
runs-on: macos-10.15
Expand Down Expand Up @@ -405,11 +391,6 @@ jobs:
with:
name: native_engine.so.${{ matrix.python-version }}.${{ runner.os }}
path: src/python/pants/engine/internals/
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Run Python tests
run: './pants --tag=+platform_specific_behavior test ::
Expand Down
39 changes: 10 additions & 29 deletions .github/workflows/test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
jobs:
bootstrap_pants_linux:
env:
rust_version: 1.49.0
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Bootstrap Pants, test+lint Rust (Linux)
runs-on: ubuntu-20.04
steps:
Expand Down Expand Up @@ -40,7 +40,7 @@ jobs:
uses: actions/cache@v2
with:
key: ${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}
path: '~/.rustup/toolchains/${{ env.rust_version }}-*
path: '~/.rustup/toolchains/1.49.0-*
~/.rustup/update-hashes
Expand Down Expand Up @@ -89,11 +89,6 @@ jobs:
src/python/pants/engine/internals/native_engine.so.metadata
'
- name: Set env vars
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Bootstrap Pants
run: './pants --version
Expand Down Expand Up @@ -136,7 +131,7 @@ jobs:
- 3.7.10
bootstrap_pants_macos:
env:
rust_version: 1.49.0
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Bootstrap Pants, test Rust (MacOS)
runs-on: macos-10.15
steps:
Expand Down Expand Up @@ -170,7 +165,7 @@ jobs:
uses: actions/cache@v2
with:
key: ${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}
path: '~/.rustup/toolchains/${{ env.rust_version }}-*
path: '~/.rustup/toolchains/1.49.0-*
~/.rustup/update-hashes
Expand Down Expand Up @@ -219,11 +214,6 @@ jobs:
src/python/pants/engine/internals/native_engine.so.metadata
'
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Bootstrap Pants
run: './pants --version
Expand Down Expand Up @@ -256,6 +246,8 @@ jobs:
python-version:
- 3.7.10
lint_python_linux:
env:
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Lint Python (Linux)
needs: bootstrap_pants_linux
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -301,11 +293,6 @@ jobs:
with:
name: native_engine.so.${{ matrix.python-version }}.${{ runner.os }}
path: src/python/pants/engine/internals/
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Lint
run: './pants validate ''**''
Expand All @@ -317,6 +304,8 @@ jobs:
python-version:
- 3.7.10
test_python_linux:
env:
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Test Python (Linux)
needs: bootstrap_pants_linux
runs-on: ubuntu-20.04
Expand Down Expand Up @@ -364,11 +353,6 @@ jobs:
with:
name: native_engine.so.${{ matrix.python-version }}.${{ runner.os }}
path: src/python/pants/engine/internals/
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Run Python tests
run: './pants test ::
Expand All @@ -378,6 +362,8 @@ jobs:
python-version:
- 3.7.10
test_python_macos:
env:
PANTS_CONFIG_FILES: +['pants.ci.toml', 'pants.remote-cache.toml']
name: Test Python (MacOS)
needs: bootstrap_pants_macos
runs-on: macos-10.15
Expand Down Expand Up @@ -405,11 +391,6 @@ jobs:
with:
name: native_engine.so.${{ matrix.python-version }}.${{ runner.os }}
path: src/python/pants/engine/internals/
- name: Set pants config files
run: 'echo ''PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{
github.workspace }}/pants.remote-cache.toml"]'' >> ${GITHUB_ENV}
'
- name: Run Python tests
run: './pants --tag=+platform_specific_behavior test ::
Expand Down
48 changes: 28 additions & 20 deletions build-support/bin/generate_github_workflows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
from __future__ import annotations

import argparse
import os
from pathlib import Path
from textwrap import dedent
from typing import Any, Dict, Sequence
from typing import Any, Dict, Sequence, cast

import toml
import yaml
from common import die

Expand All @@ -22,6 +24,7 @@

Step = Dict[str, Any]
Jobs = Dict[str, Any]
Env = Dict[str, str]


def checkout() -> Sequence[Step]:
Expand Down Expand Up @@ -64,13 +67,14 @@ def pants_virtualenv_cache() -> Sequence[Step]:
]


def pants_config_files() -> Sequence[Step]:
return [
{
"name": "Set pants config files",
"run": 'echo \'PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{ github.workspace }}/pants.remote-cache.toml"]\' >> ${GITHUB_ENV}\n',
},
]
def pants_config_files() -> Env:
return {"PANTS_CONFIG_FILES": "+['pants.ci.toml', 'pants.remote-cache.toml']"}


def rust_channel() -> str:
with open("rust-toolchain") as fp:
rust_toolchain = toml.load(fp)
return cast(str, rust_toolchain["toolchain"]["channel"])


def bootstrap_caches() -> Sequence[Step]:
Expand All @@ -79,7 +83,7 @@ def bootstrap_caches() -> Sequence[Step]:
"name": "Cache Rust toolchain",
"uses": "actions/cache@v2",
"with": {
"path": "~/.rustup/toolchains/${{ env.rust_version }}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n",
"path": f"~/.rustup/toolchains/{rust_channel()}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n",
"key": "${{ runner.os }}-rustup-${{ hashFiles('rust-toolchain') }}",
},
},
Expand Down Expand Up @@ -161,15 +165,11 @@ def test_workflow_jobs(python_versions: Sequence[str]) -> Jobs:
"name": "Bootstrap Pants, test+lint Rust (Linux)",
"runs-on": "ubuntu-20.04",
"strategy": {"matrix": {"python-version": python_versions}},
"env": {"rust_version": "1.49.0"},
"env": {**pants_config_files()},
"steps": [
*checkout(),
*setup_primary_python(),
*bootstrap_caches(),
{
"name": "Set env vars",
"run": 'echo \'PANTS_CONFIG_FILES=+["${{ github.workspace }}/pants.ci.toml", "${{ github.workspace }}/pants.remote-cache.toml"]\' >> ${GITHUB_ENV}\n',
},
{"name": "Bootstrap Pants", "run": "./pants --version\n"},
{
"name": "Run smoke tests",
Expand All @@ -188,13 +188,13 @@ def test_workflow_jobs(python_versions: Sequence[str]) -> Jobs:
"runs-on": "ubuntu-20.04",
"needs": "bootstrap_pants_linux",
"strategy": {"matrix": {"python-version": python_versions}},
"env": {**pants_config_files()},
"steps": [
*checkout(),
*setup_primary_python(),
*expose_all_pythons(),
*pants_virtualenv_cache(),
*native_engine_so_download(),
*pants_config_files(),
{"name": "Run Python tests", "run": "./pants test ::\n"},
],
},
Expand All @@ -203,12 +203,12 @@ def test_workflow_jobs(python_versions: Sequence[str]) -> Jobs:
"runs-on": "ubuntu-20.04",
"needs": "bootstrap_pants_linux",
"strategy": {"matrix": {"python-version": python_versions}},
"env": {**pants_config_files()},
"steps": [
*checkout(),
*setup_primary_python(),
*pants_virtualenv_cache(),
*native_engine_so_download(),
*pants_config_files(),
{
"name": "Lint",
"run": "./pants validate '**'\n./pants lint typecheck ::\n",
Expand All @@ -219,12 +219,11 @@ def test_workflow_jobs(python_versions: Sequence[str]) -> Jobs:
"name": "Bootstrap Pants, test Rust (MacOS)",
"runs-on": "macos-10.15",
"strategy": {"matrix": {"python-version": python_versions}},
"env": {"rust_version": "1.49.0"},
"env": {**pants_config_files()},
"steps": [
*checkout(),
*setup_primary_python(),
*bootstrap_caches(),
*pants_config_files(),
{"name": "Bootstrap Pants", "run": "./pants --version\n"},
*native_engine_so_upload(),
{
Expand All @@ -240,13 +239,13 @@ def test_workflow_jobs(python_versions: Sequence[str]) -> Jobs:
"runs-on": "macos-10.15",
"needs": "bootstrap_pants_macos",
"strategy": {"matrix": {"python-version": python_versions}},
"env": {**pants_config_files()},
"steps": [
{"name": "Check out code", "uses": "actions/checkout@v2"},
*setup_primary_python(),
*expose_all_pythons(),
*pants_virtualenv_cache(),
*native_engine_so_download(),
*pants_config_files(),
{
"name": "Run Python tests",
"run": "./pants --tag=+platform_specific_behavior test ::\n",
Expand Down Expand Up @@ -357,7 +356,16 @@ def main() -> None:
if args.check:
for path, content in generated_yaml.items():
if path.read_text() != content:
die(f"Error: Generated path mismatched: {path}")
die(
dedent(
f"""\
Error: Generated path mismatched: {path}
To re-generate, run: `./pants run build-support/bin/{
os.path.basename(__file__)
}`
"""
)
)
else:
for path, content in generated_yaml.items():
path.write_text(content)
Expand Down

0 comments on commit 306d185

Please sign in to comment.