Skip to content

Commit

Permalink
ops: Improve check-changed (ethereum-optimism#4527)
Browse files Browse the repository at this point in the history
  • Loading branch information
mslipper authored Dec 19, 2022
1 parent 874457d commit 7f8f0fe
Show file tree
Hide file tree
Showing 4 changed files with 188 additions and 97 deletions.
146 changes: 49 additions & 97 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,28 @@ commands:
gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>"
# Configure ADC
echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV"
check-changed:
description: "Conditionally halts a step if certain modules change"
parameters:
patterns:
type: string
description: "Comma-separated list of dependencies"
steps:
- run:
name: "Check for changes"
command: |
cd ops/check-changed
pip3 install -r requirements.txt
python3 main.py "<<parameters.patterns>>"
jobs:
yarn-monorepo:
docker:
- image: ethereumoptimism/ci-builder:latest
resource_class: xlarge
steps:
- checkout
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(op-bindings|op-chain-ops|packages/)" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- check-changed:
patterns: op-bindings,op-chain-ops,packages/
- run:
name: Install dependencies
command: yarn --frozen-lockfile
Expand Down Expand Up @@ -211,15 +218,9 @@ jobs:
resource_class: large
steps:
- checkout
- attach_workspace: {at: "."}
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(contracts-bedrock|hardhat-deploy-config)" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- attach_workspace: { at: "." }
- check-changed:
patterns: contracts-bedrock,hardhat-deploy-config
- run:
name: lint
command: yarn lint:check
Expand Down Expand Up @@ -268,15 +269,9 @@ jobs:
- image: ethereumoptimism/ci-builder:latest
steps:
- checkout
- attach_workspace: {at: "."}
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(contracts-bedrock/contracts)" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- attach_workspace: { at: "." }
- check-changed:
patterns: contracts-bedrock,contracts
- run:
name: Compile with metadata hash
command: yarn clean && yarn build:with-metadata
Expand All @@ -301,15 +296,9 @@ jobs:
resource_class: <<parameters.size>>
steps:
- checkout
- attach_workspace: {at: "."}
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(contracts-bedrock/contracts)" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- attach_workspace: { at: "." }
- check-changed:
patterns: contracts-bedrock,contracts
- run:
name: Echidna Fuzz <<parameters.echidna_target>>
command: yarn echidna:<<parameters.echidna_target>>
Expand All @@ -322,15 +311,9 @@ jobs:
resource_class: medium
steps:
- checkout
- attach_workspace: {at: "."}
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(contracts-bedrock|op-bindings)" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- attach_workspace: { at: "." }
- check-changed:
patterns: contracts-bedrock,op-bindings
- run:
name: check go bindings
command: make && git diff --exit-code
Expand All @@ -353,15 +336,9 @@ jobs:
resource_class: large
steps:
- checkout
- attach_workspace: {at: "."}
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(packages/<<parameters.package_name>>|packages/<<parameters.dependencies>>)" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- attach_workspace: { at: "." }
- check-changed:
patterns: <<parameters.package_name>>,<<parameters.dependencies>>
- run:
name: Lint
command: yarn lint && git diff --exit-code
Expand All @@ -379,6 +356,8 @@ jobs:
image: ubuntu-2204:2022.07.1
steps:
- checkout
- check-changed:
patterns: specs/(.*)\.md$
- run:
name: yarn dev deps # todo: what's the best way to pull in the dependencies for linting? yarn install above is using production env without dev dependencies
command: yarn install --production=false
Expand All @@ -398,14 +377,8 @@ jobs:
- image: ethereumoptimism/ci-builder:latest
steps:
- checkout
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "op-node" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- check-changed:
patterns: op-node
- run:
name: Fuzz
command: make fuzz
Expand All @@ -416,15 +389,9 @@ jobs:
- image: ethereumoptimism/ci-builder:latest
steps:
- checkout
- attach_workspace: {at: "."}
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "packages/" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- attach_workspace: { at: "." }
- check-changed:
patterns: packages
# Note: The below needs to be manually configured whenever we
# add a new package to CI.
- run:
Expand Down Expand Up @@ -501,10 +468,12 @@ jobs:
description: If the op-e2e package should use HTTP clients
type: string
docker:
- image: cimg/go:1.19
- image: ethereumoptimism/ci-builder:latest
resource_class: xlarge
steps:
- checkout
- check-changed:
patterns: op-(.+),contracts-bedrock,contracts
- run:
name: prep results dir
command: mkdir -p /tmp/test-results
Expand Down Expand Up @@ -541,15 +510,8 @@ jobs:
- image: cimg/postgres:14.1
steps:
- checkout
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "(<<parameters.working_directory>>|<<parameters.dependencies>>)" || echo "TRUE")
echo $CHANGED
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- check-changed:
patterns: <<parameters.working_directory>>,<<parameters.dependencies>>
- run:
name: Lint
command: golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" ./...
Expand All @@ -576,14 +538,8 @@ jobs:
- image: ethereumoptimism/ci-builder:latest
steps:
- checkout
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(check-changed "l2geth" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- check-changed:
patterns: l2geth
- run:
name: Test
command: make test
Expand All @@ -602,6 +558,8 @@ jobs:
DOCKER_BUILDKIT: 1
steps:
- checkout
- check-changed:
patterns: op-(.+),packages,integration-tests
- run:
name: Install latest golang
command: |
Expand Down Expand Up @@ -681,14 +639,8 @@ jobs:
parallelism: 3
steps:
- checkout
- run:
name: Check if we should run
command: |
shopt -s inherit_errexit
CHANGED=$(bash ./ops/docker/ci-builder/check-changed.sh "(l2geth|common-ts|contracts|core-utils|message-relayer|data-transport-layer|replica-healthcheck|sdk|batch-submitter|gas-oracle|bss-core|integration-tests)/" || echo "TRUE")
if [[ "$CHANGED" = "FALSE" ]]; then
circleci step halt
fi
- check-changed:
patterns: l2geth,common-ts,contracts,core-utils,message-relayer,data-transport-layer,replica-healthcheck,sdk,batch-submitter,gas-oracle,bss-core,integration-tests
- run:
name: Bring up the stack
command: |
Expand Down Expand Up @@ -781,7 +733,7 @@ jobs:
- go/load-cache
- go/mod-download
- go/save-cache
- run: {command: "go build ."}
- run: { command: "go build ." }
- run:
command: |
./hive \
Expand Down
1 change: 1 addition & 0 deletions ops/check-changed/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
venv
126 changes: 126 additions & 0 deletions ops/check-changed/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
import logging.config
import os
import re
import subprocess
import sys

from github import Github

WHITELISTED_BRANCHES = {
'master',
'develop'
}

LOGGING_CONFIG = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'standard': {
'format': '%(asctime)s [%(levelname)s]: %(message)s'
},
},
'handlers': {
'default': {
'level': 'INFO',
'formatter': 'standard',
'class': 'logging.StreamHandler',
'stream': 'ext://sys.stderr'
},
},
'loggers': {
'': {
'handlers': ['default'],
'level': 'INFO',
'propagate': False
},
}
}

logging.config.dictConfig(LOGGING_CONFIG)
log = logging.getLogger(__name__)


def main():
patterns = sys.argv[1].split(',')
patterns.append(r'^\.circleci/\.*')

fp = os.path.realpath(__file__)
monorepo_path = os.path.realpath(os.path.join(fp, '..', '..'))

log.info('Discovered monorepo path: %s', monorepo_path)
current_branch = git_cmd('rev-parse --abbrev-ref HEAD', monorepo_path)
log.info('Current branch: %s', current_branch)

if current_branch in WHITELISTED_BRANCHES:
log.info('Current branch %s is whitelisted, triggering build', current_branch)
exit_build()

pr_urls = os.getenv('CIRCLE_PULL_REQUESTS', None)
pr_urls = pr_urls.split(',') if pr_urls else []
if len(pr_urls) == 0:
log.info('Not a PR build, skipping build')
exit_nobuild()
if len(pr_urls) > 1:
log.warning('Multiple PR URLs found, choosing the first one. PRs found:')
for url in pr_urls:
log.warning(url)

gh_token = os.getenv('GITHUB_ACCESS_TOKEN')
if gh_token is None:
log.info('No GitHub access token found - likely a fork. Triggering build')
exit_build()

g = Github(gh_token)
try:
g.get_user()
repo = g.get_repo(os.getenv('CIRCLE_PROJECT_USERNAME') + '/' + os.getenv('CIRCLE_PROJECT_REPONAME'))
except Exception:
log.exception('Failed to get repo from GitHub')
exit_build()

pr = repo.get_pull(int(pr_urls[0].split('/')[-1]))
log.info('Found PR: %s', pr.url)

base_sha = pr.base.sha
head_sha = pr.head.sha

diffs = git_cmd('diff --name-only {}...{}'.format(base_sha, head_sha), monorepo_path).split('\n')
log.info('Found diff. Checking for matches...')
for diff in diffs:
if match_path(diff, patterns):
log.info('Match found, triggering build')
exit_build()
else:
log.info('❌ no match found on %s', diff)

log.info('No matches found, skipping build')
exit_nobuild()


def git_cmd(cmd, cwd):
return subprocess.check_output(['git'] + cmd.split(' '), cwd=cwd).decode('utf-8').strip()


def match_path(path, patterns):
for pattern in patterns:
if re.search(pattern, path):
log.info('✅ match found on %s: %s', path, pattern)
return True
return False


def exit_build():
sys.exit(0)


def exit_nobuild():
subprocess.check_call(['circleci', 'step', 'halt'])
sys.exit(0)


if __name__ == '__main__':
try:
main()
except Exception:
log.exception('Unhandled exception, triggering build')
exit_build()
Loading

0 comments on commit 7f8f0fe

Please sign in to comment.