Skip to content

Commit

Permalink
GEODE-5380: create PR pipeline that integrates with GitHub pull requests
Browse files Browse the repository at this point in the history
Single pipeline that runs all GitHub PRs. Status of each job is posted
back as a 'status' to the commit of the PR as they complete, like
Travis.

Runs in parallel.
PRs have links directly to their job in Concourse.

Signed-off-by: Sean Goller <[email protected]>

I hate yaml.

So close no matter how far
Couldn't be much more from the heart
forever trusting who we are
and whitespace just matters.
  • Loading branch information
robbadler authored and metatype committed Jul 3, 2018
1 parent 4ec2878 commit 2e5bff9
Show file tree
Hide file tree
Showing 8 changed files with 583 additions and 40 deletions.
15 changes: 14 additions & 1 deletion ci/pipelines/deploy_meta.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,21 @@ GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-")
TARGET=geode
GEODE_FORK=${1:-apache}
TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}')

PUBLIC=true

echo "Deploying pipline for ${GEODE_FORK}/${GEODE_BRANCH}"

if [ "${TEAM}" = "staging" ]; then
PUBLIC=false
fi

set -x
fly -t ${TARGET} set-pipeline -p meta-${SANITIZED_GEODE_BRANCH} -c meta.yml --var geode-build-branch=${GEODE_BRANCH} --var geode-fork=${GEODE_FORK}
fly -t ${TARGET} set-pipeline \
-p meta-${SANITIZED_GEODE_BRANCH} \
-c meta.yml \
--var geode-build-branch=${GEODE_BRANCH} \
--var geode-fork=${GEODE_FORK} \
--var concourse-team=${TEAM} \
--yaml-var public-pipelines=${PUBLIC}
38 changes: 32 additions & 6 deletions ci/pipelines/meta.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ resources:
target: https://concourse.apachegeode-ci.info
insecure: "true"
teams:
- name: main
- name: ((!concourse-team))
username: ((!concourse-username))
password: ((!concourse-password))
- name: geode-pipeline
Expand Down Expand Up @@ -63,7 +63,7 @@ resources:
jobs:
- name: set-pipeline
serial: true
public: true
public: ((!public-pipelines))
plan:
- get: geode-pipeline
trigger: true
Expand All @@ -82,30 +82,56 @@ jobs:
GEODE_FORK: ((!geode-fork))
CONCOURSE_USERNAME: ((!concourse-username))
CONCOURSE_PASSWORD: ((!concourse-password))
CONCOURSE_TEAM: ((!concourse-team))
run:
path: geode-pipeline/ci/pipelines/geode-build/deploy_pipeline.sh

- name: set-pr-pipeline
serial: true
public: ((!public-pipelines))
plan:
- get: geode-pipeline
trigger: true
- get: apachegeode-build-concourse-docker-image
- task: deploy-pr
image: apachegeode-build-concourse-docker-image
config:
inputs:
- name: geode-pipeline
outputs:
- name: results
platform: linux
params:
OUTPUT_DIRECTORY: results
GEODE_BRANCH: ((!geode-build-branch))
GEODE_FORK: ((!geode-fork))
CONCOURSE_USERNAME: ((!concourse-username))
CONCOURSE_PASSWORD: ((!concourse-password))
CONCOURSE_TEAM: ((!concourse-team))
run:
path: geode-pipeline/ci/pipelines/pull-request/deploy_pr_pipeline.sh

- name: set-docker-images-pipeline
serial: true
public: true
public: ((!public-pipelines))
plan:
- get: geode-docker-images-pipeline
trigger: true
- put: apachegeode-concourse
params:
pipelines:
- name: docker-images
team: main
team: ((!concourse-team))
config_file: geode-docker-images-pipeline/ci/pipelines/docker-images.yml
- name: set-metrics-pipeline
serial: true
public: true
public: ((!public-pipelines))
plan:
- get: geode-metrics-pipeline
trigger: true
- put: apachegeode-concourse
params:
pipelines:
- name: develop-metrics
team: main
team: ((!concourse-team))
config_file: geode-metrics-pipeline/ci/pipelines/metrics.yml
277 changes: 277 additions & 0 deletions ci/pipelines/pull-request/base.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,277 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

---

resource_types:
- name: gcs-resource
type: docker-image
source:
repository: frodenas/gcs-resource
- name: email
type: docker-image
source:
repository: pcfseceng/email-resource
- name: pull-request
type: docker-image
source:
repository: jtarchie/pr

resources:
- name: docker-geode-build-image
type: docker-image
source:
username: ((!docker-username))
password: ((!docker-password))
repository: gcr.io/apachegeode-ci/((!docker-image-name))
tag: latest

- name: geode
type: pull-request
source:
access_token: ((!github-pr-access-token))
repo: (( concat metadata.geode-fork "/geode" ))
base: develop
ignore_paths:
- geode-docs/*
- geode-book/*

- name: send-notification-email
type: email
source:
smtp:
host: ((!source-email-server))
port: "587"
username: ((!source-email-username))
password: ((!source-email-password))
from: ((!source-email-address))
to: [ ((!notification-email-address)) ]

groups:
- name: main
jobs:
- Build
- DistributedTest
- AcceptanceTest
- IntegrationTest
- FlakyTest
- UITests

jobs:
- name: Build
serial: false
public: true
plan:
- aggregate:
- get: geode
trigger: true
version: every
params:
fetch_merge: true
git:
depth: 100
- get: docker-geode-build-image
params:
rootfs: true
# Unmerged pull request SHA, for writing status into GitHub
- get: geode-unmerged-request
resource: geode
version: every
params:
fetch_merge: false
- aggregate:
- put: geode
params:
path: geode-unmerged-request
context: $BUILD_JOB_NAME
status: pending

- task: build
image: docker-geode-build-image
config:
platform: linux
inputs:
- name: geode
outputs:
- name: built-geode
- name: results
params:
MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
run:
path: geode/ci/scripts/build.sh
on_failure:
aggregate:
- put: geode
params:
path: geode-unmerged-request
status: failure
context: $BUILD_JOB_NAME
- put: send-notification-email
params:
subject: results/subject
body: results/body
on_success:
aggregate:
- put: geode
params:
path: geode-unmerged-request
status: success
context: $BUILD_JOB_NAME

- name: DistributedTest
serial: false
public: true
plan:
- aggregate:
- get: geode
trigger: true
version: every
params:
fetch_merge: true
git:
depth: 100
- get: docker-geode-build-image
params:
rootfs: true
# Unmerged pull request SHA, for writing status into GitHub
- get: geode-unmerged-request
resource: geode
version: every
params:
fetch_merge: false

- aggregate:
- put: geode
params:
path: geode-unmerged-request
context: $BUILD_JOB_NAME
status: pending

- task: run-distributed-core
image: docker-geode-build-image
tags: [large]
privileged: true
timeout: 8h
config:
inputs:
- name: geode
- name: docker-geode-build-image
platform: linux
outputs:
- name: built-geode
params:
MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
PARALLEL_DUNIT: true
DUNIT_PARALLEL_FORKS: 7
CALL_STACK_TIMEOUT: 25200
run:
args:
- geode-core:distributedTest
- distributedtestcore
path: geode/ci/scripts/test-run.sh
on_failure:
aggregate:
- put: send-notification-email
params:
subject: built-geode/subject
body: built-geode/body

ensure:
aggregate:
- task: archive-results-core
image: docker-geode-build-image
config:
inputs:
- name: geode
- name: built-geode
platform: linux
params:
MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
run:
args:
- geode-core:distributedTest
- distributedtestcore
path: geode/ci/scripts/test-archive.sh
- task: run-distributed-everything-else
image: docker-geode-build-image
tags: [large]
privileged: true
timeout: 8h
config:
inputs:
- name: geode
- name: docker-geode-build-image
platform: linux
outputs:
- name: built-geode
params:
MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
PARALLEL_DUNIT: true
DUNIT_PARALLEL_FORKS: 7
CALL_STACK_TIMEOUT: 25200
GRADLE_TASK_OPTIONS: -x geode-core:distributedTest
run:
args:
- distributedTest
- distributedtesteverythingelse
path: geode/ci/scripts/test-run.sh
on_failure:
aggregate:
- put: send-notification-email
params:
subject: built-geode/subject
body: built-geode/body
ensure:
aggregate:
- task: archive-results-everythingelse
image: docker-geode-build-image
config:
inputs:
- name: geode
- name: built-geode
platform: linux
params:
MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
run:
args:
- disitributedTest
- distributedtesteverythingelse
path: geode/ci/scripts/test-archive.sh
on_failure:
aggregate:
- put: geode
params:
path: geode-unmerged-request
status: failure
context: $BUILD_JOB_NAME
on_success:
aggregate:
- put: geode
params:
path: geode-unmerged-request
context: $BUILD_JOB_NAME
status: success
Loading

0 comments on commit 2e5bff9

Please sign in to comment.