GEODE-5864 Refactor pipeline deployment. (apache#2707)

* deploy_meta.sh now looks at the file meta.properties for defaults * Users should create meta.properties.local file and override values there. A common customization is to override GEODE_FORK with your personal geode repository. * GRADLE_GLOBAL_ARGS is a string of gradle arguments that will be passed to every invocation of gradle. * Change "start_instance.sh" to "create_instance.sh" * Change "stop_instance.sh" to "delete_instance.sh" * Remove unnecessary gcloud delete invocation * Add on_failure block to rsync_code_up job to delete the instance. * Make on_failure call a do block to be consistent with other on_failures. * Add system property for excluding individual test classes. * Adjust excludeTests to handle multiple tests * Add maven snapshot bucket for storing published artifacts. * Make GCP_PROJECT a property instead of deriving it. * Make deploy_meta.sh more user-friendly by displaying values and offering the option to abort. * Forks should no longer be hard-coded. * Remove UpdatePassingRef job. * Remove unused scripts. Co-authored-by: Sean Goller <[email protected]> Co-authored-by: Jacob Barrett <[email protected]> Co-authored-by: Dick Cavender <[email protected]>
shjgiser · Oct 24, 2018 · 7d9858e · 7d9858e
1 parent 404c64d
commit 7d9858e
Show file tree

Hide file tree

Showing 20 changed files with 170 additions and 162 deletions.
diff --git a/.gitignore b/.gitignore
@@ -31,3 +31,4 @@ geode-pulse/screenshots/
 .git-together
 bin/
 .metadata/
+/ci/pipelines/meta/meta.properties.local
diff --git a/ci/images/google-windows-geode-builder/windows-packer.json b/ci/images/google-windows-geode-builder/windows-packer.json
@@ -43,7 +43,7 @@
 
         "Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicParsing | Invoke-Expression",
         "choco install -y git rsync openjdk",
-        "Move-Item \"C:\\Program Files\\OpenJDK\\jdk-11\" c:\\java11",
+        "Move-Item \"C:\\Program Files\\OpenJDK\\jdk-11*\" c:\\java11",
         "choco install -y jdk8 -params 'installdir=c:\\\\java8tmp;source=false'",
         "Move-Item \"C:\\java8tmp\" c:\\java8",
         "choco install -y openssh /SSHServerFeature",

diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh
@@ -71,6 +71,8 @@ public-pipelines: ${PUBLIC_PIPELINES}
 gcp-project: ${GCP_PROJECT}
 version-bucket: ${VERSION_BUCKET}
 artifact-bucket: ${ARTIFACT_BUCKET}
+gradle-global-args: ${GRADLE_GLOBAL_ARGS}
+maven-snapshot-bucket: ${MAVEN_SNAPSHOT_BUCKET}
 YML
 
 
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
@@ -71,8 +71,9 @@ DUNIT_PARALLEL_FORKS: {{ test.DUNIT_PARALLEL_FORKS }}
 MAINTENANCE_VERSION: ((geode-build-branch ))
 PARALLEL_DUNIT: {{ test.PARALLEL_DUNIT }}
 PARALLEL_GRADLE: {{ test.PARALLEL_GRADLE }}
-ARTIFACT_BUCKET: ((!artifact-bucket))
-SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ARTIFACT_BUCKET: ((artifact-bucket))
+SERVICE_ACCOUNT: ((concourse-gcp-account))
+GRADLE_GLOBAL_ARGS: ((gradle-global-args))
 {%- endmacro %}
 
 {% macro all_gating_jobs(test) %}
@@ -88,8 +89,7 @@ groups:
   jobs:
   - {{ build_test.name }}
   {{ all_gating_jobs() | indent(2) }}
-  {%- if repository.sanitized_fork == "apache" %}
-  - UpdatePassingRef
+  {%- if repository.sanitized_fork == repository.upstream_fork %}
   - PublishArtifacts
   {%- endif %}
 - name: complete
@@ -100,8 +100,7 @@ groups:
   - {{test.name}}Test{{java_test_version.name}}
     {%- endfor -%}
   {%- endfor -%}
-  {%- if repository.sanitized_fork == "apache" %}
-  - UpdatePassingRef
+  {%- if repository.sanitized_fork == repository.upstream_fork %}
   - PublishArtifacts
   {%- endif %}
 - name: linux
@@ -130,7 +129,7 @@ groups:
 {%- endfor %}
 
 resources:
-{% if repository.sanitized_fork == "apache" %}
+{% if repository.sanitized_fork == repository.upstream_fork %}
 - name: send-notification-email
   type: email
   source:
@@ -184,7 +183,7 @@ resource_types:
   type: docker-image
   source:
     repository: frodenas/gcs-resource
-{% if repository.sanitized_fork == "apache" %}
+{% if repository.sanitized_fork == repository.upstream_fork %}
 - name: email
   type: docker-image
   source:
@@ -204,14 +203,14 @@ jobs:
         pre: build
     - do:
       - put: concourse-metadata-resource
-      - task: start_instance
+      - task: create_instance
         {{ alpine_tools_config()|indent(8) }}
           params:
             {{ common_instance_params(build_test) | indent(12) }}
             GEODE_BRANCH: {{repository.branch}}
             GEODE_FORK: {{repository.fork}}
           run:
-            path: geode-ci/ci/scripts/start_instance.sh
+            path: geode-ci/ci/scripts/create_instance.sh
           inputs:
           - name: concourse-metadata-resource
           - name: geode-ci
@@ -228,6 +227,16 @@ jobs:
       - name: geode
       - name: instance-data
     timeout: 5m
+    on_failure:
+      do:
+      - task: delete_instance
+        {{ alpine_tools_config()|indent(8) }}
+          run:
+            path: geode-ci/ci/scripts/delete_instance.sh
+          inputs:
+          - name: geode-ci
+          - name: instance-data
+        timeout: 1h
   - task: build
     {{ alpine_tools_config()|indent(4) }}
       params:
@@ -244,7 +253,7 @@ jobs:
       outputs:
       - name: built-geode
       - name: results
-{% if repository.sanitized_fork == "apache" %}
+{% if repository.sanitized_fork == repository.upstream_fork %}
     on_failure:
       aggregate:
       - put: send-notification-email
@@ -289,38 +298,16 @@ jobs:
               - name: geode-build-version
               - name: geode-results
             timeout: 1h
-          - task: stop_instance
+          - task: delete_instance
             {{ alpine_tools_config()|indent(12) }}
               run:
-                path: geode-ci/ci/scripts/stop_instance.sh
+                path: geode-ci/ci/scripts/delete_instance.sh
               inputs:
               - name: geode-ci
               - name: instance-data
             timeout: 1h
 
-{% if repository.sanitized_fork == "apache" %}
-- name: UpdatePassingRef
-  public: true
-  serial: true
-  plan:
-  - get: geode
-    passed:
-    {{ all_gating_jobs() | indent(4) }}
-    trigger: true
-  - get: geode-ci
-  - task: updatepassingref
-    {{ alpine_tools_config()|indent(4) }}
-      params:
-        MAINTENANCE_VERSION: ((geode-build-branch))
-        ARTIFACT_BUCKET: ((artifact-bucket))
-        SERVICE_ACCOUNT: ((!concourse-gcp-account))
-      run:
-        path: geode-ci/ci/scripts/update-passing-ref.sh
-      inputs:
-      - name: geode
-      - name: geode-ci
-      outputs:
-      - name: results
+{% if repository.sanitized_fork == repository.upstream_fork %}
 - name: PublishArtifacts
   public: true
   plan:
@@ -335,14 +322,14 @@ jobs:
         pre: build
     - do:
       - put: concourse-metadata-resource
-      - task: start_instance
+      - task: create_instance
         {{ alpine_tools_config()|indent(8) }}
           params:
             {{ common_instance_params(publish_artifacts) | indent(12) }}
             GEODE_BRANCH: {{repository.branch}}
             GEODE_FORK: {{repository.fork}}
           run:
-            path: geode-ci/ci/scripts/start_instance.sh
+            path: geode-ci/ci/scripts/create_instance.sh
           inputs:
           - name: concourse-metadata-resource
           - name: geode-ci
@@ -366,6 +353,7 @@ jobs:
         ARTIFACT_BUCKET: ((artifact-bucket))
         SERVICE_ACCOUNT: ((!concourse-gcp-account))
         JAVA_BUILD_VERSION: {{ java_build_version.version }}
+        MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket))
       run:
         path: geode-ci/ci/scripts/execute_publish.sh
       inputs:
@@ -375,10 +363,10 @@ jobs:
       - name: geode-build-version
     ensure:
       do:
-      - task: stop_instance
+      - task: delete_instance
         {{ alpine_tools_config()|indent(8) }}
           run:
-            path: geode-ci/ci/scripts/stop_instance.sh
+            path: geode-ci/ci/scripts/delete_instance.sh
           inputs:
           - name: geode-ci
           - name: instance-data
@@ -400,14 +388,14 @@ jobs:
     {{ plan_resource_gets() |indent(4) }}
       - do:
         - put: concourse-metadata-resource
-        - task: start_instance
+        - task: create_instance
           {{ alpine_tools_config()|indent(10) }}
             params:
               {{ common_instance_params(parameters) | indent(14) }}
               GEODE_BRANCH: {{repository.branch}}
               GEODE_FORK: {{repository.fork}}
             run:
-              path: geode-ci/ci/scripts/start_instance.sh
+              path: geode-ci/ci/scripts/create_instance.sh
             inputs:
             - name: concourse-metadata-resource
             - name: geode-ci
@@ -475,10 +463,10 @@ jobs:
               - name: geode-build-version
               - name: geode-results
             timeout: 1h
-          - task: stop_instance
+          - task: delete_instance
             {{ alpine_tools_config()|indent(12) }}
               run:
-                path: geode-ci/ci/scripts/stop_instance.sh
+                path: geode-ci/ci/scripts/delete_instance.sh
               inputs:
               - name: geode-ci
               - name: instance-data

diff --git a/ci/pipelines/meta/README.md b/ci/pipelines/meta/README.md
@@ -6,39 +6,57 @@ number of deployment scenarios it handles.
 ##### Prerequisites
 You must be logged into the appropriate Google Cloud Platform project for this to work.
 
-## Deploying to Production
-If you run `deploy_meta.sh` with no arguments, its default behavior is to deploy to the [production
+### Deployment Configuration
+In this directory is a file called `meta.properties`. This file contains variables that change
+how Geode is deployed. These values should be overriden by creating a file called 
+`meta.properties.local` and changing value definitions in that file. Note that 
+`meta.properties.local` is ignored by git because those types of changes are solely related to
+a specific deployment scenario.
+
+## Configuration Variables
+
+### UPSTREAM_FORK
+The UPSTREAM_FORK value indicates the GitHub user that hosts the master repository for Geode. Its
+default is `apache`, which results in a repository URL of `https://github.com/apache/geode.git`.
+
+### GEODE_FORK
+The GEODE_FORK value indicates the github username that hosts the repository under test. For
+example, if the GitHub user `molly` wishes to deploy a pipeline testing Geode from her fork, she
+would set this value to `molly`. This results in the repository URL 
+`https://github.com/molly/geode.git`.
+
+### GEODE_REPO_NAME
+The GEODE_REPO_NAME value indicates the name of the repository under test. This defaults to `geode`.
+
+### CONCOURSE_HOST
+The CONCOURSE_HOST value indicates the hostname of the concourse infrastructure you wish to deploy
+the Geode CI pipeline to. The default value is `concourse.apachegeode-ci.info`. You might wish to 
+change this value if you are hosting your own CI infrastructure for Geode.
+
+### ARTIFACT_BUCKET
+The ARTIFACT_BUCKET value indicates the Google Cloud Storage bucket name in which Concourse will 
+store build artifacts and test results. The default value is `files.apachegeode-ci.info`.
+
+### PUBLIC
+The PUBLIC value indicates whether the deployed pipeline should be public or not. The default value
+is `true`, though in certain situations this is overridden, such as a production deployment.
+
+### REPOSITORY_PUBLIC
+The REPOSITORY_PUBLIC value indicates whether the repository under test is public or not. The
+default value is `true`. If this value is set to false, the pipeline will expect credentials to
+be available in the vault deployment associated with the concourse deployment.
+
+### GRADLE_GLOBAL_ARGS
+The GRADLE_GLOBAL_ARGS value is for passing a set of arguments to gradle wherever it is invoked in
+CI. The default value is an empty string.
+
+## Deploying
+Once you have made your configuration choices, simply run `deploy_meta.sh` with no arguments.
+its default behavior is to deploy to the [production
 environment](https://concourse.apachegeode-ci.info). It will autodetect your branch and deploy the 
 appropriate pipeline. This will assume the branch is present in the
 [upstream repository](https://github.com/apache/geode). The resulting meta pipeline will be named
 `apache-<branch>-meta`.
 
-## Deploying from your fork to Geode Concourse
-If you wish to deploy the meta pipeline using your fork as the source, you must specify your github
-username as the first argument: `./deploy_meta.sh <github username>`. The resulting meta pipeline
-will be named `<github username>-<branch>-meta`. The default value for this argument is `apache`.
-
-## Specifying a different repository
-If your repository is named something other than `geode`, you must specify it as a second argument:
-`./deploy_meta.sh apache geode-testing`. There are no optional arguments, so if you need
-to provide an argument beyond the first, you must manually specify earlier arguments even if
-you are using default values.
-
-## Specifying a different upstream
-If your upstream repository owner is not `apache` (you forked a fork) then you must specify it as the third argument:
-`./deploy_meta.sh <github username> geode-testing not-apache`.
-
-## Specifying a different concourse infrastructure
-If you would like to deploy the pipeline to a concourse infrastructure other than
-`concourse.apachegeode-ci.info`, you must provide its hostname as the fourth argument:
-`./deploy_meta.sh apache geode apache concourse.mydomain.com`
-
-## Specifying the GCS bucket for artifact and test result storage
-If you would like to use a bucket other than `files.apachegeode-ci.info` for storing test results
-and artifacts, you must specify it as the fifth argument:
-`./deploy_meta.sh apache geode apache concourse.apachegeode-ci.info <my-gcs-bucket-name>`
-
-## Changing whether the created pipelines are public or not
-If the upstream and fork names do not match, the pipeline will not be made public. Otherwise
-the pipelines will be public by default, but can be overridden with the fifth argument:
-`./deploy_meta.sh apache geode apache concourse.apachegeode-ci.info files.apachegeode-ci.info false`
+If you change the configuration to use your own fork of Geode, the resulting meta pipeline will be
+named `<your GitHub username>-<branch>-meta`.
diff --git a/ci/pipelines/meta/deploy_meta.sh b/ci/pipelines/meta/deploy_meta.sh
@@ -34,28 +34,39 @@ for cmd in Jinja2 PyYAML; do
   fi
 done
 
-if [ -z $(command -v gcloud) ]; then
-  echo "Install gcloud (try brew cask install google-cloud-sdk, or see https://cloud.google.com/sdk/docs/downloads-interactive#mac)"
-  exit 1
+META_PROPERTIES=${SCRIPTDIR}/meta.properties
+LOCAL_META_PROPERTIES=${SCRIPTDIR}/meta.properties.local
+
+## Load default properties file
+source ${META_PROPERTIES}
+echo "**************************************************"
+echo "Default Environment variables for this deployment:"
+cat ${SCRIPTDIR}/meta.properties | grep -v "^#"
+echo "**************************************************"
+
+## Load local overrides properties file
+if [[ -f ${LOCAL_META_PROPERTIES} ]]; then
+  echo "Locally overridden environment variables for this:"
+  cat ${SCRIPTDIR}/meta.properties.local
+  echo "**************************************************"
+  source ${LOCAL_META_PROPERTIES}
 fi
 
-GCP_PROJECT=${GCP_PROJECT:-$(gcloud info --format="value(config.project)")}
-echo "GCP_PROJECT=${GCP_PROJECT}"
-if [ -z ${GCP_PROJECT} ]; then
-  echo "GCP_PROJECT not set. Quitting"
-  exit 1
+source ${META_PROPERTIES}
+
+if [[ -f ${LOCAL_META_PROPERTIES} ]]; then
+  source ${LOCAL_META_PROPERTIES}
 fi
 
+read -n 1 -s -r -p "Press any key to continue or x to abort" DEPLOY
+echo
+if [[ "${DEPLOY}" == "x" ]]; then
+  echo "x pressed, aborting deploy."
+  exit 0
+fi
 set -e
 set -x
 
-GEODE_FORK=${1:-"apache"}
-GEODE_REPO_NAME=${2:-"geode"}
-UPSTREAM_FORK=${3:-"apache"}
-CONCOURSE_HOST=${4:-"concourse.apachegeode-ci.info"}
-ARTIFACT_BUCKET=${5:-"files.apachegeode-ci.info"}
-PUBLIC=${6:-"true"}
-REPOSITORY_PUBLIC=${7:-"true"}
 if [[ "${CONCOURSE_HOST}" == "concourse.apachegeode-ci.info" ]]; then
   CONCOURSE_SCHEME=https
 fi
@@ -95,6 +106,8 @@ pushd ${SCRIPTDIR} 2>&1 > /dev/null
     --var geode-repo-name=${GEODE_REPO_NAME} \
     --var upstream-fork=${UPSTREAM_FORK} \
     --var pipeline-prefix=${PIPELINE_PREFIX} \
+    --var gradle-global-args="${GRADLE_GLOBAL_ARGS}" \
+    --var maven-snapshot-bucket="${MAVEN_SNAPSHOT_BUCKET}" \
     --var concourse-team=main \
     --yaml-var public-pipelines=${PUBLIC} 2>&1 |tee flyOutput.log
 
@@ -203,13 +216,13 @@ set +x
 if [[ "${GEODE_FORK}" != "${UPSTREAM_FORK}" ]]; then
   echo "Disabling unnecessary jobs for forks."
   pauseJobs ${META_PIPELINE} set-images set-reaper
-elif [[ "$GEODE_FORK" == "apache" ]] && [[ "$GEODE_BRANCH" == "develop" ]]; then
+elif [[ "$GEODE_FORK" == "${UPSTREAM_FORK}" ]] && [[ "$GEODE_BRANCH" == "develop" ]]; then
   echo "Disabling optional jobs for develop"
-  pauseNewJobs set-pr set-images set-metrics set-examples
+  pauseNewJobs ${META_PIPELINE} set-pr set-images set-metrics set-examples
 else
   echo "Disabling unnecessary jobs for release branches."
   echo "*** DO NOT RE-ENABLE THESE META-JOBS ***"
-  pauseJobs set-pr set-images set-reaper set-metrics set-examples
+  pauseJobs ${META_PIPELINE} set-pr set-images set-reaper set-metrics set-examples
 fi
 
 unpausePipeline ${META_PIPELINE}