Merge pull request apache#2934 from PurelyApplied/geode-6109

GEODE-6109: Reintroduce Build job to precheckin pipeline
ringles · Dec 4, 2018 · bbd36af · bbd36af
2 parents 6da5ac3 + d0fdba0
commit bbd36af
Show file tree

Hide file tree

Showing 5 changed files with 179 additions and 61 deletions.
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
@@ -137,20 +137,6 @@ groups:
   {%- endfor -%}
 {%- endfor %}
 
-resources:
-{% if repository.sanitized_fork == repository.upstream_fork %}
-- name: send-notification-email
-  type: email
-  source:
-    from: ((!source-email-address))
-    smtp:
-      host: ((!source-email-server))
-      password: ((!source-email-password))
-      port: "587"
-      username: ((!source-email-username))
-    to:
-    - ((!notification-email-address))
-{% endif %}
 - name: concourse-metadata-resource
   type: concourse-metadata-resource
   source: {}
@@ -192,12 +178,6 @@ resource_types:
   type: docker-image
   source:
     repository: frodenas/gcs-resource
-{% if repository.sanitized_fork == repository.upstream_fork %}
-- name: email
-  type: docker-image
-  source:
-    repository: pcfseceng/email-resource
-{% endif %}
 jobs:
 - name: {{build_test.name}}
   public: true
@@ -263,14 +243,6 @@ jobs:
       outputs:
       - name: built-geode
       - name: results
-{% if repository.sanitized_fork == repository.upstream_fork %}
-    on_failure:
-      aggregate:
-      - put: send-notification-email
-        params:
-          body: results/body
-          subject: results/subject
-{% endif %}
     ensure:
       do:
       - task: rsync_code_down

diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml
@@ -21,6 +21,7 @@
 groups:
 - name: main
   jobs:
+  - {{ build_test.name }}
 {%- for test in tests if not test.name.startswith("Windows") %}
   {%- for java_test_version in (java_test_versions) %}
   - {{test.name}}Test{{java_test_version.name}}
@@ -66,6 +67,140 @@ resource_types:
     tag: latest
     username: ((!docker-username))
 jobs:
+- name: {{build_test.name}}
+  public: true
+  plan:
+  - do:
+    - aggregate:
+      - get: geode
+        trigger: true
+        version: every
+      - get: geode-ci
+    - aggregate:
+      - do:
+        - put: pull-request-job-pending
+          resource: geode
+          params:
+            context: {{ build_test.name }}
+            path: geode
+            status: pending
+          get_params: {skip_download: true}
+      - do:
+        - put: concourse-metadata-resource
+        - task: create_instance
+          {{- alpine_tools_config()|indent(10) }}
+            params:
+              CPUS: {{build_test.CPUS}}
+              RAM: {{build_test.RAM}}
+              GCP_PROJECT: ((gcp-project))
+              GEODE_BRANCH: {{repository.branch}}
+              GEODE_FORK: {{repository.fork}}
+              JAVA_BUILD_VERSION: 8
+              JAVA_TEST_VERSION: 8
+            run:
+              path: geode-ci/ci/scripts/create_instance.sh
+            inputs:
+            - name: concourse-metadata-resource
+            - name: geode
+            - name: geode-ci
+            outputs:
+            - name: instance-data
+          timeout: 15m
+          attempts: 100
+    - task: rsync_code_up
+      {{- alpine_tools_config()|indent(6) }}
+        run:
+          path: geode-ci/ci/scripts/rsync_code_up.sh
+        inputs:
+        - name: geode-ci
+        - name: geode
+        - name: instance-data
+      timeout: 5m
+    - task: build
+      {{- alpine_tools_config()|indent(6) }}
+        params:
+          JAVA_BUILD_VERSION: 8
+          GRADLE_TASK: {{ build_test.GRADLE_TASK }}
+          ARTIFACT_SLUG: {{build_test.ARTIFACT_SLUG}}
+          JAVA_TEST_VERSION: 8
+          CALL_STACK_TIMEOUT: {{build_test.CALL_STACK_TIMEOUT}}
+          DUNIT_PARALLEL_FORKS: {{build_test.DUNIT_PARALLEL_FORKS}}
+          GRADLE_TASK_OPTIONS: ""
+          MAINTENANCE_VERSION: {{repository.branch}}
+          PARALLEL_DUNIT: {{build_test.PARALLEL_DUNIT}}
+          PARALLEL_GRADLE: {{build_test.PARALLEL_GRADLE}}
+          ARTIFACT_BUCKET: ((artifact-bucket))
+          SERVICE_ACCOUNT: ((concourse-gcp-account))
+          GRADLE_GLOBAL_ARGS: ((gradle-global-args))
+        run:
+          path: geode-ci/ci/scripts/execute_build.sh
+        inputs:
+        - name: geode-ci
+        - name: geode
+        - name: instance-data
+      timeout: {{build_test.execute_test_timeout}}
+    on_failure:
+      do:
+      - put: pull-request-job-failure
+        resource: geode
+        params:
+          context: {{ build_test.name }}
+          path: geode
+          status: failure
+        get_params: {skip_download: true}
+    ensure:
+      do:
+      - task: rsync_code_down
+        {{- alpine_tools_config()|indent(8) }}
+          params:
+            JAVA_BUILD_VERSION: 8
+            ARTIFACT_SLUG: {{build_test.ARTIFACT_SLUG}}
+          run:
+            path: geode-ci/ci/scripts/rsync_code_down.sh
+          inputs:
+          - name: geode
+          - name: geode-ci
+          - name: instance-data
+          outputs:
+          - name: geode-results
+        timeout: 5m
+      ensure:
+        aggregate:
+        - task: archive_results
+          {{- alpine_tools_config()|indent(10) }}
+            params:
+              ARTIFACT_SLUG: {{build_test.ARTIFACT_SLUG}}
+              GRADLE_TASK: {{build_test.GRADLE_TASK}}
+              MAINTENANCE_VERSION: {{repository.branch}}
+              ARTIFACT_BUCKET: ((artifact-bucket))
+              SERVICE_ACCOUNT: ((!concourse-gcp-account))
+            run:
+              path: geode-ci/ci/scripts/archive_results.sh
+            inputs:
+            - name: concourse-metadata-resource
+            - name: geode
+            - name: geode-ci
+            - name: geode-results
+        - task: delete_instance
+          {{- alpine_tools_config()|indent(10) }}
+            run:
+              path: geode-ci/ci/scripts/delete_instance.sh
+            inputs:
+            - name: geode
+            - name: geode-ci
+            - name: instance-data
+          timeout: 1h
+    on_success:
+      do:
+      - put: pull-request-job-success
+        resource: geode
+        params:
+          context: {{ build_test.name }}
+          path: geode
+          status: success
+        get_params: {skip_download: true}
+
+
 {% for test in tests if not test.name.startswith("Windows")%}
   {%- for java_test_version in (java_test_versions) %}
 - name: {{test.name}}Test{{java_test_version.name}}

diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
@@ -26,9 +26,10 @@ build_test:
   CALL_STACK_TIMEOUT: "1800"
   PARALLEL_DUNIT: "false"
   DUNIT_PARALLEL_FORKS: "4"
-  # Actual gradle task is "build install" but is hard-coded in execute_build.sh
+  # Actual gradle task is quite a bit more than "build" but is hard-coded in execute_build.sh
   # It's consumption is more like a "TEST_RESULTS_DESTINATIONS_DIRECTORY_STUB_THING.
   GRADLE_TASK: build
+  PARALLEL_GRADLE: "false"
   execute_test_timeout: 10m
 examples_test:
   name: "TestExamples"

diff --git a/ci/scripts/execute_build.sh b/ci/scripts/execute_build.sh
@@ -36,36 +36,43 @@ fi
 
 ROOT_DIR=$(pwd)
 BUILD_DATE=$(date +%s)
-EMAIL_SUBJECT="results/subject"
-EMAIL_BODY="results/body"
-
-GEODE_BUILD_VERSION_FILE=${ROOT_DIR}/geode-build-version/number
-GEODE_RESULTS_VERSION_FILE=${ROOT_DIR}/results/number
-GEODE_BUILD_VERSION_NUMBER=$(grep "versionNumber *=" geode/gradle.properties | awk -F "=" '{print $2}' | tr -d ' ')
-GEODE_BUILD_DIR=/tmp/geode-build
-GEODE_PULL_REQUEST_ID_FILE=${ROOT_DIR}/geode/.git/id
-
-if [ -e "${GEODE_PULL_REQUEST_ID_FILE}" ]; then
-  GEODE_PULL_REQUEST_ID=$(cat ${GEODE_PULL_REQUEST_ID_FILE})
-  FULL_PRODUCT_VERSION="geode-pr-${GEODE_PULL_REQUEST_ID}"
-else
-  CONCOURSE_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE})
-  CONCOURSE_PRODUCT_VERSION=${CONCOURSE_VERSION%%-*}
-  GEODE_PRODUCT_VERSION=${GEODE_BUILD_VERSION_NUMBER}
-  CONCOURSE_BUILD_SLUG=${CONCOURSE_VERSION##*-}
-  BUILD_ID=${CONCOURSE_VERSION##*.}
-  FULL_PRODUCT_VERSION=${GEODE_PRODUCT_VERSION}-${CONCOURSE_BUILD_SLUG}
-  echo "Concourse VERSION is ${CONCOURSE_VERSION}"
-  echo "Geode product VERSION is ${GEODE_PRODUCT_VERSION}"
-  echo "Build ID is ${BUILD_ID}"
-fi
 
-echo -n "${FULL_PRODUCT_VERSION}" > ${GEODE_RESULTS_VERSION_FILE}
+# Precheckin does not get a geode-build-version
+if [ -e "${ROOT_DIR}/geode-build-version" ] ; then
+  GEODE_BUILD_VERSION_FILE=${ROOT_DIR}/geode-build-version/number
+  GEODE_RESULTS_VERSION_FILE=${ROOT_DIR}/results/number
+  GEODE_BUILD_VERSION_NUMBER=$(grep "versionNumber *=" geode/gradle.properties | awk -F "=" '{print $2}' | tr -d ' ')
+  GEODE_BUILD_DIR=/tmp/geode-build
+  GEODE_PULL_REQUEST_ID_FILE=${ROOT_DIR}/geode/.git/id
+
+  if [ -e "${GEODE_PULL_REQUEST_ID_FILE}" ]; then
+    GEODE_PULL_REQUEST_ID=$(cat ${GEODE_PULL_REQUEST_ID_FILE})
+    FULL_PRODUCT_VERSION="geode-pr-${GEODE_PULL_REQUEST_ID}"
+  else
+    CONCOURSE_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE})
+    CONCOURSE_PRODUCT_VERSION=${CONCOURSE_VERSION%%-*}
+    GEODE_PRODUCT_VERSION=${GEODE_BUILD_VERSION_NUMBER}
+    CONCOURSE_BUILD_SLUG=${CONCOURSE_VERSION##*-}
+    BUILD_ID=${CONCOURSE_VERSION##*.}
+    FULL_PRODUCT_VERSION=${GEODE_PRODUCT_VERSION}-${CONCOURSE_BUILD_SLUG}
+    echo "Concourse VERSION is ${CONCOURSE_VERSION}"
+    echo "Geode product VERSION is ${GEODE_PRODUCT_VERSION}"
+    echo "Build ID is ${BUILD_ID}"
+  fi
 
-DEFAULT_GRADLE_TASK_OPTIONS="--parallel --console=plain --no-daemon"
+  echo -n "${FULL_PRODUCT_VERSION}" > ${GEODE_RESULTS_VERSION_FILE}
+fi
+
+if [[ ${PARALLEL_GRADLE:-"true"} == "true" ]]; then
+  PARALLEL_GRADLE="--parallel"
+else
+  PARALLEL_GRADLE=""
+fi
+DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon"
+GRADLE_SKIP_TASK_OPTIONS=""
 
 SSHKEY_FILE="instance-data/sshkey"
-SSH_OPTIONS="-i ${SSHKEY_FILE} -o ConnectionAttempts=60 -o StrictHostKeyChecking=no"
+SSH_OPTIONS="-i ${SSHKEY_FILE} -o ConnectionAttempts=60 -o StrictHostKeyChecking=no -o ServerAliveInterval=60"
 
 INSTANCE_IP_ADDRESS="$(cat instance-data/instance-ip-address)"
 
@@ -88,10 +95,12 @@ if [ -v CALL_STACK_TIMEOUT ]; then
   ssh ${SSH_OPTIONS} geode@${INSTANCE_IP_ADDRESS} "tmux new-session -d -s callstacks; tmux send-keys  ~/capture-call-stacks.sh\ ${PARALLEL_DUNIT}\ ${CALL_STACK_TIMEOUT} C-m"
 fi
 
-GRADLE_COMMAND="./gradlew \
+GRADLE_ARGS="\
     ${DEFAULT_GRADLE_TASK_OPTIONS} \
+    ${GRADLE_SKIP_TASK_OPTIONS} \
     ${GRADLE_GLOBAL_ARGS} \
-    build install -x test"
+    build install javadoc spotlessCheck rat checkPom -x test"
 
-echo "${GRADLE_COMMAND}"
-ssh ${SSH_OPTIONS} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${SET_JAVA_HOME} && ${GRADLE_COMMAND}"
+EXEC_COMMAND="mkdir -p tmp && cd geode && ${SET_JAVA_HOME} && ./gradlew ${GRADLE_ARGS}"
+echo "${EXEC_COMMAND}"
+ssh ${SSH_OPTIONS} geode@${INSTANCE_IP_ADDRESS} "${EXEC_COMMAND}"
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
@@ -41,8 +41,8 @@ if [[ ${PARALLEL_GRADLE:-"true"} == "true" ]]; then
 else
   PARALLEL_GRADLE=""
 fi
-DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon -x javadoc -x spotlessCheck -x rat"
-
+DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon"
+GRADLE_SKIP_TASK_OPTIONS="-x javadoc -x spotlessCheck -x rat"
 
 SSHKEY_FILE="instance-data/sshkey"
 SSH_OPTIONS="-i ${SSHKEY_FILE} -o ConnectionAttempts=60 -o StrictHostKeyChecking=no -o ServerAliveInterval=60"
@@ -90,6 +90,7 @@ GRADLE_ARGS=" \
     ${DUNIT_PARALLEL_FORKS} \
     -PdunitDockerImage=\$(docker images --format '{{.Repository}}:{{.Tag}}') \
     ${DEFAULT_GRADLE_TASK_OPTIONS} \
+    ${GRADLE_SKIP_TASK_OPTIONS} \
     ${GRADLE_TASK} \
     ${GRADLE_TASK_OPTIONS} \
     ${GRADLE_GLOBAL_ARGS}"