Move CI job configuration to elasticsearch repository

belkasoft · May 7, 2021 · 5e8bcde · 5e8bcde
1 parent 4d76163
commit 5e8bcde
Show file tree

Hide file tree

Showing 47 changed files with 752 additions and 83 deletions.
diff --git a/.ci/build.sh b/.ci/build.sh
diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties
@@ -6,5 +6,4 @@
 
 ES_BUILD_JAVA=openjdk11
 ES_RUNTIME_JAVA=java8
-GRADLE_TASK=build
-GRADLE_EXTRA_ARGS=--no-parallel
+
diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml
@@ -17,38 +17,50 @@
       daysToKeep: 30
       numToKeep: 90
       artifactDaysToKeep: 7
+    parameters:
+      - string:
+          name: branch_specifier
+          default: "refs/heads/%BRANCH%"
+          description: "the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;, &lt;commitId&gt;, etc.)\n"
     scm:
       - git:
           name: origin
           # master node jenkins user ~/.ssh
           credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
           reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git"
           branches:
-            - "%BRANCH%"
+            - "${branch_specifier}"
           url: "https://github.com/elastic/elasticsearch.git"
           basedir: ""
-          wipe-workspace: "yes"
+          wipe-workspace: true
+    triggers: []
     wrappers:
       - timeout:
           type: absolute
-          timeout: "120"
+          timeout: 480
           fail: true
       - ansicolor
       - timestamps
       - gradle-build-scan
+      - inject-passwords:
+          global: false
+          job-passwords:
+            - name: VAULT_ADDR
+              password: https://secrets.elastic.co:8200
+          mask-password-params: true
     properties:
-        - github:
-            url: https://github.com/elastic/elasticsearch/
-        - inject:
-            properties-content: |
-              HOME=$JENKINS_HOME
-    builders:
-       - shell: |
-          #!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push
-          .ci/build.sh -Dbwc.checkout.align=true check
+      - github:
+          url: https://github.com/elastic/elasticsearch/
+      - inject:
+          properties-content: |
+            HOME=$JENKINS_HOME
+            GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+            GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
     publishers:
-      - email:
-          recipients: [email protected]
+      - junit:
+          results: "**/*Junit/*.xml, **/test-results/*/*.xml"
+          keep-long-stdio: true
+          allow-empty-results: true
       # Upload additional logs
       - google-cloud-storage:
           credentials-id: 'elasticsearch-ci-gcs-plugin'
@@ -64,16 +76,13 @@
           builders:
             - role: SLAVE
               build-on:
-                  - SUCCESS
-                  - FAILURE
-                  - UNSTABLE
+                - SUCCESS
+                - FAILURE
+                - UNSTABLE
               build-steps:
-                - shell: |
-                    curl -sS -X POST \
-                      -H "Content-Type: text/plain" \
-                      --data "$BUILD_URL" \
-                      "https://homer.app.elstc.co/webhook/jenkins/build-finished" || true
-
-
-
-            
+                - http-request:
+                    url: https://homer.app.elstc.co/webhook/jenkins/build-finished
+                    mode: GET
+                    custom-headers:
+                      - name: X-Jenkins-Build
+                        value: ${BUILD_URL}
diff --git a/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml b/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml
@@ -0,0 +1,15 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+branch-consistency
+    display-name: "elastic / elasticsearch # %BRANCH% - branch consistency"
+    description: Testing of the Elasticsearch master branch version consistency.
+    triggers:
+      - timed: "H 7 * * *"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh branchConsistency
diff --git a/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml b/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+branch-protection
+    display-name: "elastic / elasticsearch # %BRANCH% - branch protection"
+    description: Elasticsearch %BRANCH% branch protection.
+    node: master
+    triggers:
+      - timed: "H 7 * * *"
+    scm: []
+    parameters: []
+    builders:
+      - shell: |
+          #!/bin/bash
+          set +x
+          STATUS=$(curl -s https://api.github.com/repos/elastic/elasticsearch/branches/%BRANCH% | jq '.protected')
+          echo "Branch %BRANCH% protection status is: $STATUS"
+          if [[ "$STATUS" == "false" ]]; then
+            echo "Development branch %BRANCH% is not set as protected in GitHub but should be."
+            exit 1
+          fi
diff --git a/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml b/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml
@@ -0,0 +1,4 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+pull-request
+    display-name: Pull Requests
+    project-type: folder
diff --git a/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml b/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml
@@ -0,0 +1,4 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+triggers
+    display-name: Periodic Triggers
+    project-type: folder
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake bwc"
+  - job-description: Elasticsearch %BRANCH% branch intake backwards compatibility checks.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files bwcTestSnapshots"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 1"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 1.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 2"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 2.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake sanity check"
+  - job-description: Elasticsearch %BRANCH% branch intake sanity check.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml
@@ -0,0 +1,11 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
+    display-name: "elastic / elasticsearch # %BRANCH% - update last good commit"
+    description: Elasticsearch %BRANCH% branch update last good commit in build-stats.
+    node: light
+    properties: []
+    builders:
+      - shell: |
+          #!/usr/local/bin/runbld --job-name elastic+elasticsearch+%BRANCH%+git+push
+          /usr/bin/true
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake.yml b/.ci/jobs.t/elastic+elasticsearch+intake.yml
@@ -0,0 +1,55 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+intake
+    display-name: "elastic / elasticsearch # %BRANCH% - intake"
+    description: "Testing of the Elasticsearch %BRANCH% branch on every push.\n"
+    project-type: multijob
+    node: master
+    vault: []
+    triggers:
+      # We use this trigger instead of the provided "github" webhook trigger because it's more robust.
+      # Here we only trigger builds for pushes to the corresponding branch, rather than a push to any branch of the
+      # configured git repository. This avoids duplicate builds being triggered when pushes to multiple branches are
+      # done in quick succession.
+      - generic-webhook-trigger:
+          post-content-params:
+            - type: JSONPath
+              key: ref
+              value: '$.ref'
+          regex-filter-text: '$ref'
+          regex-filter-expression: "^refs/heads/%BRANCH%$"
+          cause: Push to GitHub (refs/heads/%BRANCH%)
+          silent-response: true
+    scm:
+      - git:
+          wipe-workspace: false
+    builders:
+      - multijob:
+          name: Sanity Check
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+      - multijob:
+          name: Verification
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+      - multijob:
+          name: Update last good commit
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml
@@ -0,0 +1,15 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-darwin
+    display-name: "elastic / elasticsearch # %BRANCH% - darwin compatibility"
+    description: "Elasticsearch %BRANCH% MacOS compatibility testing.\n"
+    node: "macosx && x86_64"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml
@@ -0,0 +1,41 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix
+    display-name: "elastic / elasticsearch # %BRANCH% - unix compatibility"
+    description: "Elasticsearch %BRANCH% unix compatibility testing.\n"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - amazon
+            - "centos-6&&immutable"
+            - "centos-7&&immutable"
+            - "centos-8&&immutable"
+            - "debian-8&&immutable"
+            - "debian-9&&immutable"
+            - "debian-10&&immutable"
+            - "fedora-32&&immutable"
+            - "opensuse-15-1&&immutable"
+            - "oraclelinux-6&&immutable"
+            - "oraclelinux-7&&immutable"
+            - "oraclelinux-8&&immutable"
+            - "sles-12&&immutable"
+            - "sles-15&&immutable"
+            - "ubuntu-16.04&&immutable"
+            - "ubuntu-18.04&&immutable"
+            - "ubuntu-20.04&&immutable"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml
@@ -0,0 +1,43 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-windows
+    display-name: "elastic / elasticsearch # %BRANCH% - windows compatibility"
+    description: "Elasticsearch %BRANCH% Windows compatibility testing.\n"
+    project-type: matrix
+    node: master
+    # Use a hard-coded workspace directory to avoid hitting file path limits with auto-generated workspace path
+    child-workspace: "C:\\Users\\jenkins\\workspace\\platform-support\\${BUILD_NUMBER}"
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - "windows-2012-r2"
+            - "windows-2016"
+            - "windows-2019"
+      # We shred out Windows testing into 4 parallel builds like on intake for expediency.
+      # Our tests run much slower on Windows so this avoids issues with builds timing out.
+      - axis:
+          type: user-defined
+          name: GRADLE_TASK
+          values:
+            - 'checkPart1'
+            - 'checkPart2'
+            - 'bwcTestSnapshots'
+            - 'checkRestCompat'
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+      - batch: |
+          del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
+          mkdir %USERPROFILE%\.gradle\init.d
+          copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\
+          (
+            echo call %GRADLEW_BAT% --max-workers=4 -Dbwc.checkout.align=true %GRADLE_TASK% ^|^| exit /b 1
+          ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr -