[CI] Enable build stage for arm architecture (elastic#21284)

Richardjgq · Oct 15, 2020 · f8ff7ce · f8ff7ce
1 parent be5c290
commit f8ff7ce
Show file tree

Hide file tree

Showing 13 changed files with 170 additions and 9 deletions.
diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh
@@ -5,7 +5,8 @@ MSG="environment variable missing"
 GO_VERSION=${GO_VERSION:?$MSG}
 PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
 HOME=${HOME:?$MSG}
-ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
 GVM_CMD="${HOME}/bin/gvm"
 
 if command -v go
@@ -19,10 +20,20 @@ then
     fi
 fi
 
+if [ "${ARCH}" == "aarch64" ] ; then
+    GVM_ARCH_SUFFIX=arm64
+elif [ "${ARCH}" == "x86_64" ] ; then
+    GVM_ARCH_SUFFIX=amd64
+elif [ "${ARCH}" == "i686" ] ; then
+    GVM_ARCH_SUFFIX=386
+else
+    GVM_ARCH_SUFFIX=arm
+fi
+
 echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}"
 chmod +x "${GVM_CMD}"
 
 gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -17,7 +17,6 @@ pipeline {
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_COMPOSE_VERSION = "1.21.0"
     DOCKER_REGISTRY = 'docker.elastic.co'
-    GOX_FLAGS = "-arch amd64"
     JOB_GCS_BUCKET = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
@@ -29,7 +28,7 @@ pipeline {
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
   }
   options {
-    timeout(time: 2, unit: 'HOURS')
+    timeout(time: 3, unit: 'HOURS')
     buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
@@ -46,6 +45,7 @@ pipeline {
     booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
     string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
+    booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.')
     booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
     string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
   }
@@ -223,10 +223,17 @@ def withBeatsEnv(Map args = [:], Closure body) {
   def withModule = args.get('withModule', false)
   def directory = args.get('directory', '')
 
-  def goRoot, path, magefile, pythonEnv, testResults, artifacts
+  def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags
 
   if(isUnix()) {
-    goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64"
+    if (isArm() && is64arm()) {
+      // TODO: nodeOS() should support ARM
+      goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64"
+      gox_flags = '-arch arm'
+    } else {
+      goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64"
+      gox_flags = '-arch amd64'
+    }
     path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}"
     magefile = "${WORKSPACE}/.magefile"
     pythonEnv = "${WORKSPACE}/python-env"
@@ -241,6 +248,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
     magefile = "${env.WORKSPACE}\\.magefile"
     testResults = "**\\build\\TEST*.xml"
     artifacts = "**\\build\\TEST*.out"
+    gox_flags = '-arch amd64'
   }
 
   deleteDir()
@@ -258,7 +266,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
     "PYTHON_ENV=${pythonEnv}",
     "RACE_DETECTOR=true",
     "TEST_COVERAGE=true",
-    "TEST_TAGS=${env.TEST_TAGS},oracle"
+    "TEST_TAGS=${env.TEST_TAGS},oracle",
+    "GOX_FLAGS=${gox_flags}"
   ]) {
     if(isDockerInstalled()) {
       dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
@@ -369,7 +378,9 @@ def archiveTestOutput(Map args = [:]) {
       def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
       log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
       if (folder.trim()) {
-        def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS()
+        // TODO: nodeOS() should support ARM
+        def os_suffix = isArm() ? 'linux' : nodeOS()
+        def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
         tar(file: "${name}.tgz", archive: true, dir: folder)
       }
     }

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -13,9 +13,22 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test auditbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
-    crosscompile: 
+    crosscompile:
         make: "make -C auditbeat crosscompile"
     macos:
         mage: "mage build unitTest"

diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -13,6 +13,19 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test filebeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -13,6 +13,19 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test heartbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
     macos:

diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml
@@ -13,5 +13,18 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test journalbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     unitTest:
         mage: "mage build unitTest"
diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml
@@ -12,6 +12,17 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test libbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
     build:
         mage: "mage build test"
     crosscompile:

diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
@@ -13,6 +13,19 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test packetbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
     macos:

diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
@@ -13,6 +13,19 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/auditbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage update build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -13,6 +13,19 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/elastic-agent for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
     macos:

diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
@@ -13,6 +13,19 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/filebeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
@@ -13,6 +13,17 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/functionbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
     build:
         mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions"
     macos:

diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml
@@ -13,5 +13,18 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+    arm:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "arm"
+        when:                  ## Override the top-level when.
+            comments:
+                - "/test x-pack/libbeat for arm"
+            labels:
+                - "arm"
+            parameters:
+                - "armTest"
+            branches: true     ## for all the branches
+            tags: true         ## for all the tags
     build:
         mage: "mage build test"