From f8ff7cec3c28efb747738442fec0a47b2a968841 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 15 Oct 2020 10:28:43 +0100 Subject: [PATCH] [CI] Enable build stage for arm architecture (#21284) --- .ci/scripts/install-go.sh | 15 +++++++++++++-- Jenkinsfile | 23 +++++++++++++++++------ auditbeat/Jenkinsfile.yml | 15 ++++++++++++++- filebeat/Jenkinsfile.yml | 13 +++++++++++++ heartbeat/Jenkinsfile.yml | 13 +++++++++++++ journalbeat/Jenkinsfile.yml | 13 +++++++++++++ libbeat/Jenkinsfile.yml | 11 +++++++++++ packetbeat/Jenkinsfile.yml | 13 +++++++++++++ x-pack/auditbeat/Jenkinsfile.yml | 13 +++++++++++++ x-pack/elastic-agent/Jenkinsfile.yml | 13 +++++++++++++ x-pack/filebeat/Jenkinsfile.yml | 13 +++++++++++++ x-pack/functionbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/libbeat/Jenkinsfile.yml | 13 +++++++++++++ 13 files changed, 170 insertions(+), 9 deletions(-) diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh index 49e12c7a18d3..59f56445a802 100755 --- a/.ci/scripts/install-go.sh +++ b/.ci/scripts/install-go.sh @@ -5,7 +5,8 @@ MSG="environment variable missing" GO_VERSION=${GO_VERSION:?$MSG} PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} HOME=${HOME:?$MSG} -ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') +OS=$(uname -s| tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') GVM_CMD="${HOME}/bin/gvm" if command -v go @@ -19,10 +20,20 @@ then fi fi +if [ "${ARCH}" == "aarch64" ] ; then + GVM_ARCH_SUFFIX=arm64 +elif [ "${ARCH}" == "x86_64" ] ; then + GVM_ARCH_SUFFIX=amd64 +elif [ "${ARCH}" == "i686" ] ; then + GVM_ARCH_SUFFIX=386 +else + GVM_ARCH_SUFFIX=arm +fi + echo "UNMET DEP: Installing Go" mkdir -p "${HOME}/bin" -curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}" chmod +x "${GVM_CMD}" gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} diff --git a/Jenkinsfile b/Jenkinsfile index 31d62c416ce8..f3618d6615f1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,6 @@ pipeline { DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' @@ -29,7 +28,7 @@ pipeline { XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 3, unit: 'HOURS') buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') @@ -46,6 +45,7 @@ pipeline { booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`') } @@ -223,10 +223,17 @@ def withBeatsEnv(Map args = [:], Closure body) { def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def goRoot, path, magefile, pythonEnv, testResults, artifacts + def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags if(isUnix()) { - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + if (isArm() && is64arm()) { + // TODO: nodeOS() should support ARM + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64" + gox_flags = '-arch arm' + } else { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + gox_flags = '-arch amd64' + } path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" @@ -241,6 +248,7 @@ def withBeatsEnv(Map args = [:], Closure body) { magefile = "${env.WORKSPACE}\\.magefile" testResults = "**\\build\\TEST*.xml" artifacts = "**\\build\\TEST*.out" + gox_flags = '-arch amd64' } deleteDir() @@ -258,7 +266,8 @@ def withBeatsEnv(Map args = [:], Closure body) { "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", "TEST_COVERAGE=true", - "TEST_TAGS=${env.TEST_TAGS},oracle" + "TEST_TAGS=${env.TEST_TAGS},oracle", + "GOX_FLAGS=${gox_flags}" ]) { if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") @@ -369,7 +378,9 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() + // TODO: nodeOS() should support ARM + def os_suffix = isArm() ? 'linux' : nodeOS() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix tar(file: "${name}.tgz", archive: true, dir: folder) } } diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 33d708e26352..b3f20af2d379 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,9 +13,22 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" - crosscompile: + crosscompile: make: "make -C auditbeat crosscompile" macos: mage: "mage build unitTest" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index ffe2cec98b49..09dbe948c72c 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test filebeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 8d8b00a865d5..1d41dbe581ed 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test heartbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 12bb63f4cc64..5715712dd4aa 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -13,5 +13,18 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test journalbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags unitTest: mage: "mage build unitTest" diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 64a43269b13d..692400e7253b 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,6 +12,17 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test libbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" crosscompile: diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 9fbe272b382c..adf6471b82ad 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test packetbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 5e0e2223dfab..1294c4681b49 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index a36472d123bd..bf1bfed3ddd6 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/elastic-agent for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 4b17c13b6ac2..5bd2bcd40cf8 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/filebeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 29f2ecf8b5a4..ecb2bd14e0ea 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,6 +13,17 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/functionbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" macos: diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 87019f071a09..ed22a8dfe70d 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,5 +13,18 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/libbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test"