Skip to content

Commit

Permalink
Separate out restricted and unrestricted tasks (dmlc#3736)
Browse files Browse the repository at this point in the history
  • Loading branch information
hcho3 authored Sep 28, 2018
1 parent 5a7f7e7 commit baef574
Show file tree
Hide file tree
Showing 3 changed files with 192 additions and 82 deletions.
101 changes: 19 additions & 82 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,15 @@
// Jenkins pipeline
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/

import groovy.transform.Field

/* Unrestricted tasks: tasks that do NOT generate artifacts */

// Command to run command inside a docker container
dockerRun = 'tests/ci_build/ci_build.sh'
def dockerRun = 'tests/ci_build/ci_build.sh'
// Utility functions
@Field
def utils

def buildMatrix = [
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ],
Expand All @@ -26,80 +33,37 @@ pipeline {

// Build stages
stages {
stage('Get sources') {
agent any
steps {
checkoutSrcs()
stash name: 'srcs', excludes: '.git/'
milestone label: 'Sources ready', ordinal: 1
stage('Jenkins: Get sources') {
agent {
label 'unrestricted'
}
}
stage('Build doc') {
agent any
steps {
script {
if (env.CHANGE_ID == null) { // This is a branch
def commit_id = "${GIT_COMMIT}"
def branch_name = "${GIT_LOCAL_BRANCH}"
echo 'Building doc...'
dir ('jvm-packages') {
sh "bash ./build_doc.sh ${commit_id}"
archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true
echo 'Deploying doc...'
withAWS(credentials:'xgboost-doc-bucket') {
s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2"
}
}
} else { // This is a pull request
echo 'Skipping doc build step for pull request'
}
utils = load('tests/ci_build/jenkins_tools.Groovy')
utils.checkoutSrcs()
}
stash name: 'srcs', excludes: '.git/'
milestone label: 'Sources ready', ordinal: 1
}
}
stage('Build & Test') {
stage('Jenkins: Build & Test') {
steps {
script {
parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
def buildName = getBuildName(c)
buildFactory(buildName, c)
def buildName = utils.getBuildName(c)
utils.buildFactory(buildName, c, false, this.&buildPlatformCmake)
})
}
}
}
}
}

// initialize source codes
def checkoutSrcs() {
retry(5) {
try {
timeout(time: 2, unit: 'MINUTES') {
checkout scm
sh 'git submodule update --init'
}
} catch (exc) {
deleteDir()
error "Failed to fetch source codes"
}
}
}

/**
* Creates cmake and make builds
*/
def buildFactory(buildName, conf) {
def os = conf["os"]
def nodeReq = conf["withGpu"] ? "${os} && gpu" : "${os}"
def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
[ ("${buildName}") : { buildPlatformCmake("${buildName}", conf, nodeReq, dockerTarget) }
]
}

/**
* Build platform and test it via cmake.
*/
def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
def opts = cmakeOptions(conf)
def opts = utils.cmakeOptions(conf)
// Destination dir for artifacts
def distDir = "dist/${buildName}"
def dockerArgs = ""
Expand All @@ -119,33 +83,6 @@ def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
sh """
${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/test_${dockerTarget}.sh
${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
rm -rf "${distDir}"; mkdir -p "${distDir}/py"
cp xgboost "${distDir}"
cp -r lib "${distDir}"
cp -r python-package/dist "${distDir}/py"
# Test the wheel for compatibility on a barebones CPU container
${dockerRun} release ${dockerArgs} bash -c " \
auditwheel show xgboost-*-py2-none-any.whl
pip install --user python-package/dist/xgboost-*-none-any.whl && \
python -m nose tests/python"
"""
archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
}
}

def cmakeOptions(conf) {
return ([
conf["withGpu"] ? '-DUSE_CUDA=ON' : '-DUSE_CUDA=OFF',
conf["withNccl"] ? '-DUSE_NCCL=ON' : '-DUSE_NCCL=OFF',
conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : '']
).join(" ")
}

def getBuildName(conf) {
def gpuLabel = conf['withGpu'] ? ("_cuda" + conf['cudaVersion'] + (conf['withNccl'] ? "_nccl" : "_nonccl")) : "_cpu"
def ompLabel = conf['withOmp'] ? "_omp" : ""
def pyLabel = "_py${conf['pythonVersion']}"
return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"
}

121 changes: 121 additions & 0 deletions Jenkinsfile-restricted
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#!/usr/bin/groovy
// -*- mode: groovy -*-
// Jenkins pipeline
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/

import groovy.transform.Field

/* Restricted tasks: tasks generating artifacts, such as binary wheels and
documentation */

// Command to run command inside a docker container
def dockerRun = 'tests/ci_build/ci_build.sh'
// Utility functions
@Field
def utils

def buildMatrix = [
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ],
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ],
]

pipeline {
// Each stage specify its own agent
agent none

// Setup common job properties
options {
ansiColor('xterm')
timestamps()
timeout(time: 120, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
}

// Build stages
stages {
stage('Jenkins: Get sources') {
agent {
label 'restricted'
}
steps {
script {
utils = load('tests/ci_build/jenkins_tools.Groovy')
utils.checkoutSrcs()
}
stash name: 'srcs', excludes: '.git/'
milestone label: 'Sources ready', ordinal: 1
}
}
stage('Jenkins: Build doc') {
agent {
label 'linux && cpu && restricted'
}
steps {
unstash name: 'srcs'
script {
def commit_id = "${GIT_COMMIT}"
def branch_name = "${GIT_LOCAL_BRANCH}"
echo 'Building doc...'
dir ('jvm-packages') {
sh "bash ./build_doc.sh ${commit_id}"
archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true
echo 'Deploying doc...'
withAWS(credentials:'xgboost-doc-bucket') {
s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2"
}
}
}
}
}

stage('Jenkins: Build artifacts') {
steps {
script {
parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c ->
def buildName = utils.getBuildName(c)
utils.buildFactory(buildName, c, true, this.&buildPlatformCmake)
})
}
}
}
}
}

/**
* Build platform and test it via cmake.
*/
def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) {
def opts = utils.cmakeOptions(conf)
// Destination dir for artifacts
def distDir = "dist/${buildName}"
def dockerArgs = ""
if(conf["withGpu"]){
dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"]
}
// Build node - this is returned result
node(nodeReq) {
unstash name: 'srcs'
echo """
|===== XGBoost CMake build =====
| dockerTarget: ${dockerTarget}
| cmakeOpts : ${opts}
|=========================
""".stripMargin('|')
// Invoke command inside docker
sh """
${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts}
${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal"
rm -rf "${distDir}"; mkdir -p "${distDir}/py"
cp xgboost "${distDir}"
cp -r lib "${distDir}"
cp -r python-package/dist "${distDir}/py"
# Test the wheel for compatibility on a barebones CPU container
${dockerRun} release ${dockerArgs} bash -c " \
auditwheel show xgboost-*-py2-none-any.whl
pip install --user python-package/dist/xgboost-*-none-any.whl && \
python -m nose tests/python"
"""
archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true
}
}
52 changes: 52 additions & 0 deletions tests/ci_build/jenkins_tools.Groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#!/usr/bin/groovy
// -*- mode: groovy -*-

/* Utility functions for Jenkins */

// Command to run command inside a docker container
dockerRun = 'tests/ci_build/ci_build.sh'

// initialize source codes
def checkoutSrcs() {
retry(5) {
try {
timeout(time: 2, unit: 'MINUTES') {
checkout scm
sh 'git submodule update --init'
}
} catch (exc) {
deleteDir()
error "Failed to fetch source codes"
}
}
}

/**
* Creates cmake and make builds
*/
def buildFactory(buildName, conf, restricted, build_func) {
def os = conf["os"]
def device = conf["withGpu"] ? "gpu" : "cpu"
def restricted_flag = restricted ? "restricted" : "unrestricted"
def nodeReq = "${os} && ${device} && ${restricted_flag}"
def dockerTarget = conf["withGpu"] ? "gpu" : "cpu"
[ ("${buildName}") : { build_func("${buildName}", conf, nodeReq, dockerTarget) }
]
}

def cmakeOptions(conf) {
return ([
conf["withGpu"] ? '-DUSE_CUDA=ON' : '-DUSE_CUDA=OFF',
conf["withNccl"] ? '-DUSE_NCCL=ON' : '-DUSE_NCCL=OFF',
conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : '']
).join(" ")
}

def getBuildName(conf) {
def gpuLabel = conf['withGpu'] ? ("_cuda" + conf['cudaVersion'] + (conf['withNccl'] ? "_nccl" : "_nonccl")) : "_cpu"
def ompLabel = conf['withOmp'] ? "_omp" : ""
def pyLabel = "_py${conf['pythonVersion']}"
return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}"
}

return this

0 comments on commit baef574

Please sign in to comment.