forked from dmlc/xgboost
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Separate out restricted and unrestricted tasks (dmlc#3736)
- Loading branch information
Showing
3 changed files
with
192 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
#!/usr/bin/groovy | ||
// -*- mode: groovy -*- | ||
// Jenkins pipeline | ||
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ | ||
|
||
import groovy.transform.Field | ||
|
||
/* Restricted tasks: tasks generating artifacts, such as binary wheels and | ||
documentation */ | ||
|
||
// Command to run command inside a docker container | ||
def dockerRun = 'tests/ci_build/ci_build.sh' | ||
// Utility functions | ||
@Field | ||
def utils | ||
|
||
def buildMatrix = [ | ||
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "9.2" ], | ||
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": true, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ], | ||
[ "enabled": true, "os" : "linux", "withGpu": true, "withNccl": false, "withOmp": true, "pythonVersion": "2.7", "cudaVersion": "8.0" ], | ||
] | ||
|
||
pipeline { | ||
// Each stage specify its own agent | ||
agent none | ||
|
||
// Setup common job properties | ||
options { | ||
ansiColor('xterm') | ||
timestamps() | ||
timeout(time: 120, unit: 'MINUTES') | ||
buildDiscarder(logRotator(numToKeepStr: '10')) | ||
} | ||
|
||
// Build stages | ||
stages { | ||
stage('Jenkins: Get sources') { | ||
agent { | ||
label 'restricted' | ||
} | ||
steps { | ||
script { | ||
utils = load('tests/ci_build/jenkins_tools.Groovy') | ||
utils.checkoutSrcs() | ||
} | ||
stash name: 'srcs', excludes: '.git/' | ||
milestone label: 'Sources ready', ordinal: 1 | ||
} | ||
} | ||
stage('Jenkins: Build doc') { | ||
agent { | ||
label 'linux && cpu && restricted' | ||
} | ||
steps { | ||
unstash name: 'srcs' | ||
script { | ||
def commit_id = "${GIT_COMMIT}" | ||
def branch_name = "${GIT_LOCAL_BRANCH}" | ||
echo 'Building doc...' | ||
dir ('jvm-packages') { | ||
sh "bash ./build_doc.sh ${commit_id}" | ||
archiveArtifacts artifacts: "${commit_id}.tar.bz2", allowEmptyArchive: true | ||
echo 'Deploying doc...' | ||
withAWS(credentials:'xgboost-doc-bucket') { | ||
s3Upload file: "${commit_id}.tar.bz2", bucket: 'xgboost-docs', acl: 'PublicRead', path: "${branch_name}.tar.bz2" | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
stage('Jenkins: Build artifacts') { | ||
steps { | ||
script { | ||
parallel (buildMatrix.findAll{it['enabled']}.collectEntries{ c -> | ||
def buildName = utils.getBuildName(c) | ||
utils.buildFactory(buildName, c, true, this.&buildPlatformCmake) | ||
}) | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
/** | ||
* Build platform and test it via cmake. | ||
*/ | ||
def buildPlatformCmake(buildName, conf, nodeReq, dockerTarget) { | ||
def opts = utils.cmakeOptions(conf) | ||
// Destination dir for artifacts | ||
def distDir = "dist/${buildName}" | ||
def dockerArgs = "" | ||
if(conf["withGpu"]){ | ||
dockerArgs = "--build-arg CUDA_VERSION=" + conf["cudaVersion"] | ||
} | ||
// Build node - this is returned result | ||
node(nodeReq) { | ||
unstash name: 'srcs' | ||
echo """ | ||
|===== XGBoost CMake build ===== | ||
| dockerTarget: ${dockerTarget} | ||
| cmakeOpts : ${opts} | ||
|========================= | ||
""".stripMargin('|') | ||
// Invoke command inside docker | ||
sh """ | ||
${dockerRun} ${dockerTarget} ${dockerArgs} tests/ci_build/build_via_cmake.sh ${opts} | ||
${dockerRun} ${dockerTarget} ${dockerArgs} bash -c "cd python-package; rm -f dist/*; python setup.py bdist_wheel --universal" | ||
rm -rf "${distDir}"; mkdir -p "${distDir}/py" | ||
cp xgboost "${distDir}" | ||
cp -r lib "${distDir}" | ||
cp -r python-package/dist "${distDir}/py" | ||
# Test the wheel for compatibility on a barebones CPU container | ||
${dockerRun} release ${dockerArgs} bash -c " \ | ||
auditwheel show xgboost-*-py2-none-any.whl | ||
pip install --user python-package/dist/xgboost-*-none-any.whl && \ | ||
python -m nose tests/python" | ||
""" | ||
archiveArtifacts artifacts: "${distDir}/**/*.*", allowEmptyArchive: true | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
#!/usr/bin/groovy | ||
// -*- mode: groovy -*- | ||
|
||
/* Utility functions for Jenkins */ | ||
|
||
// Command to run command inside a docker container | ||
dockerRun = 'tests/ci_build/ci_build.sh' | ||
|
||
// initialize source codes | ||
def checkoutSrcs() { | ||
retry(5) { | ||
try { | ||
timeout(time: 2, unit: 'MINUTES') { | ||
checkout scm | ||
sh 'git submodule update --init' | ||
} | ||
} catch (exc) { | ||
deleteDir() | ||
error "Failed to fetch source codes" | ||
} | ||
} | ||
} | ||
|
||
/** | ||
* Creates cmake and make builds | ||
*/ | ||
def buildFactory(buildName, conf, restricted, build_func) { | ||
def os = conf["os"] | ||
def device = conf["withGpu"] ? "gpu" : "cpu" | ||
def restricted_flag = restricted ? "restricted" : "unrestricted" | ||
def nodeReq = "${os} && ${device} && ${restricted_flag}" | ||
def dockerTarget = conf["withGpu"] ? "gpu" : "cpu" | ||
[ ("${buildName}") : { build_func("${buildName}", conf, nodeReq, dockerTarget) } | ||
] | ||
} | ||
|
||
def cmakeOptions(conf) { | ||
return ([ | ||
conf["withGpu"] ? '-DUSE_CUDA=ON' : '-DUSE_CUDA=OFF', | ||
conf["withNccl"] ? '-DUSE_NCCL=ON' : '-DUSE_NCCL=OFF', | ||
conf["withOmp"] ? '-DOPEN_MP:BOOL=ON' : ''] | ||
).join(" ") | ||
} | ||
|
||
def getBuildName(conf) { | ||
def gpuLabel = conf['withGpu'] ? ("_cuda" + conf['cudaVersion'] + (conf['withNccl'] ? "_nccl" : "_nonccl")) : "_cpu" | ||
def ompLabel = conf['withOmp'] ? "_omp" : "" | ||
def pyLabel = "_py${conf['pythonVersion']}" | ||
return "${conf['os']}${gpuLabel}${ompLabel}${pyLabel}" | ||
} | ||
|
||
return this |