Skip to content

Commit

Permalink
Uploading of the build artifacts (hyperledger-iroha#1170)
Browse files Browse the repository at this point in the history
* upload artifacts on successful build

Signed-off-by: Artyom Bakhtin <[email protected]>

* temporarily disable mac coverage

Signed-off-by: Artyom Bakhtin <[email protected]>

* remove WIP

Signed-off-by: Artyom Bakhtin <[email protected]>

* replace artifacts.groovy paths concat with a more elegant version

Signed-off-by: Artyom Bakhtin <[email protected]>
  • Loading branch information
bakhtin authored Apr 10, 2018
1 parent 1d1a02a commit 80bae88
Show file tree
Hide file tree
Showing 5 changed files with 154 additions and 77 deletions.
44 changes: 44 additions & 0 deletions .jenkinsci/artifacts.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/usr/bin/env groovy

def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.co.jp']) {
def baseUploadPath = 'files'
def filePathsConverted = []
agentType = sh(script: 'uname', returnStdout: true).trim()
uploadPath = baseUploadPath + uploadPath
filePaths.each {
fp = sh(script: "ls -d ${it} | tr '\n' ','", returnStdout: true).trim()
filePathsConverted.addAll(fp.split(','))
}
def shaSumBinary = 'sha256sum'
def md5SumBinary = 'md5sum'
if (agentType == 'Darwin') {
shaSumBinary = 'shasum -a 256'
md5SumBinary = 'md5 -r'
}
sh "> \$(pwd)/batch.txt"
filePathsConverted.each {
sh "echo put ${it} $uploadPath >> \$(pwd)/batch.txt;"
sh "$shaSumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).sha256"
sh "$md5SumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).md5"
sh "echo put \$(pwd)/\$(basename ${it}).sha256 $uploadPath >> \$(pwd)/batch.txt;"
sh "echo put \$(pwd)/\$(basename ${it}).md5 $uploadPath >> \$(pwd)/batch.txt;"
}
// mkdirs recursively
uploadPath = uploadPath.split('/')
def p = ''
sh "> \$(pwd)/mkdirs.txt"
uploadPath.each {
p += "/${it}"
sh("echo -mkdir $p >> \$(pwd)/mkdirs.txt")
}

sshagent(['jenkins-artifact']) {
sh "ssh-agent"
artifactServers.each {
sh "sftp -b \$(pwd)/mkdirs.txt jenkins@${it} || true"
sh "sftp -b \$(pwd)/batch.txt jenkins@${it}"
}
}
}

return this
11 changes: 4 additions & 7 deletions .jenkinsci/debug-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ def doDebugBuild(coverageEnabled=false) {
+ " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+ " --network=${env.IROHA_NETWORK}"
+ " -v /var/jenkins/ccache:${CCACHE_DIR}") {
+ " -v /var/jenkins/ccache:${CCACHE_DIR}"
+ " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {

def scmVars = checkout scm
def cmakeOptions = ""
Expand Down Expand Up @@ -95,12 +96,8 @@ def doDebugBuild(coverageEnabled=false) {
sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
}

// TODO: replace with upload to artifactory server
// develop branch only
if ( env.BRANCH_NAME == "develop" ) {
//archive(includes: 'build/bin/,compile_commands.json')
}
// copy built binaries to the volume
sh "cp ./build/bin/* /tmp/${GIT_COMMIT}/"
}
}
return this
22 changes: 22 additions & 0 deletions .jenkinsci/linux-post-step.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
def linuxPostStep() {
timeout(time: 600, unit: "SECONDS") {
try {
if (currentBuild.result != "UNSTABLE" && BRANCH_NAME ==~ /(master|develop)/) {
def artifacts = load ".jenkinsci/artifacts.groovy"
def commit = env.GIT_COMMIT
def platform = sh(script: 'uname -m', returnStdout: true).trim()
filePaths = [ '/tmp/${GIT_COMMIT}-${BUILD_NUMBER}/*' ]
artifacts.uploadArtifacts(filePaths, sprintf('/iroha/linux/%4%s/%1$s-%2$s-%3$s', [BRANCH_NAME, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6), platform]))
}
}
finally {
if (env.BUILD_TYPE == 'Debug') {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
}
cleanWs()
}
}
}

return this
28 changes: 28 additions & 0 deletions .jenkinsci/mac-release-build.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/env groovy

def doReleaseBuild(coverageEnabled=false) {
def scmVars = checkout scm
env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
env.IROHA_HOME = "/opt/iroha"
env.IROHA_BUILD = "${env.IROHA_HOME}/build"

sh """
export CCACHE_DIR=${CCACHE_RELEASE_DIR}
ccache --version
ccache --show-stats
ccache --zero-stats
ccache --max-size=1G
cmake -H. \
-Bbuild \
-DCOVERAGE=OFF \
-DPACKAGE_TGZ=ON \
-DCMAKE_BUILD_TYPE=Release \
-DIROHA_VERSION=${env.IROHA_VERSION}
cmake --build build --target package -- -j${params.PARALLELISM}
ccache --show-stats
"""
}

return this
126 changes: 56 additions & 70 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,8 @@ pipeline {
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
cleanWs()
}
post = load ".jenkinsci/linux-post-step.groovy"
post.linuxPostStep()
}
}
}
Expand All @@ -140,11 +137,8 @@ pipeline {
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
cleanWs()
}
post = load ".jenkinsci/linux-post-step.groovy"
post.linuxPostStep()
}
}
}
Expand All @@ -171,11 +165,8 @@ pipeline {
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
cleanWs()
}
post = load ".jenkinsci/linux-post-step.groovy"
post.linuxPostStep()
}
}
}
Expand Down Expand Up @@ -246,25 +237,34 @@ pipeline {
sh "cmake --build build --target coverage.info"
sh "python /usr/local/bin/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false

}

// TODO: replace with upload to artifactory server
// only develop branch
if ( env.BRANCH_NAME == "develop" ) {
//archive(includes: 'build/bin/,compile_commands.json')
if (BRANCH_NAME ==~ /(master|develop)/) {
releaseBuild = load ".jenkinsci/mac-release-build.groovy"
releaseBuild.doReleaseBuild()
}
}
}
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
cleanWs()
sh """
pg_ctl -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ stop && \
rm -rf /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/
"""
timeout(time: 600, unit: "SECONDS") {
if (currentBuild.result != "UNSTABLE") {
if (BRANCH_NAME ==~ /(master|develop)/) {
try {
def artifacts = load ".jenkinsci/artifacts.groovy"
def commit = env.GIT_COMMIT
filePaths = [ '\$(pwd)/build/*.tar.gz' ]
artifacts.uploadArtifacts(filePaths, sprintf('/iroha/macos/%1$s-%2$s-%3$s', [BRANCH_NAME, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)]))
}
finally {
cleanWs()
sh """
pg_ctl -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ stop && \
rm -rf /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/
"""
}
}
}
}
}
}
Expand All @@ -282,7 +282,7 @@ pipeline {
parallel {
stage('Linux') {
when { expression { return params.Linux } }
agent { label 'linux && x86_64' }
agent { label 'x86_64' }
steps {
script {
def releaseBuild = load ".jenkinsci/release-build.groovy"
Expand All @@ -292,11 +292,8 @@ pipeline {
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
cleanWs()
}
post = load ".jenkinsci/linux-post-step.groovy"
post.linuxPostStep()
}
}
}
Expand All @@ -313,14 +310,11 @@ pipeline {
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
cleanWs()
}
post = load ".jenkinsci/linux-post-step.groovy"
post.linuxPostStep()
}
}
}
}
}
stage('ARMv8') {
when { expression { return params.ARMv8 } }
Expand All @@ -334,44 +328,36 @@ pipeline {
post {
always {
script {
timeout(time: 60, unit: "SECONDS") {
def cleanup = load ".jenkinsci/docker-cleanup.groovy"
cleanup.doDockerCleanup()
cleanWs()
}
post = load ".jenkinsci/linux-post-step.groovy"
post.linuxPostStep()
}
}
}
}
}
stage('MacOS') {
when { expression { return params.MacOS } }
steps {
script {
def scmVars = checkout scm
env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
env.IROHA_HOME = "/opt/iroha"
env.IROHA_BUILD = "${env.IROHA_HOME}/build"

sh """
ccache --version
ccache --show-stats
ccache --zero-stats
ccache --max-size=5G
"""
sh """
cmake \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=${params.BUILD_TYPE} \
-DIROHA_VERSION=${env.IROHA_VERSION}
"""
sh "cmake --build build -- -j${params.PARALLELISM}"
sh "ccache --show-stats"

// TODO: replace with upload to artifactory server
// only develop branch
if ( env.BRANCH_NAME == "develop" ) {
//archive(includes: 'build/bin/,compile_commands.json')
def releaseBuild = load ".jenkinsci/mac-release-build.groovy"
releaseBuild.doReleaseBuild()
}
}
post {
always {
script {
timeout(time: 600, unit: "SECONDS") {
if (BRANCH_NAME ==~ /(master|develop)/) {
try {
def artifacts = load ".jenkinsci/artifacts.groovy"
def commit = env.GIT_COMMIT
filePaths = [ '\$(pwd)/build/*.tar.gz' ]
artifacts.uploadArtifacts(filePaths, sprintf('/iroha/macos/%1$s-%2$s-%3$s', [BRANCH_NAME, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)]))
}
finally {
cleanWs()
}
}
}
}
}
}
Expand Down

0 comments on commit 80bae88

Please sign in to comment.