Jenkinsfile fixes (hyperledger-iroha#1103)

* Jenkinsfile: fix PARALLELISM param always null on first build run, macos ccache fix, macos relative path binaries, post step timeout Signed-off-by: Artyom Bakhtin <[email protected]> * fix Docker release dependencies installation Signed-off-by: Artyom Bakhtin <[email protected]>
bcolucci · Mar 30, 2018 · 26a56b2 · 26a56b2
1 parent 7cc66ba
commit 26a56b2
Show file tree

Hide file tree

Showing 7 changed files with 116 additions and 75 deletions.
diff --git a/.jenkinsci/bindings.groovy b/.jenkinsci/bindings.groovy
@@ -1,7 +1,6 @@
 #!/usr/bin/env groovy
 
 def doBindings() {
-
   def cmake_options = ""
   if (params.JavaBindings) {
     cmake_options += " -DSWIG_JAVA=ON "
@@ -10,7 +9,7 @@ def doBindings() {
     cmake_options += " -DSWIG_PYTHON=ON "
   }
   // In case language specific options were not set,
-  // build for every language
+  // build for each language
   if (!params.JavaBindings && !params.PythonBindings) {
     cmake_options += " -DSWIG_JAVA=ON -DSWIG_PYTHON=ON "
   }
@@ -21,8 +20,8 @@ def doBindings() {
       -DCMAKE_BUILD_TYPE=Release \
       ${cmake_options}
   """
+  sh "cmake --build build --target python_tests"
   sh "cd build; make -j${params.PARALLELISM} irohajava irohapy"
-  archive(includes: 'build/shared_model/bindings/')
 }
 
 return this
diff --git a/.jenkinsci/debug-build.groovy b/.jenkinsci/debug-build.groovy
@@ -1,7 +1,14 @@
 #!/usr/bin/env groovy
 
 def doDebugBuild(coverageEnabled=false) {
-  def parallelism = env.PARALLELISM
+  def parallelism = params.PARALLELISM
+  // params are always null unless job is started
+  // this is the case for the FIRST build only.
+  // So just set this to same value as default. 
+  // This is a known bug. See https://issues.jenkins-ci.org/browse/JENKINS-41929
+  if (parallelism == null) {
+    parallelism = 4
+  }
   if ("arm7" in env.NODE_NAME) {
     parallelism = 1
   }
@@ -16,7 +23,7 @@ def doDebugBuild(coverageEnabled=false) {
   def platform = sh(script: 'uname -m', returnStdout: true).trim()
   sh "curl -L -o /tmp/${env.GIT_COMMIT}/Dockerfile --create-dirs https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_COMMIT}/docker/develop/${platform}/Dockerfile"
   // pull docker image in case we don't have one
-  // speeds up consequent image builds as we simply tag them 
+  // speeds up consequent image builds as we simply tag them
   sh "docker pull ${DOCKER_BASE_IMAGE_DEVELOP}"
   if (env.BRANCH_NAME == 'develop') {
     iC = docker.build("hyperledger/iroha:${GIT_COMMIT}-${BUILD_NUMBER}", "--build-arg PARALLELISM=${parallelism} -f /tmp/${env.GIT_COMMIT}/Dockerfile /tmp/${env.GIT_COMMIT}")
@@ -36,6 +43,10 @@ def doDebugBuild(coverageEnabled=false) {
     + " -v /var/jenkins/ccache:${CCACHE_DIR}") {
 
     def scmVars = checkout scm
+    def cmakeOptions = ""
+    if ( coverageEnabled ) {
+      cmakeOptions = " -DCOVERAGE=ON "
+    }
     env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
     env.IROHA_HOME = "/opt/iroha"
     env.IROHA_BUILD = "${env.IROHA_HOME}/build"
@@ -48,19 +59,24 @@ def doDebugBuild(coverageEnabled=false) {
     """  
     sh """
       cmake \
-        -DCOVERAGE=ON \
         -DTESTING=ON \
         -H. \
         -Bbuild \
         -DCMAKE_BUILD_TYPE=Debug \
-        -DIROHA_VERSION=${env.IROHA_VERSION}
+        -DIROHA_VERSION=${env.IROHA_VERSION} \
+        ${cmakeOptions}
     """
     sh "cmake --build build -- -j${parallelism}"
     sh "ccache --show-stats"
-    sh "cmake --build build --target test"
-    sh "cmake --build build --target cppcheck"
-
     if ( coverageEnabled ) {
+      sh "lcov -i --capture --directory build --config-file .lcovrc --output-file build/reports/coverage.init.info"
+    }
+    def testExitCode = sh(script: 'cmake --build build --target test', returnStatus: true)
+    if (testExitCode != 0) {
+      currentBuild.result = "UNSTABLE"
+    }
+    if ( coverageEnabled ) {
+      sh "cmake --build build --target cppcheck"
       // Sonar
       if (env.CHANGE_ID != null) {
         sh """
@@ -75,9 +91,10 @@ def doDebugBuild(coverageEnabled=false) {
         """
       }
 
-      sh "lcov --capture --directory build --config-file .lcovrc --output-file build/reports/coverage_full.info"
-      sh "lcov --remove build/reports/coverage_full.info '/usr/*' 'schema/*' --config-file .lcovrc -o build/reports/coverage_full_filtered.info"
-      sh "python /tmp/lcov_cobertura.py build/reports/coverage_full_filtered.info -o build/reports/coverage.xml"                
+      sh "lcov --capture --directory build --config-file .lcovrc --output-file build/reports/coverage.info"
+      sh "lcov -a build/reports/coverage.init.info -a build/reports/coverage.info --config-file .lcovrc -o build/reports/coverage.info"
+      sh "lcov --remove build/reports/coverage.info 'external/*' '/usr/*' 'schema/*' --config-file .lcovrc -o build/reports/coverage.info"
+      sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
       cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
     }
 

diff --git a/.jenkinsci/docker-cleanup.groovy b/.jenkinsci/docker-cleanup.groovy
@@ -3,8 +3,7 @@
 def doDockerCleanup() {
 
     sh """
-      docker stop $IROHA_POSTGRES_HOST || true
-      docker rm $IROHA_POSTGRES_HOST || true
+      docker rm -f $IROHA_POSTGRES_HOST || true
       # Check whether the image is the last-standing man
       # i.e., no other tags exist for this image
       docker rmi \$(docker images --no-trunc --format '{{.Repository}}:{{.Tag}}\\t{{.ID}}' | grep \$(docker images --no-trunc --format '{{.ID}}' ${iC.id}) | head -n -1 | cut -f 1) || true

diff --git a/.jenkinsci/release-build.groovy b/.jenkinsci/release-build.groovy
@@ -1,7 +1,14 @@
 #!/usr/bin/env groovy
 
 def doReleaseBuild() {
-  def parallelism = env.PARALLELISM
+  def parallelism = params.PARALLELISM
+  // params are always null unless job is started
+  // this is the case for the FIRST build only.
+  // So just set this to same value as default. 
+  // This is a known bug. See https://issues.jenkins-ci.org/browse/JENKINS-41929
+  if (parallelism == null) {
+    parallelism = 4
+  }
   if ("arm7" in env.NODE_NAME) {
     parallelism = 1
   }

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -126,4 +126,3 @@ if (FUZZING)
 endif()
 
 
-
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,15 +1,15 @@
 // Overall pipeline looks like the following
 //         
 //   |--Linux-----|----Debug
-//   |      |----Release 
-//   |  OR
-//   |       
+//   |            |----Release 
+//   |    OR
+//   |           
 //-- |--Linux ARM-|----Debug
-//   |      |----Release
-//   |  OR
+//   |            |----Release
+//   |    OR
 //   |
 //   |--MacOS-----|----Debug
-//   |      |----Release
+//   |            |----Release
 properties([parameters([
   choice(choices: 'Debug\nRelease', description: '', name: 'BUILD_TYPE'),
   booleanParam(defaultValue: true, description: '', name: 'Linux'),
@@ -81,9 +81,11 @@ pipeline {
           post {
             always {
               script {
-                def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-                cleanup.doDockerCleanup()
-                cleanWs()
+                timeout(time: 60, unit: "SECONDS") {
+                  def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+                  cleanup.doDockerCleanup()
+                  cleanWs()
+                }
               }
             }
           }
@@ -109,9 +111,11 @@ pipeline {
           post {
             always {
               script {
-                def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-                cleanup.doDockerCleanup()
-                cleanWs()
+                timeout(time: 60, unit: "SECONDS") {
+                  def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+                  cleanup.doDockerCleanup()
+                  cleanWs()
+                }
               }
             }
           }
@@ -137,9 +141,11 @@ pipeline {
           post {
             always {
               script {
-                def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-                cleanup.doDockerCleanup()
-                cleanWs()
+                timeout(time: 60, unit: "SECONDS") {
+                  def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+                  cleanup.doDockerCleanup()
+                  cleanWs()
+                }
               }
             }
           }
@@ -150,47 +156,54 @@ pipeline {
           steps {
             script {
               def coverageEnabled = false
+              def cmakeOptions = ""
               if (!params.Linux) {
                 coverageEnabled = true
+                cmakeOptions = " -DCOVERAGE=ON "
               }
               def scmVars = checkout scm
               env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
               env.IROHA_HOME = "/opt/iroha"
               env.IROHA_BUILD = "${env.IROHA_HOME}/build"
 
               sh """
-                /usr/local/bin/ccache --version
-                /usr/local/bin/ccache --show-stats
-                /usr/local/bin/ccache --zero-stats
-                /usr/local/bin/ccache --max-size=5G
+                ccache --version
+                ccache --show-stats
+                ccache --zero-stats
+                ccache --max-size=5G
               """
               sh """
-                /usr/local/bin/cmake \
-                  -DCOVERAGE=ON \
+                cmake \
                   -DTESTING=ON \
                   -H. \
                   -Bbuild \
                   -DCMAKE_BUILD_TYPE=${params.BUILD_TYPE} \
-                  -DIROHA_VERSION=${env.IROHA_VERSION}
+                  -DIROHA_VERSION=${env.IROHA_VERSION} \
+                  ${cmakeOptions}
               """
-              sh "/usr/local/bin/cmake --build build -- -j${params.PARALLELISM}"
-              sh "/usr/local/bin/ccache --show-stats"
+              sh "cmake --build build -- -j${params.PARALLELISM}"
+              sh "ccache --show-stats"
+              if ( coverageEnabled ) {
+                sh "lcov -i --capture --directory build --config-file .lcovrc --output-file build/reports/coverage.init.info"
+              }
               sh """
                 export IROHA_POSTGRES_PASSWORD=${IROHA_POSTGRES_PASSWORD}; \
                 export IROHA_POSTGRES_USER=${IROHA_POSTGRES_USER}; \
                 mkdir -p /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}; \
-                /usr/local/bin/initdb -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ -U ${IROHA_POSTGRES_USER} --pwfile=<(echo ${IROHA_POSTGRES_PASSWORD}); \
-                /usr/local/bin/pg_ctl -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ -o '-p 5433' -l /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/events.log start; \
-                /usr/local/bin/psql -h localhost -d postgres -p 5433 -U ${IROHA_POSTGRES_USER} --file=<(echo create database ${IROHA_POSTGRES_USER};)
+                initdb -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ -U ${IROHA_POSTGRES_USER} --pwfile=<(echo ${IROHA_POSTGRES_PASSWORD}); \
+                pg_ctl -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ -o '-p 5433' -l /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/events.log start; \
+                psql -h localhost -d postgres -p 5433 -U ${IROHA_POSTGRES_USER} --file=<(echo create database ${IROHA_POSTGRES_USER};)
               """
-              sh "IROHA_POSTGRES_HOST=localhost IROHA_POSTGRES_PORT=5433 /usr/local/bin/cmake --build build --target test"
-              sh "/usr/local/bin/cmake --build build --target cppcheck"
-
+              def testExitCode = sh(script: 'IROHA_POSTGRES_HOST=localhost IROHA_POSTGRES_PORT=5433 cmake --build build --target test', returnStatus: true)
+              if (testExitCode != 0) {
+                currentBuild.result = "UNSTABLE"
+              }
               if ( coverageEnabled ) {
+                sh "cmake --build build --target cppcheck"
                 // Sonar
                 if (env.CHANGE_ID != null) {
                   sh """
-                    /usr/local/bin/sonar-scanner \
+                    sonar-scanner \
                       -Dsonar.github.disableInlineComments \
                       -Dsonar.github.repository='hyperledger/iroha' \
                       -Dsonar.analysis.mode=preview \
@@ -199,11 +212,11 @@ pipeline {
                       -Dsonar.github.oauth=${SORABOT_TOKEN}
                   """
                 }
-                sh "/usr/local/bin/lcov --capture --directory build --config-file .lcovrc --output-file build/reports/coverage_full.info"
-                sh "/usr/local/bin/lcov --remove build/reports/coverage_full.info '/usr/*' 'schema/*' --config-file .lcovrc -o build/reports/coverage_full_filtered.info"
-                sh "python /usr/local/bin/lcov_cobertura.py build/reports/coverage_full_filtered.info -o build/reports/coverage.xml"                
+                sh "lcov --capture --directory build --config-file .lcovrc --output-file build/reports/coverage.info"
+                sh "lcov -a build/reports/coverage.init.info -a build/reports/coverage.info --config-file .lcovrc -o build/reports/coverage.info"
+                sh "lcov --remove build/reports/coverage.info 'external/*' '/usr/*' 'schema/*' --config-file .lcovrc -o build/reports/coverage.info"
+                sh "python /usr/local/bin/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
                 cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
-
               }
 
               // TODO: replace with upload to artifactory server
@@ -216,11 +229,13 @@ pipeline {
           post {
             always {
               script {
-                cleanWs()
-                sh """
-                  /usr/local/bin/pg_ctl -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ stop && \
-                  rm -rf /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/
-                """
+                timeout(time: 60, unit: "SECONDS") {
+                  cleanWs()
+                  sh """
+                    pg_ctl -D /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/ stop && \
+                    rm -rf /var/jenkins/${GIT_COMMIT}-${BUILD_NUMBER}/
+                  """
+                }
               }
             }
           }
@@ -247,9 +262,11 @@ pipeline {
           post {
             always {
               script {
-                def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-                cleanup.doDockerCleanup()
-                cleanWs()
+                timeout(time: 60, unit: "SECONDS") {
+                  def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+                  cleanup.doDockerCleanup()
+                  cleanWs()
+                }
               }
             }
           }
@@ -266,9 +283,11 @@ pipeline {
           post {
             always {
               script {
-                def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-                cleanup.doDockerCleanup()
-                cleanWs()
+                timeout(time: 60, unit: "SECONDS") {
+                  def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+                  cleanup.doDockerCleanup()
+                  cleanWs()
+                }
               }
             }
           }            
@@ -285,9 +304,11 @@ pipeline {
           post {
             always {
               script {
-                def cleanup = load ".jenkinsci/docker-cleanup.groovy"
-                cleanup.doDockerCleanup()
-                cleanWs()
+                timeout(time: 60, unit: "SECONDS") {
+                  def cleanup = load ".jenkinsci/docker-cleanup.groovy"
+                  cleanup.doDockerCleanup()
+                  cleanWs()
+                }
               }
             }
           }            
@@ -300,23 +321,22 @@ pipeline {
               env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
               env.IROHA_HOME = "/opt/iroha"
               env.IROHA_BUILD = "${env.IROHA_HOME}/build"
-              env.CCACHE_DIR = "${env.IROHA_HOME}/.ccache"
 
               sh """
-                /usr/local/bin/ccache --version
-                /usr/local/bin/ccache --show-stats
-                /usr/local/bin/ccache --zero-stats
-                /usr/local/bin/ccache --max-size=5G
+                ccache --version
+                ccache --show-stats
+                ccache --zero-stats
+                ccache --max-size=5G
               """  
               sh """
-                /usr/local/bin/cmake \
+                cmake \
                   -H. \
                   -Bbuild \
                   -DCMAKE_BUILD_TYPE=${params.BUILD_TYPE} \
                   -DIROHA_VERSION=${env.IROHA_VERSION}
               """
-              sh "/usr/local/bin/cmake --build build -- -j${params.PARALLELISM}"
-              sh "/usr/local/bin/ccache --show-stats"
+              sh "cmake --build build -- -j${params.PARALLELISM}"
+              sh "ccache --show-stats"
 
               // TODO: replace with upload to artifactory server
               // only develop branch

diff --git a/docker/release/x86_64/Dockerfile b/docker/release/x86_64/Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update;\
 
 #Install iroha
 COPY iroha.deb /tmp/iroha.deb
-RUN dpkg -i /tmp/iroha.deb; rm -f /tmp/iroha.deb
+RUN apt-get install -y /tmp/iroha.deb; rm -f /tmp/iroha.deb
 
 WORKDIR /opt/iroha_data