Skip to content

Commit

Permalink
Merge branch 'develop' into trunk/mst_experimental
Browse files Browse the repository at this point in the history
Signed-off-by: Kitsu <[email protected]>
  • Loading branch information
l4l committed May 1, 2018
2 parents 0b64bb4 + c93d30e commit ee8fd0c
Show file tree
Hide file tree
Showing 86 changed files with 4,201 additions and 3,507 deletions.
79 changes: 63 additions & 16 deletions .jenkinsci/bindings.groovy
Original file line number Diff line number Diff line change
@@ -1,27 +1,74 @@
#!/usr/bin/env groovy

def doBindings() {
def cmake_options = ""
if (params.JavaBindings) {
cmake_options += " -DSWIG_JAVA=ON "
}
if (params.PythonBindings) {
cmake_options += " -DSWIG_PYTHON=ON "
}
// In case language specific options were not set,
// build for each language
if (!params.JavaBindings && !params.PythonBindings) {
cmake_options += " -DSWIG_JAVA=ON -DSWIG_PYTHON=ON "
}
def doJavaBindings(buildType=Release) {
def currentPath = sh(script: "pwd", returnStdout: true).trim()
def commit = env.GIT_COMMIT
def artifactsPath = sprintf('%1$s/java-bindings-%2$s-%3$s-%4$s.zip',
[currentPath, buildType, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
sh """
cmake \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=Release \
${cmake_options}
-DCMAKE_BUILD_TYPE=$buildType \
-DSWIG_JAVA=ON
"""
sh "cd build; make -j${params.PARALLELISM} irohajava"
sh "zip -j $artifactsPath build/shared_model/bindings/*.java build/shared_model/bindings/libirohajava.so"
sh "cp $artifactsPath /tmp/bindings-artifact"
return artifactsPath
}

def doPythonBindings(buildType=Release) {
def currentPath = sh(script: "pwd", returnStdout: true).trim()
def commit = env.GIT_COMMIT
def supportPython2 = "OFF"
def artifactsPath = sprintf('%1$s/python-bindings-%2$s-%3$s-%4$s-%5$s.zip',
[currentPath, env.PBVersion, buildType, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
// do not use preinstalled libed25519
sh "rm -rf /usr/local/include/ed25519*; unlink /usr/local/lib/libed25519.so; rm -f /usr/local/lib/libed25519.so.1.2.2"
if (env.PBVersion == "python2") { supportPython2 = "ON" }
sh """
cmake \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=$buildType \
-DSWIG_PYTHON=ON \
-DSUPPORT_PYTHON2=$supportPython2
"""
sh "cmake --build build --target python_tests"
sh "cd build; make -j${params.PARALLELISM} irohajava irohapy"
sh "cd build; make -j${params.PARALLELISM} irohapy"
sh "protoc --proto_path=schema --python_out=build/shared_model/bindings block.proto primitive.proto commands.proto queries.proto responses.proto endpoint.proto"
sh "${env.PBVersion} -m grpc_tools.protoc --proto_path=schema --python_out=build/shared_model/bindings --grpc_python_out=build/shared_model/bindings endpoint.proto yac.proto ordering.proto loader.proto"
sh "zip -j $artifactsPath build/shared_model/bindings/*.py build/shared_model/bindings/*.so"
sh "cp $artifactsPath /tmp/bindings-artifact"
return artifactsPath
}

def doAndroidBindings(abiVersion) {
def currentPath = sh(script: "pwd", returnStdout: true).trim()
def commit = env.GIT_COMMIT
def artifactsPath = sprintf('%1$s/android-bindings-%2$s-%3$s-%4$s-%5$s-%6$s.zip',
[currentPath, "\$PLATFORM", abiVersion, "\$BUILD_TYPE_A", sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)])
sh """
(cd /iroha; git init; git remote add origin https://github.com/hyperledger/iroha.git; \
git fetch --depth 1 origin develop; git checkout -t origin/develop)
"""
sh """
. /entrypoint.sh; \
sed -i.bak "s~find_package(JNI REQUIRED)~SET(CMAKE_SWIG_FLAGS \\\${CMAKE_SWIG_FLAGS} -package \${PACKAGE})~" /iroha/shared_model/bindings/CMakeLists.txt; \
# TODO: might not be needed in the future
sed -i.bak "/target_include_directories(\\\${SWIG_MODULE_irohajava_REAL_NAME} PUBLIC/,+3d" /iroha/shared_model/bindings/CMakeLists.txt; \
sed -i.bak "s~swig_link_libraries(irohajava~swig_link_libraries(irohajava \"/protobuf/.build/lib\${PROTOBUF_LIB_NAME}.a\" \"\${NDK_PATH}/platforms/android-$abiVersion/\${ARCH}/usr/\${LIBP}/liblog.so\"~" /iroha/shared_model/bindings/CMakeLists.txt; \
sed -i.bak "s~find_library(protobuf_LIBRARY protobuf)~find_library(protobuf_LIBRARY \${PROTOBUF_LIB_NAME})~" /iroha/cmake/Modules/Findprotobuf.cmake; \
sed -i.bak "s~find_program(protoc_EXECUTABLE protoc~set(protoc_EXECUTABLE \"/protobuf/host_build/protoc\"~" /iroha/cmake/Modules/Findprotobuf.cmake; \
cmake -H/iroha/shared_model -B/iroha/shared_model/build -DCMAKE_SYSTEM_NAME=Android -DCMAKE_SYSTEM_VERSION=$abiVersion -DCMAKE_ANDROID_ARCH_ABI=\$PLATFORM \
-DANDROID_NDK=\$NDK_PATH -DCMAKE_ANDROID_STL_TYPE=c++_static -DCMAKE_BUILD_TYPE=\$BUILD_TYPE_A -DTESTING=OFF \
-DSHARED_MODEL_DISABLE_COMPATIBILITY=ON -DSWIG_JAVA=ON -DCMAKE_PREFIX_PATH=\$DEPS_DIR
"""
sh "cmake --build /iroha/shared_model/build --target irohajava -- -j${params.PARALLELISM}"
sh "zip -j $artifactsPath /iroha/shared_model/build/bindings/*.java /iroha/shared_model/build/bindings/libirohajava.so"
sh "cp $artifactsPath /tmp/bindings-artifact"
return artifactsPath
}

return this
125 changes: 63 additions & 62 deletions .jenkinsci/debug-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
def doDebugBuild(coverageEnabled=false) {
def dPullOrBuild = load ".jenkinsci/docker-pull-or-build.groovy"
def parallelism = params.PARALLELISM
def platform = sh(script: 'uname -m', returnStdout: true).trim()
// params are always null unless job is started
// this is the case for the FIRST build only.
// So just set this to same value as default.
Expand All @@ -14,78 +15,78 @@ def doDebugBuild(coverageEnabled=false) {
parallelism = 1
}
sh "docker network create ${env.IROHA_NETWORK}"

docker.image('postgres:9.5').run(""
def iC = dPullOrBuild.dockerPullOrUpdate("${platform}-develop-build",
"${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/develop/Dockerfile",
"${env.GIT_RAW_BASE_URL}/${env.GIT_PREVIOUS_COMMIT}/docker/develop/Dockerfile",
"${env.GIT_RAW_BASE_URL}/develop/docker/develop/Dockerfile",
['PARALLELISM': parallelism])
docker.image('postgres:9.5').withRun(""
+ " -e POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+ " --name ${env.IROHA_POSTGRES_HOST}"
+ " --network=${env.IROHA_NETWORK}")

def iC = dPullOrBuild.dockerPullOrUpdate()
iC.inside(""
+ " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
+ " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
+ " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+ " --network=${env.IROHA_NETWORK}"
+ " -v /var/jenkins/ccache:${CCACHE_DIR}"
+ " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {
+ " --network=${env.IROHA_NETWORK}") {
iC.inside(""
+ " -e IROHA_POSTGRES_HOST=${env.IROHA_POSTGRES_HOST}"
+ " -e IROHA_POSTGRES_PORT=${env.IROHA_POSTGRES_PORT}"
+ " -e IROHA_POSTGRES_USER=${env.IROHA_POSTGRES_USER}"
+ " -e IROHA_POSTGRES_PASSWORD=${env.IROHA_POSTGRES_PASSWORD}"
+ " --network=${env.IROHA_NETWORK}"
+ " -v /var/jenkins/ccache:${CCACHE_DIR}"
+ " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}") {

def scmVars = checkout scm
def cmakeOptions = ""
if ( coverageEnabled ) {
cmakeOptions = " -DCOVERAGE=ON "
}
env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
env.IROHA_HOME = "/opt/iroha"
env.IROHA_BUILD = "${env.IROHA_HOME}/build"

sh """
ccache --version
ccache --show-stats
ccache --zero-stats
ccache --max-size=5G
"""
sh """
cmake \
-DTESTING=ON \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=Debug \
-DIROHA_VERSION=${env.IROHA_VERSION} \
${cmakeOptions}
"""
sh "cmake --build build -- -j${parallelism}"
sh "ccache --show-stats"
if ( coverageEnabled ) {
sh "cmake --build build --target coverage.init.info"
}
def testExitCode = sh(script: 'cmake --build build --target test', returnStatus: true)
if (testExitCode != 0) {
currentBuild.result = "UNSTABLE"
}
if ( coverageEnabled ) {
sh "cmake --build build --target cppcheck"
// Sonar
if (env.CHANGE_ID != null) {
sh """
sonar-scanner \
-Dsonar.github.disableInlineComments \
-Dsonar.github.repository='hyperledger/iroha' \
-Dsonar.analysis.mode=preview \
-Dsonar.login=${SONAR_TOKEN} \
-Dsonar.projectVersion=${BUILD_TAG} \
-Dsonar.github.oauth=${SORABOT_TOKEN} \
-Dsonar.github.pullRequest=${CHANGE_ID}
"""
def scmVars = checkout scm
def cmakeOptions = ""
if ( coverageEnabled ) {
cmakeOptions = " -DCOVERAGE=ON "
}
env.IROHA_VERSION = "0x${scmVars.GIT_COMMIT}"
env.IROHA_HOME = "/opt/iroha"
env.IROHA_BUILD = "${env.IROHA_HOME}/build"

sh """
ccache --version
ccache --show-stats
ccache --zero-stats
ccache --max-size=5G
"""
sh """
cmake \
-DTESTING=ON \
-H. \
-Bbuild \
-DCMAKE_BUILD_TYPE=Debug \
-DIROHA_VERSION=${env.IROHA_VERSION} \
${cmakeOptions}
"""
sh "cmake --build build -- -j${parallelism}"
sh "ccache --show-stats"
if ( coverageEnabled ) {
sh "cmake --build build --target coverage.init.info"
}
def testExitCode = sh(script: 'CTEST_OUTPUT_ON_FAILURE=1 cmake --build build --target test', returnStatus: true)
if (testExitCode != 0) {
currentBuild.result = "UNSTABLE"
}
if ( coverageEnabled ) {
sh "cmake --build build --target cppcheck"
// Sonar
if (env.CHANGE_ID != null) {
sh """
sonar-scanner \
-Dsonar.github.disableInlineComments \
-Dsonar.github.repository='hyperledger/iroha' \
-Dsonar.analysis.mode=preview \
-Dsonar.login=${SONAR_TOKEN} \
-Dsonar.projectVersion=${BUILD_TAG} \
-Dsonar.github.oauth=${SORABOT_TOKEN} \
-Dsonar.github.pullRequest=${CHANGE_ID}
"""
}
}
sh "cmake --build build --target coverage.info"
sh "python /tmp/lcov_cobertura.py build/reports/coverage.info -o build/reports/coverage.xml"
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: '**/build/reports/coverage.xml', conditionalCoverageTargets: '75, 50, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '75, 50, 0', maxNumberOfBuilds: 50, methodCoverageTargets: '75, 50, 0', onlyStable: false, zoomCoverageChart: false
}
// copy built binaries to the volume
sh "cp ./build/bin/* /tmp/${GIT_COMMIT}/"
}
}

Expand Down
64 changes: 39 additions & 25 deletions .jenkinsci/docker-pull-or-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -4,43 +4,57 @@ def remoteFilesDiffer(f1, f2) {
sh "curl -L -o /tmp/${env.GIT_COMMIT}/f1 --create-dirs ${f1}"
sh "curl -L -o /tmp/${env.GIT_COMMIT}/f2 ${f2}"
diffExitCode = sh(script: "diff -q /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}/f2", returnStatus: true)
if (diffExitCode == 0) {
return false
}
return true
return diffExitCode != 0
}

def dockerPullOrUpdate() {
def platform = sh(script: 'uname -m', returnStdout: true).trim()
def commit = sh(script: "echo ${BRANCH_NAME} | md5sum | cut -c 1-8", returnStdout: true).trim()
if (remoteFilesDiffer("https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_COMMIT}/docker/develop/${platform}/Dockerfile",
"https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_PREVIOUS_COMMIT}/docker/develop/${platform}/Dockerfile")) {
iC = docker.build("hyperledger/iroha:${commit}", "--build-arg PARALLELISM=${parallelism} -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
// develop branch Docker image has been modified
if (BRANCH_NAME == 'develop') {
docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
iC.push("${platform}-develop")
}
def buildOptionsString(options) {
def s = ''
if (options) {
options.each { k, v ->
s += "--build-arg ${k}=${v} "
}
}
return s
}

def dockerPullOrUpdate(imageName, currentDockerfileURL, previousDockerfileURL, referenceDockerfileURL, buildOptions=null) {
buildOptions = buildOptionsString(buildOptions)
// GIT_PREVIOUS_COMMIT is null for first PR build
if (!previousDockerfileURL) {
previousDockerfileURL = currentDockerfileURL
}
def commit = sh(script: "echo ${GIT_LOCAL_BRANCH} | md5sum | cut -c 1-8", returnStdout: true).trim()
if (remoteFilesDiffer(currentDockerfileURL, previousDockerfileURL)) {
// Dockerfile has been changed compared to the previous commit
// Worst case scenario. We cannot count on the local cache
// because Dockerfile may contain apt-get entries that would try to update
// from invalid (stale) addresses
iC = docker.build("hyperledger/iroha:${commit}-${BUILD_NUMBER}", "${buildOptions} --no-cache -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
}
else {
// reuse develop branch Docker image
if (BRANCH_NAME == 'develop') {
iC = docker.image("hyperledger/iroha:${platform}-develop")
iC.pull()
// first commit in this branch or Dockerfile modified
if (remoteFilesDiffer(currentDockerfileURL, referenceDockerfileURL)) {
// if we're lucky to build on the same agent, image will be built using cache
iC = docker.build("hyperledger/iroha:${commit}-${BUILD_NUMBER}", "$buildOptions -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
}
else {
// first commit in this branch or Dockerfile modified
if (remoteFilesDiffer("https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_COMMIT}/docker/develop/${platform}/Dockerfile",
"https://raw.githubusercontent.com/hyperledger/iroha/develop/docker/develop/${platform}/Dockerfile")) {
iC = docker.build("hyperledger/iroha:${commit}", "--build-arg PARALLELISM=${parallelism} -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
// try pulling image from Dockerhub, probably image is already there
def testExitCode = sh(script: "docker pull hyperledger/iroha:${imageName}", returnStatus: true)
if (testExitCode != 0) {
// image does not (yet) exist on Dockerhub. Build it
iC = docker.build("hyperledger/iroha:${commit}-${BUILD_NUMBER}", "$buildOptions --no-cache -f /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}")
}
// reuse develop branch Docker image
else {
iC = docker.image("hyperledger/iroha:${platform}-develop")
// no difference found compared to both previous and reference Dockerfile
iC = docker.image("hyperledger/iroha:${imageName}")
}
}
}
if (GIT_LOCAL_BRANCH ==~ /develop|master/) {
docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
iC.push(imageName)
}
}
return iC
}

Expand Down
4 changes: 2 additions & 2 deletions .jenkinsci/linux-post-step.groovy
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
def linuxPostStep() {
timeout(time: 600, unit: "SECONDS") {
try {
if (currentBuild.currentResult == "SUCCESS" && BRANCH_NAME ==~ /(master|develop)/) {
if (currentBuild.currentResult == "SUCCESS" && GIT_LOCAL_BRANCH ==~ /(master|develop)/) {
def artifacts = load ".jenkinsci/artifacts.groovy"
def commit = env.GIT_COMMIT
def platform = sh(script: 'uname -m', returnStdout: true).trim()
filePaths = [ '/tmp/${GIT_COMMIT}-${BUILD_NUMBER}/*' ]
artifacts.uploadArtifacts(filePaths, sprintf('/iroha/linux/%4$s/%1$s-%2$s-%3$s', [BRANCH_NAME, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6), platform]))
artifacts.uploadArtifacts(filePaths, sprintf('/iroha/linux/%4$s/%1$s-%2$s-%3$s', [GIT_LOCAL_BRANCH, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6), platform]))
}
}
finally {
Expand Down
3 changes: 2 additions & 1 deletion .jenkinsci/mac-release-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ def doReleaseBuild(coverageEnabled=false) {
-DPACKAGE_TGZ=ON \
-DCMAKE_BUILD_TYPE=Release \
-DIROHA_VERSION=${env.IROHA_VERSION}
cmake --build build --target package -- -j${params.PARALLELISM}
mv ./build/iroha-${env.IROHA_VERSION}-*.tar.gz ./build/iroha.tar.gz
ccache --show-stats
"""
}
Expand Down
19 changes: 10 additions & 9 deletions .jenkinsci/release-build.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def doReleaseBuild() {
}
def platform = sh(script: 'uname -m', returnStdout: true).trim()
sh "mkdir /tmp/${env.GIT_COMMIT}-${BUILD_NUMBER} || true"
iC = docker.image("hyperledger/iroha:${platform}-develop")
iC = docker.image("hyperledger/iroha:${platform}-develop-build")
iC.pull()
iC.inside(""
+ " -v /tmp/${GIT_COMMIT}-${BUILD_NUMBER}:/tmp/${GIT_COMMIT}"
Expand Down Expand Up @@ -45,20 +45,21 @@ def doReleaseBuild() {
sh "cmake --build build --target package -- -j${parallelism}"
sh "ccache --show-stats"

// copy build package to the volume
sh "cp ./build/iroha-*.deb /tmp/${GIT_COMMIT}/iroha.deb"
// move build package to the volume
sh "mv ./build/iroha-*.deb /tmp/${GIT_COMMIT}/iroha.deb"
sh "mv ./build/*.tar.gz /tmp/${GIT_COMMIT}/iroha.tar.gz"
}

sh "curl -L -o /tmp/${env.GIT_COMMIT}/Dockerfile --create-dirs https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_COMMIT}/docker/release/${platform}/Dockerfile"
sh "curl -L -o /tmp/${env.GIT_COMMIT}/entrypoint.sh https://raw.githubusercontent.com/hyperledger/iroha/${env.GIT_COMMIT}/docker/release/${platform}/entrypoint.sh"
sh "cp /tmp/${GIT_COMMIT}-${BUILD_NUMBER}/iroha.deb /tmp/${env.GIT_COMMIT}"
sh "curl -L -o /tmp/${env.GIT_COMMIT}/Dockerfile --create-dirs ${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/release/Dockerfile"
sh "curl -L -o /tmp/${env.GIT_COMMIT}/entrypoint.sh ${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/release/entrypoint.sh"
sh "mv /tmp/${GIT_COMMIT}-${BUILD_NUMBER}/iroha.deb /tmp/${env.GIT_COMMIT}"
sh "chmod +x /tmp/${env.GIT_COMMIT}/entrypoint.sh"
iCRelease = docker.build("hyperledger/iroha:${GIT_COMMIT}-${BUILD_NUMBER}-release", "--no-cache -f /tmp/${env.GIT_COMMIT}/Dockerfile /tmp/${env.GIT_COMMIT}")
docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
if (env.BRANCH_NAME == 'develop') {
iCRelease.push("${platform}-develop-latest")
if (env.GIT_LOCAL_BRANCH == 'develop') {
iCRelease.push("${platform}-develop")
}
else if (env.BRANCH_NAME == 'master') {
else if (env.GIT_LOCAL_BRANCH == 'master') {
iCRelease.push("${platform}-latest")
}
}
Expand Down
10 changes: 10 additions & 0 deletions .jenkinsci/remote-files-differ.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env groovy

def remoteFilesDiffer(f1, f2) {
sh "curl -sSL -o /tmp/${env.GIT_COMMIT}/f1 --create-dirs ${f1}"
sh "curl -sSL -o /tmp/${env.GIT_COMMIT}/f2 ${f2}"
diffExitCode = sh(script: "diff -q /tmp/${env.GIT_COMMIT}/f1 /tmp/${env.GIT_COMMIT}/f2", returnStatus: true)
return diffExitCode != 0
}

return this
Loading

0 comments on commit ee8fd0c

Please sign in to comment.