reproducible builds CI integration (cosmos#4337)

Enable reproducible builds on master.

Overhaul build script, update docs accordingly.

.circleci/config.yml

@@ -370,6 +370,31 @@ jobs:
           docker login --password-stdin -u $DOCKER_USER <<<$DOCKER_PASS
           docker push tendermint/gaia:$CIRCLE_TAG
 
+  reproducible_builds:
+    <<: *linux_defaults
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Build gaia
+          no_output_timeout: 20m
+          command: |
+            sudo apt-get install -y ruby
+            bash -x ./cmd/gaia/contrib/gitian-build.sh all
+            for os in darwin linux windows; do
+              cp gitian-build-${os}/result/gaia-${os}-res.yml .
+              rm -rf gitian-build-${os}/
+            done
+      - store_artifacts:
+          path: /go/src/github.com/cosmos/cosmos-sdk/gaia-darwin-res.yml
+      - store_artifacts:
+          path: /go/src/github.com/cosmos/cosmos-sdk/gaia-linux-res.yml
+      - store_artifacts:
+          path: /go/src/github.com/cosmos/cosmos-sdk/gaia-windows-res.yml
+
 workflows:
   version: 2
   test-suite:
@@ -441,3 +466,10 @@ workflows:
       - upload_coverage:
           requires:
             - test_cover
+      - reproducible_builds:
+          filters:
+            branches:
+              only:
+                - master
+          requires:
+            - setup_dependencies

cmd/gaia/contrib/gitian-build.sh

@@ -7,18 +7,18 @@
 
 set -euo pipefail
 
-THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-THIS="${THIS_DIR}/$(basename ${BASH_SOURCE[0]})"
 GITIAN_CACHE_DIRNAME='.gitian-builder-cache'
 GO_DEBIAN_RELEASE='1.12.5-1'
 GO_TARBALL="golang-debian-${GO_DEBIAN_RELEASE}.tar.gz"
+GO_TARBALL_URL="https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}/${GO_TARBALL}"
 
 # Defaults
 
 DEFAULT_SIGN_COMMAND='gpg --detach-sign'
 DEFAULT_GAIA_SIGS=${GAIA_SIGS:-'gaia.sigs'}
 DEFAULT_GITIAN_REPO='https://github.com/devrandom/gitian-builder'
 DEFAULT_GBUILD_FLAGS=''
+DEFAULT_SIGS_REPO='https://github.com/cosmos/gaia.sigs'
 
 # Overrides
 
@@ -30,187 +30,172 @@ GBUILD_FLAGS=${GBUILD_FLAGS:-${DEFAULT_GBUILD_FLAGS}}
 
 g_workdir=''
 g_gitian_cache=''
+g_cached_gitian=''
+g_cached_go_tarball=''
 g_sign_identity=''
-g_gitian_skip_download=''
-
-f_main() {
-  local l_dirname \
-    l_sdk \
-    l_commit \
-    l_platform \
-    l_result \
-    l_descriptor \
-    l_release \
-    l_sigs_dir
-
-  l_platform=$1
-  l_sdk=$2
-  l_sigs_dir=$3
-
-  pushd ${l_sdk}
-  l_commit="$(git rev-parse HEAD)"
-  l_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${l_platform}"
-  popd
+g_sigs_dir=''
+g_flag_commit=''
 
-  l_descriptor=${THIS_DIR}/gitian-descriptors/gitian-${l_platform}.yml
-  [ -f ${l_descriptor} ]
 
-  if [ "${g_gitian_skip_download}" != "y" ]; then
-    echo "Cloning ${GITIAN_REPO} to ${g_workdir}" >&2
-    git clone ${GITIAN_REPO} ${g_workdir}
-  fi
+f_help() {
+  cat >&2 <<EOF
+Usage: $(basename $0) [-h] PLATFORM
+Launch a gitian build from the current source directory for the given PLATFORM.
+The following platforms are supported:
+  darwin
+  linux
+  windows
+  all
 
-  echo "Fetching Go sources" >&2
-  f_ensure_go_source_tarball
+  Options:
+   -h               display this help and exit
+   -c               clone the signatures repository and commit signatures;
+                    ignored if sign identity is not supplied
+   -s IDENTITY      sign build as IDENTITY
 
-  echo "Prepare gitian-target docker image" >&2
-  f_prep_docker_image
+If a GPG identity is supplied via the -s flag, the build will be signed and verified.
+The signature will be saved in '${DEFAULT_GAIA_SIGS}/'. An alternative output directory
+for signatures can be supplied via the environment variable \$GAIA_SIGS.
 
-  echo "Start the build" >&2
-  f_build "${l_descriptor}" "${l_commit}"
-  echo "You may find the result in $(echo ${g_workdir}/result/*.yml)" >&2
+The default signing command used to sign the build is '$DEFAULT_SIGN_COMMAND'.
+An alternative signing command can be supplied via the environment
+variable \$SIGN_COMMAND.
+EOF
+}
 
-  if [ -n "${g_sign_identity}" ]; then
-    f_sign "${l_descriptor}" "${l_release}" "${l_sigs_dir}"
-    echo "Build signed as ${g_sign_identity}, signatures can be found in ${l_sigs_dir}"
-    f_verify "${l_descriptor}" "${l_release}" "${l_sigs_dir}"
-    echo "Signatures in ${l_sigs_dir} have been verified"
-  else
-    echo "You can now sign the build with the following command:" >&2
-    echo "cd ${g_workdir} ; bin/gsign -p 'gpg --detach-sign' -s GPG_IDENTITY --release=${l_release} ${l_descriptor}" >&2
-  fi
 
-  return 0
+f_builddir() {
+  printf '%s' "${g_workdir}/gitian-build-$1"
 }
 
-f_prep_docker_image() {
-  pushd ${g_workdir}
-  bin/make-base-vm --docker --suite bionic --arch amd64
-  popd
-}
+f_prep_build() {
+  local l_platforms \
+    l_os \
+    l_builddir
 
-f_ensure_go_source_tarball() {
-  local l_cached_tar
-
-  l_cached_tar="${g_gitian_cache}/${GO_TARBALL}"
-  mkdir -p ${g_workdir}/inputs
-  if [ -f "${l_cached_tar}" ]; then
-    echo "Fetching cached tarball from ${l_cached_tar}" >&2
-    cp "${l_cached_tar}" ${g_workdir}/inputs/${GO_TARBALL}
-  else
-    f_download_go
-    echo "Caching ${GO_TARBALL}" >&2
-    cp ${g_workdir}/inputs/${GO_TARBALL} "${l_cached_tar}"
+  l_platforms="$1"
+
+  if [ -n "${g_flag_commit}" -a ! -d "${g_sigs_dir}" ]; then
+    git clone ${DEFAULT_SIGS_REPO} "${g_sigs_dir}"
   fi
-}
 
-f_download_go() {
-  local l_remote
+  for l_os in ${l_platforms}; do
+    l_builddir="$(f_builddir ${l_os})"
 
-  l_remote=https://salsa.debian.org/go-team/compiler/golang/-/archive/debian/${GO_DEBIAN_RELEASE}
-  curl -L "${l_remote}/${GO_TARBALL}" > ${g_workdir}/inputs/${GO_TARBALL}
+    f_echo_stderr "Preparing build directory $(basename ${l_builddir}), restoring files from cache"
+    cp -ar "${g_cached_gitian}" "${l_builddir}" >&2
+    mkdir "${l_builddir}/inputs/"
+    cp -v "${g_cached_go_tarball}" "${l_builddir}/inputs/"
+  done
 }
 
 f_build() {
-  local l_sdk l_descriptor
+  local l_descriptor
 
   l_descriptor=$1
-  l_commit=$2
 
-  [ -f ${l_descriptor} ]
-
-  cd ${g_workdir}
-  export USE_DOCKER=1
-  bin/gbuild --commit cosmos-sdk="$l_commit" ${GBUILD_FLAGS} "$l_descriptor"
-  libexec/stop-target || echo "warning: couldn't stop target" >&2
+  bin/gbuild --commit cosmos-sdk="$g_commit" ${GBUILD_FLAGS} "$l_descriptor"
+  libexec/stop-target || f_echo_stderr "warning: couldn't stop target"
 }
 
-f_sign() {
-  local l_descriptor l_release_name l_sigs_dir
+f_sign_verify() {
+  local l_descriptor
 
   l_descriptor=$1
-  l_release_name=$2
-  l_sigs_dir=$3
 
-  pushd ${g_workdir}
-  bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${l_sigs_dir}" --release=${l_release_name} ${l_descriptor}
-  popd
+  bin/gsign -p "${SIGN_COMMAND}" -s "${g_sign_identity}" --destination="${g_sigs_dir}" --release=${g_release} ${l_descriptor}
+  bin/gverify --destination="${g_sigs_dir}" --release="${g_release}" ${l_descriptor}
 }
 
-f_verify() {
-  local l_descriptor l_release_name l_sigs_dir
+f_commit_sig() {
+  local l_release_name
 
-  l_descriptor=$1
-  l_release_name=$2
-  l_sigs_dir=$3
+  l_release_name=$1
 
-  pushd ${g_workdir}
-  bin/gverify --destination="${l_sigs_dir}" --release="${l_release_name}" ${l_descriptor}
+  pushd "${g_sigs_dir}"
+  git add . || echo "git add failed" >&2
+  git commit -m "Add ${l_release_name} reproducible build" || echo "git commit failed" >&2
+  popd
+}
+
+f_prep_docker_image() {
+  pushd $1
+  bin/make-base-vm --docker --suite bionic --arch amd64
   popd
 }
 
-f_validate_platform() {
+f_ensure_cache() {
+  g_gitian_cache="${g_workdir}/${GITIAN_CACHE_DIRNAME}"
+  [ -d "${g_gitian_cache}" ] || mkdir "${g_gitian_cache}"
+
+  g_cached_go_tarball="${g_gitian_cache}/${GO_TARBALL}"
+  if [ ! -f "${g_cached_go_tarball}" ]; then
+  f_echo_stderr "${g_cached_go_tarball}: cache miss, caching..."
+    curl -L "${GO_TARBALL_URL}" --output "${g_cached_go_tarball}"
+  fi
+
+  g_cached_gitian="${g_gitian_cache}/gitian-builder"
+  if [ ! -d "${g_cached_gitian}" ]; then
+  f_echo_stderr "${g_cached_gitian}: cache miss, caching..."
+    git clone ${GITIAN_REPO} "${g_cached_gitian}"
+  fi
+}
+
+f_demangle_platforms() {
   case "${1}" in
+  all)
+    printf '%s' 'darwin linux windows' ;;
   linux|darwin|windows)
-    ;;
+    printf '%s' "${1}" ;;
   *)
     echo "invalid platform -- ${1}"
     exit 1
   esac
 }
 
-f_abspath() {
-  echo "$(cd "$(dirname "$1")"; pwd -P)/$(basename "$1")"
+f_echo_stderr() {
+  echo $@ >&2
 }
 
-f_help() {
-  cat >&2 <<EOF
-Usage: $(basename $0) [-h] GOOS GIT_REPO
-Launch a gitian build from the local clone of cosmos-sdk available at GIT_REPO.
-
-  Options:
-   -h               display this help and exit
-   -d DIRNAME       set working directory name and skip gitian-builder download
-   -s IDENTITY      sign build as IDENTITY
-
-If a GPG identity is supplied via the -s flag, the build will be signed and verified.
-The signature will be saved in '${DEFAULT_GAIA_SIGS}/'. An alternative output directory
-for signatures can be supplied via the environment variable \$GAIA_SIGS.
-
-The default signing command used to sign the build is '$DEFAULT_SIGN_COMMAND'.
-An alternative signing command can be supplied via the environment
-variable \$SIGN_COMMAND.
-EOF
-}
 
-while getopts ":d:s:h" opt; do
+while getopts ":cs:h" opt; do
   case "${opt}" in
     h)  f_help ; exit 0 ;;
-    d)  g_dirname="${OPTARG}" ;;
+    c)  g_flag_commit=y ;;
     s)  g_sign_identity="${OPTARG}" ;;
   esac
 done
 
 shift "$((OPTIND-1))"
 
-g_platform="${1}"
-f_validate_platform "${g_platform}"
+g_platforms=$(f_demangle_platforms "${1}")
+g_workdir="$(pwd)"
+g_commit="$(git rev-parse HEAD)"
+g_sigs_dir=${GAIA_SIGS:-"${g_workdir}/${DEFAULT_GAIA_SIGS}"}
 
-g_dirname="${g_dirname:-gitian-build-${g_platform}}"
-g_workdir="$(pwd)/${g_dirname}"
-if [ -d "${g_workdir}" ]; then
-  echo "Directory ${g_workdir} exists and will be preserved" >&2
-  g_gitian_skip_download=y
-fi
+f_ensure_cache
 
-g_sdk="$(f_abspath ${2})"
-[ -d "${g_sdk}" ]
+f_prep_docker_image "${g_cached_gitian}"
 
-g_sigs_dir=${GAIA_SIGS:-"$(pwd)/${DEFAULT_GAIA_SIGS}"}
+f_prep_build "${g_platforms}"
 
-# create local cache directory for all gitian builds
-g_gitian_cache="$(pwd)/${GITIAN_CACHE_DIRNAME}"
-echo "Ensure cache directory ${g_gitian_cache} exists" >&2
-mkdir -p "${g_gitian_cache}"
+export USE_DOCKER=1
+for g_os in ${g_platforms}; do
+  g_release="$(git describe --tags --abbrev=9 | sed 's/^v//')-${g_os}"
+  g_descriptor="${g_workdir}/cmd/gaia/contrib/gitian-descriptors/gitian-${g_os}.yml"
+  [ -f ${g_descriptor} ]
+  g_builddir="$(f_builddir ${g_os})"
+
+  pushd "${g_builddir}"
+  f_build "${g_descriptor}"
+  if [ -n "${g_sign_identity}" ]; then
+    f_sign_verify "${g_descriptor}"
+  fi
+  popd
+
+  if [ -n "${g_sign_identity}" -a -n "${g_flag_commit}" ]; then
+    [ -d "${g_sigs_dir}/.git/" ] && f_commit_sig ${g_release} || f_echo_stderr "couldn't commit, ${g_sigs_dir} is not a git clone"
+  fi
+done
 
-f_main "${g_platform}" "${g_sdk}" "${g_sigs_dir}"
+exit 0

docs/cosmos-hub/reproducible-builds.md

@@ -39,7 +39,7 @@ Run the following command to launch a build for `linux` and sign the final build
 report (replace `[email protected]` with the GPG identity you want to sign the report with):
 
 ```
-./cmd/gaia/contrib/gitian-build.sh -s [email protected] linux `pwd`
+./cmd/gaia/contrib/gitian-build.sh -s [email protected] linux
 ```
 
 The above command generates two directories in the current working directory:
@@ -57,7 +57,7 @@ for platform in darwin linux windows; do ./cmd/gaia/contrib/gitian-build.sh -s u
 If you want to generate unsigned builds, just remove the option `-s` from the command line:
 
 ```
-./cmd/gaia/contrib/gitian-build.sh linux `pwd`
+./cmd/gaia/contrib/gitian-build.sh linux
 ```
 
 At the end of the procedure, build results can be found in the `./gaia.sigs` directory: