build_docker_remote: add ability to specify the RELEASE arg to the cl…

…oud-based docker build, and add a release script (broadinstitute#8247)

* Added a -r argument to build_docker_remote.sh to toggle the RELEASE flag during
  docker builds

* Added a release_prebuilt_docker_image.sh to release a prebuilt docker image to the
  official repos

build_docker_remote.sh

@@ -1,12 +1,26 @@
 #!/usr/bin/env bash
 #
-# Build (and optionally push) a GATK docker image to GCR using Google Cloud Build. Images are built in the cloud rather than locally. Pushing to dockerhub is not supported by this script.
+# Build (and optionally push) a GATK docker image to GCR using Google Cloud Build. Images are built
+# in the cloud rather than locally. Pushing to dockerhub is not supported by this script.
 #
-# If you are pushing an image to our release repositories, be sure that you've followed
-# the setup instructions here:
-# https://github.com/broadinstitute/gatk/wiki/How-to-release-GATK4#setup_docker
-# and here:
-# https://github.com/broadinstitute/gatk/wiki/How-to-release-GATK4#setup_gcloud
+# By default the images are pushed to the following GCR repository:
+#
+# us.gcr.io/broad-dsde-methods/broad-gatk-snapshots/gatk-remote-builds
+#
+# with a name like "${YOUR_USERNAME}-${GITHUB_TAG}-${GIT_HASH_FOR_TAG}"
+#
+# This script should not be used to push to our release repositories. After
+# you've built and staged an image, run the scripts/docker/release_prebuilt_docker_image.sh
+# script to push to the release repositories.
+#
+# Usage: build_docker_remote.sh -e <GITHUB_TAG> -d <STAGING_DIRECTORY> [-str]
+#
+# where <GITHUB_TAG> is the github tag (or hash when -s is used) to use in building the docker image (e.g. 4.2.6.1)
+# and <STAGING_DIRECTORY> is a directory in which to clone the repo and stage the build (DO NOT SPECIFY YOUR WORKING DIRECTORY)
+# Optional arguments:
+#   -s  The GITHUB_TAG (-e parameter) is actually a github hash, not tag.
+#   -r  Build this image with the release flag set to true, causing the version number to not end with SNAPSHOT
+#   -t <IMAGE_TAG>  The tag to assign image once it is finished constructing.  NOTE: currently this MUST be on either GCR or the Google Artifact Registry.
 #
 
 # Have script stop if there is an error
@@ -20,24 +34,35 @@ STAGING_CLONE_DIR=${PROJECT}_staging_temp
 #################################################
 # Parsing arguments
 #################################################
-while getopts "e:sd:t:" option; do
+while getopts "e:sd:t:r" option; do
 	case "$option" in
 		e) GITHUB_TAG="$OPTARG" ;;
 		s) IS_HASH=true ;;
 		d) STAGING_DIR="$OPTARG" ;;
 		t) DOCKER_IMAGE_TAG="$OPTARG" ;;
+		r) RELEASE=true ;;
 	esac
 done
 
-if [ -z "$GITHUB_TAG" ]; then
-	printf "Option -e requires an argument.\n \
-Usage: %s: -e <GITHUB_TAG> [-sdt] \n \
-where <GITHUB_TAG> is the github tag (or hash when -s is used) to use in building the docker image\n \
-(e.g. bash build_docker_remote.sh -e 4.2.6.1 )\n \
+function usage() {
+  MESSAGE=$1
+	printf "%s\n \
+Usage: build_docker_remote.sh -e <GITHUB_TAG> -d <STAGING_DIRECTORY> [-str] \n \
+where <GITHUB_TAG> is the github tag (or hash when -s is used) to use in building the docker image (e.g. 4.2.6.1)\n \
+and <STAGING_DIRECTORY> is a directory in which to clone the repo and stage the build (DO NOT SPECIFY YOUR WORKING DIRECTORY)\n \
 Optional arguments:  \n \
 -s \t The GITHUB_TAG (-e parameter) is actually a github hash, not tag.  \n \
--d <STAGING_DIR> \t staging directory to grab code from repo and build the docker image.  If unspecified, then use whatever is in current dir (do not go to the repo).  NEVER SPECIFY YOUR WORKING DIR \n \
--t <IMAGE_TAG>\t  The tag to assign image once it is finished constructing.  NOTE: currently this MUST be on either GCR or the Google Artifact Registry. \n" $0
+-r \t Build this image with the release flag set to true, causing the version number to not end with SNAPSHOT \n \
+-t <IMAGE_TAG>\t  The tag to assign image once it is finished constructing.  NOTE: currently this MUST be on either GCR or the Google Artifact Registry. \n" "$MESSAGE"
+}
+
+if [ -z "$GITHUB_TAG" ]; then
+	usage "Option -e (github tag) requires an argument."
+	exit 1
+fi
+
+if [ -z "$STAGING_DIR" ]; then
+	usage "Option -d (staging directory) requires an argument."
 	exit 1
 fi
 
@@ -49,6 +74,7 @@ echo "Other options (Blank is false)"
 echo "---------------"
 echo "This is a git hash: ${IS_HASH}"
 echo "Staging directory: ${STAGING_DIR}"
+echo "Release: ${RELEASE}"
 
 ORIGINAL_WORKING_DIRECTORY=$(pwd)
 
@@ -81,9 +107,19 @@ if [ -z "$DOCKER_IMAGE_TAG" ]; then
   DOCKER_IMAGE_TAG=${GCR_REPO}:$(whoami)-${GITHUB_TAG}-${GIT_HASH_FOR_TAG}
 fi
 
+echo "steps:" >> cloudbuild.yaml
+echo "- name: 'gcr.io/cloud-builders/docker'" >> cloudbuild.yaml
+if [ -n "$RELEASE" ]; then
+  echo "  args: [ 'build', '-t', '${DOCKER_IMAGE_TAG}', '--build-arg', 'RELEASE=true', '.' ]" >> cloudbuild.yaml
+else
+  echo "  args: [ 'build', '-t', '${DOCKER_IMAGE_TAG}', '.' ]" >> cloudbuild.yaml
+fi
+echo "- name: 'gcr.io/cloud-builders/docker'" >> cloudbuild.yaml
+echo "  args: [ 'push', '${DOCKER_IMAGE_TAG}' ]" >> cloudbuild.yaml
+
 echo "Building image with the tag ${DOCKER_IMAGE_TAG}..."
 
-SUBMIT_COMMAND="gcloud builds submit --tag ${DOCKER_IMAGE_TAG} --timeout=24h --machine-type n1_highcpu_8"
+SUBMIT_COMMAND="gcloud builds submit --config cloudbuild.yaml --timeout=24h --machine-type n1_highcpu_32"
 echo "running the following gcloud command: ${SUBMIT_COMMAND}"
 ## We need to override the default .gcloudignore to preserve the .git directory which we need in order to download LFS files in the remote build.
 echo -n "" >> .gcloudignore

scripts/docker/release_prebuilt_docker_image.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+#
+# A script that takes a prebuilt docker image, and pushes it to the GATK release repositories on
+# dockerhub and GCR
+#
+# Usage: release_prebuilt_docker_image.sh <prebuilt_image> <version_tag_for_release>
+#
+# If the prebuilt image exists locally, this script will push the local version. Otherwise,
+# it will pull the image from the remote repository before pushing it to the GATK release
+# repositories.
+#
+# prebuilt_image: The pre-built image you want to release (make sure you've tested it!)
+#                 May be either local or remote
+# version_tag_for_release: The version of GATK you're releasing (eg., 4.0.0.0)
+#
+
+if [ $# -ne 2 ]; then
+  echo "Usage: $0 <prebuilt_image> <version_tag_for_release>"
+  exit 1
+fi
+
+PREBUILT_IMAGE="$1"
+VERSION="$2"
+DOCKERHUB_REPO="broadinstitute/gatk"
+GCR_REPO="us.gcr.io/broad-gatk/gatk"
+
+function fatal_error() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+function docker_push() {
+  echo "Pushing to ${1}"
+  docker push "${1}"
+  if [ $? -ne 0 ]; then
+    fatal_error "Failed to push to ${1}"
+  fi
+}
+
+# Test if the prebuilt image exists locally, and pull it if it doesn't
+docker image inspect "${PREBUILT_IMAGE}" > /dev/null 2>&1
+if [ $? -ne 0 ]; then
+  echo "Image ${PREBUILT_IMAGE} not found locally: attempting to pull it now"
+  docker pull "${PREBUILT_IMAGE}"
+  if [ $? -ne 0 ]; then
+    fatal_error "Failed to pull pre-built image ${PREBUILT_IMAGE}"
+  fi
+else
+  echo "Image ${PREBUILT_IMAGE} found locally: pushing it to the release repositories"
+fi
+
+docker tag "${PREBUILT_IMAGE}" "${DOCKERHUB_REPO}:${VERSION}"
+docker tag "${DOCKERHUB_REPO}:${VERSION}" "${DOCKERHUB_REPO}:latest"
+docker tag "${PREBUILT_IMAGE}" "${GCR_REPO}:${VERSION}"
+docker tag "${GCR_REPO}:${VERSION}" "${GCR_REPO}:latest"
+
+docker_push "${DOCKERHUB_REPO}:${VERSION}"
+docker_push "${DOCKERHUB_REPO}:latest"
+docker_push "${GCR_REPO}:${VERSION}"
+docker_push "${GCR_REPO}:latest"
+
+exit 0