automation of more steps of the release process

dev-tools/release/README.md

@@ -4,5 +4,8 @@ Most of the release steps are documented in the wiki.
 See https://cwiki.apache.org/confluence/display/GEODE/Releasing+Apache+Geode
 
 Scripts
-prepare_rc.sh: Checks out the various geode repos and builds a release candidate, ready to be published
-print_rc_email.sh: Generates an email to send to the dev list announcing a release candidate
+prepare_rc.sh: Checks out the various geode repos, builds a release candidate, and publishes to nexus staging repo
+commit_rc.sh: Pushes the tags and artifacts staged by prepare_rc.sh and then runs print_rc_email.sh 
+  print_rc_email.sh: Generates an email to send to the dev list announcing a release candidate
+promote_rc.sh: Makes an RC the final release and starts the mirroring process
+finalize_release.sh: Makes the docker and brew releases, etc

dev-tools/release/commit_rc.sh

@@ -18,14 +18,14 @@
 set -e
 
 usage() {
-      echo "Usage: print_rc_email.sh -v version_number -m maven_repo_id"
-      echo "  -v   The #.#.#.RC# version number"
-      echo "  -m   The 4 digit id of the nexus maven repo"
-      exit 1
+    echo "Usage: print_rc_email.sh -v version_number -m maven_repo_id"
+    echo "  -v   The #.#.#.RC# version number"
+    echo "  -m   The 4 digit id of the nexus maven repo"
+    exit 1
 }
 
 FULL_VERSION=""
-SIGNING_KEY=""
+MAVEN=""
 
 while getopts ":v:m:" opt; do
   case ${opt} in
@@ -42,21 +42,29 @@ while getopts ":v:m:" opt; do
 done
 
 if [[ ${FULL_VERSION} == "" ]] || [[ ${MAVEN} == "" ]]; then
-  usage
+    usage
 fi
 
-if [[ $FULL_VERSION =~ ([0-9]+\.[0-9]+\.[0-9]+)\.(RC[0-9]+) ]]; then
+if [[ $FULL_VERSION =~ ^([0-9]+\.[0-9]+\.[0-9]+)\.(RC[0-9]+)$ ]]; then
     VERSION=${BASH_REMATCH[1]}
-    RC=${BASH_REMATCH[2]}
 else
-    echo "Malformed version number ${FULL_VERSION}. Example valid number - 1.9.0.RC1"
+    echo "Malformed version number ${FULL_VERSION}. Example valid version: 1.9.0.RC1"
+    exit 1
+fi
+
+WORKSPACE=$PWD/release-${VERSION}-workspace
+GEODE=$WORKSPACE/geode
+GEODE_EXAMPLES=$WORKSPACE/geode-examples
+GEODE_NATIVE=$WORKSPACE/geode-native
+SVN_DIR=$WORKSPACE/dist/dev/geode
+
+if [ -d "$GEODE" ] && [ -d "$GEODE_EXAMPLES" ] && [ -d "$GEODE_NATIVE" ] && [ -d "$SVN_DIR" ] ; then
+    true
+else
+    echo "Please run this script from the same working directory as you initially ran prepare_rc.sh"
     exit 1
 fi
 
-GEODE=$PWD/build/geode
-GEODE_EXAMPLES=$PWD/build/geode-examples
-GEODE_NATIVE=$PWD/build/geode-native
-SVN_DIR=$PWD/build/dist/dev/geode
 
 echo "============================================================"
 echo "Publishing artifacts to apache release location..."
@@ -65,18 +73,6 @@ cd ${SVN_DIR}
 svn commit -m "Releasing Apache Geode ${FULL_VERSION} distribution"
 
 
-echo "============================================================"
-echo "Pushing tags..."
-echo "============================================================"
-
-cd ${GEODE}
-git push origin rel/v${FULL_VERSION}
-cd ${GEODE_EXAMPLES}
-git push origin rel/v${FULL_VERSION}
-cd ${GEODE_NATIVE}
-git push origin rel/v${FULL_VERSION}
-
-
 echo "============================================================"
 echo "Adding temporary commit for geode-examples to build against staged ${FULL_VERSION}..."
 echo "============================================================"
@@ -91,11 +87,21 @@ git push
 
 
 echo "============================================================"
-echo "Done publishing the release!  Send the email below to announce it:"
+echo "Pushing tags..."
 echo "============================================================"
+
+for DIR in ${GEODE} ${GEODE_EXAMPLES} ${GEODE_NATIVE} ; do
+    cd ${DIR}
+    git push origin rel/v${FULL_VERSION}
+done
+
+
+echo "============================================================"
+echo "Done publishing the release candidate!  Send the email below to announce it:"
+echo "============================================================"
+cd ${GEODE}/../..
 echo "To: [email protected]"
 echo "Subject: [VOTE] Apache Geode ${FULL_VERSION}"
-cd ${GEODE}/../..
 ${0%/*}/print_rc_email.sh -v ${FULL_VERSION} -m ${MAVEN}
 echo ""
 which pbcopy >/dev/null && ${0%/*}/print_rc_email.sh -v ${FULL_VERSION} -m ${MAVEN} | pbcopy && echo "(copied to clipboard)"

dev-tools/release/finalize_release.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+    echo "Usage: finalize_release.sh -v version_number -k your_full_gpg_public_key -g your_github_username"
+    echo "  -v   The #.#.# version number to finalize"
+    echo "  -k   Your 40 digit GPG fingerprint"
+    echo "  -g   Your github username"
+    exit 1
+}
+
+VERSION=""
+SIGNING_KEY=""
+GITHUB_USER=""
+
+while getopts ":v:k:g:" opt; do
+  case ${opt} in
+    v )
+      VERSION=$OPTARG
+      ;;
+    k )
+      SIGNING_KEY=$OPTARG
+      ;;
+    g )
+      GITHUB_USER=$OPTARG
+      ;;
+    \? )
+      usage
+      ;;
+  esac
+done
+
+if [[ ${VERSION} == "" ]] || [[ ${SIGNING_KEY} == "" ]] || [[ ${GITHUB_USER} == "" ]] ; then
+    usage
+fi
+
+SIGNING_KEY=$(echo $SIGNING_KEY|tr -d ' ')
+if [[ $SIGNING_KEY =~ ^[0-9A-Fa-f]{40}$ ]]; then
+    true
+else
+    echo "Malformed signing key ${SIGNING_KEY}. Example valid key: '0000 0000 1111 1111 2222 2222 3333 3333 ABCD 1234'"
+    exit 1
+fi
+
+if [[ $VERSION =~ ^([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
+    true
+else
+    echo "Malformed version number ${VERSION}. Example valid version: 1.9.0"
+    exit 1
+fi
+
+WORKSPACE=$PWD/release-${VERSION}-workspace
+GEODE=$WORKSPACE/geode
+GEODE_DEVELOP=$WORKSPACE/geode-develop
+GEODE_EXAMPLES=$WORKSPACE/geode-examples
+GEODE_NATIVE=$WORKSPACE/geode-native
+BREW_DIR=$WORKSPACE/homebrew-core
+SVN_DIR=$WORKSPACE/dist/dev/geode
+
+if [ -d "$GEODE" ] && [ -d "$GEODE_DEVELOP" ] && [ -d "$GEODE_EXAMPLES" ] && [ -d "$GEODE_NATIVE" ] && [ -d "$BREW_DIR" ] && [ -d "$SVN_DIR" ] ; then
+    true
+else
+    echo "Please run this script from the same working directory as you initially ran prepare_rc.sh"
+    exit 1
+fi
+
+
+echo "============================================================"
+echo "Updating brew"
+echo "============================================================"
+cd ${BREW}/Formula
+git pull
+git remote add myfork [email protected]:${GITHUB_USER}/homebrew-core.git
+if ! git fetch myfork ; then
+    echo "Please fork https://github.com/Homebrew/homebrew-core"
+    exit 1
+fi
+git checkout -b apache-geode-${VERSION}
+GEODE_SHA=$(awk '{print $1}' < $WORKSPACE/dist/release/geode/${VERSION}/apache-geode-${VERSION}.tgz.sha256)
+sed -e 's# *url ".*#  url "https://www.apache.org/dyn/closer.cgi?path=geode/'"${VERSION}"'/apache-geode-'"${VERSION}"'.tgz"#' \
+    -e 's/ *sha256 ".*/  sha256 "'"${GEODE_SHA}"'"/' \
+    -i.bak apache-geode.rb
+rm apache-geode.rb.bak
+git add apache-geode.rb
+git diff --staged
+git commit -m "apache-geode ${VERSION}"
+git push -u myfork
+
+
+echo "============================================================"
+echo "Updating Dockerfile"
+echo "============================================================"
+cd ${GEODE}/docker
+sed -e "s/^ENV GEODE_GPG.*/ENV GEODE_GPG ${SIGNING_KEY}/" \
+    -e "s/^ENV GEODE_VERSION.*/ENV GEODE_VERSION ${VERSION}/" \
+    -e "s/^ENV GEODE_SHA256.*/ENV GEODE_SHA256 ${GEODE_SHA}/" \
+    -i.bak Dockerfile
+rm Dockerfile.bak
+git add Dockerfile
+git diff --staged
+git commit -m "apache-geode ${VERSION}"
+git push
+
+
+echo "============================================================"
+echo "Building docker image"
+echo "============================================================"
+set -x
+cd ${GEODE}/docker
+docker build .
+docker build -t apachegeode/geode:${VERSION} .
+docker build -t apachegeode/geode:latest .
+docker push apachegeode/geode:${VERSION}
+docker push apachegeode/geode:latest
+set +x
+
+
+echo "============================================================"
+echo "Destroying pipeline"
+echo "============================================================"
+set -x
+cd ${GEODE}
+fly -t concourse.apachegeode-ci.info login --concourse-url https://concourse.apachegeode-ci.info/
+cd ci/pipelines/meta
+./destroy_pipelines.sh
+set +x
+
+
+echo "============================================================"
+echo "Merging to master"
+echo "============================================================"
+for DIR in ${GEODE} ${GEODE_EXAMPLES} ${GEODE_NATIVE} ; do
+    set -x
+    cd ${DIR}
+    git fetch origin
+    git checkout release/${VERSION}
+    #this creates a merge commit that will then be ff-merged to master, so word it from that perspective
+    git merge -s ours origin/master -m "Replacing master with contents of release/${VERSION}"
+    git checkout master
+    git merge release/${VERSION}
+    git push origin master
+    set +x
+done
+
+
+echo "============================================================"
+echo "Destroying release branches"
+echo "============================================================"
+for DIR in ${GEODE} ${GEODE_EXAMPLES} ${GEODE_NATIVE} ; do
+    set -x
+    cd ${DIR}
+    git push origin --delete release/${VERSION}
+    git branch -D release/${VERSION}
+    set +x
+done
+
+
+echo "============================================================"
+echo "Updating 'old' versions"
+echo "============================================================"
+cd ${GEODE_DEVELOP}
+git pull
+#before:
+# '1.9.0'].each {
+#after:
+# '1.9.0',
+# '1.10.0'].each {
+sed -e "s/].each/,\\
+ '${VERSION}'].each/" \
+    -i.bak settings.gradle
+rm settings.gradle.bak
+git add settings.gradle
+git diff --staged
+git commit -m "add ${VERSION} to old versions"
+git push
+
+
+echo "============================================================"
+echo "Removing old versions from mirrors"
+echo "============================================================"
+cd $WORKSPACE/dist/release/geode
+svn update --set-depth immediates
+#identify the latest patch release for the latest 2 major.minor releases, remove anything else from mirrors (all releases remain available on non-mirrored archive site)
+RELEASES_TO_KEEP=2
+ls | awk -F. '/KEYS/{next}{print 1000000*$1+1000*$2+$3,$1"."$2"."$3}'| sort -n | awk '{mm=$2;sub(/\.[^.]*$/,"",mm);V[mm]=$2}END{for(v in V){print V[v]}}'|tail -$RELEASES_TO_KEEP > keep
+(ls | grep -v KEYS; cat keep keep)|sort|uniq -u|while read oldVersion; do
+    set -x
+    svn rm $oldVersion
+    svn commit -m "remove $oldVersion from mirrors (it is still available at http://archive.apache.org/dist/geode)"
+    set +x
+done
+rm keep
+
+
+echo "============================================================"
+echo "Done finalizing the release!"
+echo "============================================================"
+cd ${GEODE}/../..
+echo "Don't forget to:"
+echo "- Go to https://github.com/${GITHUB_USERNAME}/homebrew-core/pull/new/apache-geode-${VERSION} and submit the pull request"
+echo "- Validate docker image: docker run -it -p 10334:10334 -p 7575:7575 -p 1099:1099  apachegeode/geode"
+echo "- Update mirror links for old releases that were removed from mirrors"
+echo "- Publish documentation to docs site"
+echo "- Ask for a volunteer to Update Dependencies"
+echo "- Send announce email"