GEODE-8026: release improvements (apache#5002)

* make release artifact names consistent * verify tgz structure including presence of LICENSE and NOTICE and correct copyright in NOTICE * verify file size is reasonable * check that gfsh version --full contains correct SHA, version, and was built with an open-licensed JDK
ybz216 · Apr 27, 2020 · edcc077 · edcc077
1 parent fad52b8
commit edcc077
Show file tree

Hide file tree

Showing 2 changed files with 117 additions and 55 deletions.
diff --git a/dev-tools/release/deploy_rc_pipeline.sh b/dev-tools/release/deploy_rc_pipeline.sh
@@ -105,9 +105,16 @@ jobs:
             - -ec
             - |
               set -ex
+              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
+              SHA=$(cd geode && git rev-parse HEAD)
               java -version
               cd geode
-              ./gradlew test
+              ./gradlew test installDist
+              gfsh=geode-assembly/build/install/apache-geode/bin/gfsh
+              $gfsh version --full | grep "^Source-Revision: ${SHA}$"
+              $gfsh version --full | grep "^Product-Version: ${VERSION}$"
+              ! $gfsh version --full | grep Oracle
   - name: build-geode-from-src-tgz
     serial: true
     plan:
@@ -133,11 +140,50 @@ jobs:
               set -ex
               FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
-              curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}-src.tgz > src.tgz
+              SHA=$(cd geode && git rev-parse HEAD)
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}-src.tgz > src.tgz
               tar xzf src.tgz
               cd apache-geode-${VERSION}-src
               java -version
-              ./gradlew test
+              ./gradlew test installDist
+              gfsh=geode-assembly/build/install/apache-geode/bin/gfsh
+              $gfsh version --full | grep "^Source-Revision: ${SHA}$"
+              $gfsh version --full | grep "^Product-Version: ${VERSION}$"
+              ! $gfsh version --full | grep Oracle
+  - name: run-gfsh-from-tgz
+    serial: true
+    plan:
+      - aggregate:
+          - get: geode
+            trigger: true
+      - task: validate
+        timeout: 1h
+        config:
+          image_resource:
+            type: docker-image
+            source:
+              repository: openjdk
+              tag: 8
+          inputs:
+            - name: geode
+          platform: linux
+          run:
+            path: /bin/sh
+            args:
+            - -ec
+            - |
+              set -ex
+              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
+              SHA=$(cd geode && git rev-parse HEAD)
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > bin.tgz
+              tar xzf bin.tgz
+              cd apache-geode-${VERSION}
+              java -version
+              gfsh=bin/gfsh
+              $gfsh version --full | grep "^Source-Revision: ${SHA}$"
+              $gfsh version --full | grep "^Product-Version: ${VERSION}$"
+              ! $gfsh version --full | grep Oracle
   - name: run-geode-examples-jdk11
     serial: true
     plan:
@@ -164,7 +210,7 @@ jobs:
               cd geode-examples
               java -version
               ./gradlew runAll
-  - name: run-geode-examples-from-src-tar-gz-jdk8
+  - name: run-geode-examples-from-src-tgz-jdk8
     serial: true
     plan:
       - aggregate:
@@ -190,9 +236,9 @@ jobs:
               FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##' -e 's#-.*##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
-              curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}.tar.gz > src.tgz
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}-src.tgz > src.tgz
               tar xzf src.tgz
-              cd apache-geode-examples-${VERSION}
+              cd apache-geode-examples-${VERSION}-src
               java -version
               ./gradlew -PgeodeReleaseUrl=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} -PgeodeRepositoryUrl=${STAGING_MAVEN} build runAll
   - name: build-geode-native-from-tag
@@ -221,7 +267,7 @@ jobs:
               FULL_VERSION=$(cd geode-native && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               #use geode from binary dist
-              curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > geode-bin.tgz
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > geode-bin.tgz
               tar xzf geode-bin.tgz
               # needed to get cmake >= 3.12
               echo 'APT::Default-Release "stable";' >> /etc/apt/apt.conf.d/99defaultrelease
@@ -237,7 +283,7 @@ jobs:
               cmake --build . -- -j 4
               cmake --build . --target docs -- -j 4
               cmake --build . --target install -- -j 4
-  - name: build-geode-native-from-src-tar-gz
+  - name: build-geode-native-from-src-tgz
     serial: true
     plan:
       - aggregate:
@@ -275,9 +321,9 @@ jobs:
               echo 'deb     http://security.debian.org/         stable/updates  main contrib non-free' >> /etc/apt/sources.list.d/stable.list
               apt-get update
               DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y cmake openssl doxygen build-essential libssl-dev zlib1g-dev
-              curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-native-${VERSION}-src.tar.gz > src.tgz
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-native-${VERSION}-src.tgz > src.tgz
               tar xzf src.tgz
-              cd apache-geode-native*
+              cd apache-geode-native-${VERSION}-src
               mkdir build
               cd build
               cmake .. -DGEODE_ROOT=$PWD/../../geode/geode-assembly/build/install/apache-geode
@@ -315,7 +361,7 @@ jobs:
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
               cd upthewaterspout-tests
-              curl -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
               gpg --import KEYS
               java -version
               ./gradlew build -PmavenURL=${STAGING_MAVEN} -PdownloadURL=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/ -Pversion=${FULL_VERSION}
@@ -343,7 +389,7 @@ jobs:
               set -ex
               FULL_VERSION=$(cd geode-benchmarks && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
-              curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-benchmarks-${VERSION}-src.tgz > src.tgz
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-benchmarks-${VERSION}-src.tgz > src.tgz
               tar xzf src.tgz
               cd apache-geode-benchmarks-${VERSION}-src
               java -version
@@ -380,30 +426,54 @@ jobs:
               set -ex
               FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
-              curl -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
+              curl -L -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
               gpg --import KEYS
               url=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}
-              function verifyArtifactSignature {
-                file=$1
+              function verifyArtifactSizeSignatureLicenseNoticeAndCopyright {
+                tld=$1
+                file=${tld}.tgz
+                minfilesize=$2
+                maxfilesize=$3
                 echo Verifying $file...
                 asc=${file}.asc
-                sha=${file}.sha$2
-                sum=sha${2}sum
-                curl -s $url/$file > $file
-                curl -s $url/$asc > $asc
-                curl -s $url/$sha > $sha
+                sha=${file}.sha256
+                sum=sha256sum
+                curl -L -s $url/$file > $file
+                actualfilesize=$(wc -c < $file)
+                if [ $actualfilesize -lt $minfilesize ] ; then
+                  echo "File size of $file is only $actualfilesize bytes, expected at least $minfilesize"
+                  return 1
+                fi
+                if [ $actualfilesize -gt $maxfilesize ] ; then
+                  echo "File size of $file is $actualfilesize, expected no more than $maxfilesize bytes"
+                  return 1
+                fi
+                curl -L -s $url/$asc > $asc
+                curl -L -s $url/$sha > $sha
                 gpg --verify $asc
                 $sum -c $sha
                 echo $file >> exp
                 echo $asc >> exp
                 echo $sha >> exp
+                #check that each archive contains all content below a top-level-directory with the same name as the file (sans .tgz)
+                ! tar tvzf $file | grep -v " ${tld}/"
+                #check that each archive contains LICENSE and NOTICE
+                tar tvzf $file | grep " ${tld}/LICENSE"
+                tar tvzf $file | grep " ${tld}/NOTICE"
+                #check that NOTICE contains current copyright year and correctly assigns copyright to ASF
+                tar xzf $file "${tld}/NOTICE"
+                year=$(date +%Y)
+                grep "Copyright" "${tld}/NOTICE"
+                grep -q "Copyright.*${year}.*Apache Software Foundation" "${tld}/NOTICE"
+                #check that the declared license is of the correct type
+                head -1 "${tld}/LICENSE" | grep -q "Apache License"
               }
-              verifyArtifactSignature apache-geode-${VERSION}-src.tgz 256
-              verifyArtifactSignature apache-geode-${VERSION}.tgz 256
-              verifyArtifactSignature apache-geode-examples-${VERSION}.tar.gz 256
-              verifyArtifactSignature apache-geode-native-${VERSION}-src.tar.gz 512
-              verifyArtifactSignature apache-geode-benchmarks-${VERSION}-src.tgz 256
-              curl -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION}-src 10000000 30000000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION} 100000000 150000000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-examples-${VERSION}-src 50000 2000000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-native-${VERSION}-src 2000000 4000000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-benchmarks-${VERSION}-src 50000 500000
+              curl -L -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list
               sort < exp > expected-file-list
               set +x
               echo ""
@@ -453,11 +523,11 @@ jobs:
                 file=$1
                 echo ""
                 echo Checking $file...
-                curl -s $url/$file | tar tvzf - | egrep '\.('"${BINARY_EXTENSIONS}"')$' | tee -a bins
+                curl -L -s $url/$file | tar tvzf - | egrep '\.('"${BINARY_EXTENSIONS}"')$' | tee -a bins
               }
               verifyNoBinaries apache-geode-${VERSION}-src.tgz
-              verifyNoBinaries apache-geode-examples-${VERSION}.tar.gz
-              verifyNoBinaries apache-geode-native-${VERSION}-src.tar.gz
+              verifyNoBinaries apache-geode-examples-${VERSION}-src.tgz
+              verifyNoBinaries apache-geode-native-${VERSION}-src.tgz
               verifyNoBinaries apache-geode-benchmarks-${VERSION}-src.tgz
               echo ""
               echo ""
@@ -470,4 +540,5 @@ jobs:
 EOF
 fly -t concourse.apachegeode-ci.info-main login --team-name main --concourse-url https://concourse.apachegeode-ci.info/
 fly -t concourse.apachegeode-ci.info-main set-pipeline -p apache-support-${VERSION_MM//./-}-rc -c $PIPEYML
+fly -t concourse.apachegeode-ci.info-main unpause-pipeline -p apache-support-${VERSION_MM//./-}-rc
 rm $PIPEYML
diff --git a/dev-tools/release/prepare_rc.sh b/dev-tools/release/prepare_rc.sh
@@ -104,6 +104,11 @@ GEODE_NATIVE=$WORKSPACE/geode-native
 GEODE_BENCHMARKS=$WORKSPACE/geode-benchmarks
 BREW_DIR=$WORKSPACE/homebrew-core
 SVN_DIR=$WORKSPACE/dist/dev/geode
+if which shasum >/dev/null; then
+  SHASUM="shasum -a 256"
+else
+  SHASUM=sha256sum
+fi
 set +x
 
 
@@ -193,25 +198,18 @@ which brew >/dev/null && OPENSSL_ROOT_DIR=$(brew --prefix openssl) || OPENSSL_RO
 cd ${GEODE_NATIVE}/build
 cmake .. -DPRODUCT_VERSION=${VERSION} -DOPENSSL_ROOT_DIR=$OPENSSL_ROOT_DIR -DGEODE_ROOT=${GEODE}/geode-assembly/build/install/apache-geode
 cpack -G TGZ --config CPackSourceConfig.cmake
-NCTAR=apache-geode-native-${VERSION}-src.tar.gz
+NCOUT=apache-geode-native-${VERSION}-src.tar.gz
+NCTGZ=apache-geode-native-${VERSION}-src.tgz
 mkdir repkg-temp
 cd repkg-temp
-tar xzf ../${NCTAR}
-rm ../${NCTAR}
-mv apache-geode-native apache-geode-native-${VERSION}
-tar czf ../${NCTAR} *
+tar xzf ../${NCOUT}
+rm ../${NCOUT}*
+mv apache-geode-native apache-geode-native-${VERSION}-src
+tar czf ../${NCTGZ} *
 cd ..
 rm -Rf repkg-temp
-gpg --armor -u ${SIGNING_KEY} -b ${NCTAR}
-
-if which shasum >/dev/null; then
-  SHASUM=shasum
-  SHASUM_OPTS="-a 512"
-else
-  SHASUM=sha512sum
-  SHASUM_OPTS=""
-fi
-${SHASUM} ${SHASUM_OPTS} ${NCTAR} > ${NCTAR}.sha512
+gpg --armor -u ${SIGNING_KEY} -b ${NCTGZ}
+${SHASUM} ${NCTGZ} > ${NCTGZ}.sha256
 set +x
 
 
@@ -229,14 +227,7 @@ cp -r .travis.yml * ../${BMDIR}
 tar czf ${BMTAR} -C .. ${BMDIR}
 rm -Rf ../${BMDIR}
 gpg --armor -u ${SIGNING_KEY} -b ${BMTAR}
-if which shasum >/dev/null; then
-  SHASUM=shasum
-  SHASUM_OPTS="-a 256"
-else
-  SHASUM=sha256sum
-  SHASUM_OPTS=""
-fi
-${SHASUM} ${SHASUM_OPTS} ${BMTAR} > ${BMTAR}.sha256
+${SHASUM} ${BMTAR} > ${BMTAR}.sha256
 set +x
 
 
@@ -276,9 +267,9 @@ cp ${GEODE_BENCHMARKS}/apache-geode-benchmarks-${VERSION}* ${FULL_VERSION}
 set +x
 
 # verify all files are signed.  sometimes gradle "forgets" to make the .asc file
-for f in ${FULL_VERSION}/*.tgz ${FULL_VERSION}/*.tar.gz ; do
-  if ! [ -r $f.sha256 ] && ! [ -r $f.sha512 ] ; then
-    echo missing $f.sha256 or $f.sha512
+for f in ${FULL_VERSION}/*.tgz ; do
+  if ! [ -r $f.sha256 ] ; then
+    echo missing $f.sha256
     exit 1
   fi
   if ! [ -r $f.asc ] ; then