Skip to content

Commit

Permalink
GEODE-7915: rc pipeline fixes and improvements (apache#4853)
Browse files Browse the repository at this point in the history 
* remove examples.zip test from pipeline as .zip is no longer produced

* check all source artifacts to ensure they do not contain any binaries

* check that no unexpected artifacts are present

* align release retention with "N-2" proposal
  • Loading branch information
@onichols-pivotal
onichols-pivotal authored Mar 27, 2020
1 parent 2c26de8 commit 80ffe65
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 40 deletions.
114 changes: 76 additions & 38 deletions dev-tools/release/deploy_rc_pipeline.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ jobs:
cd apache-geode-${VERSION}-src
java -version
./gradlew test
- name: run-geode-examples
- name: run-geode-examples-jdk11
serial: true
plan:
- aggregate:
Expand All @@ -151,7 +151,7 @@ jobs:
type: docker-image
source:
repository: openjdk
tag: 8
tag: 11
inputs:
- name: geode-examples
platform: linux
Expand All @@ -164,7 +164,7 @@ jobs:
cd geode-examples
java -version
./gradlew runAll
- name: run-geode-examples-from-src-tar-gz
- name: run-geode-examples-from-src-tar-gz-jdk8
serial: true
plan:
- aggregate:
Expand All @@ -187,45 +187,14 @@ jobs:
- -ec
- |
set -ex
FULL_VERSION=$(cat geode-examples/gradle.properties | grep geodeReleaseUrl | sed -e 's#.*/geode/##')
FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##' -e 's#-.*##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}.tar.gz > src.tgz
tar xzf src.tgz
cd apache-geode-examples-${VERSION}
java -version
./gradlew -PgeodeReleaseUrl=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} -PgeodeRepositoryUrl=${STAGING_MAVEN} build runAll
- name: run-geode-examples-from-src-zip-11
serial: true
plan:
- aggregate:
- get: geode-examples
trigger: true
- task: validate
timeout: 1h
config:
image_resource:
type: docker-image
source:
repository: openjdk
tag: 11
inputs:
- name: geode-examples
platform: linux
run:
path: /bin/sh
args:
- -ec
- |
set -ex
FULL_VERSION=$(cat geode-examples/gradle.properties | grep geodeReleaseUrl | sed -e 's#.*/geode/##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}.zip > src.zip
unzip src.zip
cd apache-geode-examples-${VERSION}
java -version
./gradlew -PgeodeReleaseUrl=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} -PgeodeRepositoryUrl=${STAGING_MAVEN} build runAll
- name: build-geode-native-from-tag
serial: true
plan:
Expand Down Expand Up @@ -284,6 +253,7 @@ jobs:
repository: openjdk
tag: 8
inputs:
- name: geode-native
- name: geode
platform: linux
run:
Expand All @@ -292,7 +262,7 @@ jobs:
- -ec
- |
set -ex
FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
FULL_VERSION=$(cd geode-native && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
# build geode from source
cd geode
Expand Down Expand Up @@ -331,6 +301,7 @@ jobs:
repository: openjdk
tag: 8
inputs:
- name: geode
- name: upthewaterspout-tests
- name: geode-examples
platform: linux
Expand All @@ -340,7 +311,7 @@ jobs:
- -ec
- |
set -ex
FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##')
FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
cd upthewaterspout-tests
Expand Down Expand Up @@ -384,7 +355,7 @@ jobs:
echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
service ssh start
./gradlew build test
- name: verify-keys
- name: verify-expected-files-and-keys
serial: true
plan:
- aggregate:
Expand Down Expand Up @@ -423,12 +394,79 @@ jobs:
curl -s $url/$sha > $sha
gpg --verify $asc
$sum -c $sha
echo $file >> exp
echo $asc >> exp
echo $sha >> exp
}
verifyArtifactSignature apache-geode-${VERSION}-src.tgz 256
verifyArtifactSignature apache-geode-${VERSION}.tgz 256
verifyArtifactSignature apache-geode-examples-${VERSION}.tar.gz 256
verifyArtifactSignature apache-geode-native-${VERSION}-src.tar.gz 512
verifyArtifactSignature apache-geode-benchmarks-${VERSION}-src.tgz 256
curl -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list
sort < exp > expected-file-list
set +x
echo ""
if diff -q expected-file-list actual-file-list ; then
echo "The file list at $url matches what is expected and all signatures were verified :)"
else
echo "Expected:"
cat expected-file-list
echo ""
echo "Actual:"
cat actual-file-list
echo ""
echo "Diff:"
diff expected-file-list actual-file-list
exit 1
fi
- name: verify-no-binaries
serial: true
plan:
- aggregate:
- get: geode
trigger: true
- task: validate
timeout: 1h
config:
image_resource:
type: docker-image
source:
repository: openjdk
tag: 8
inputs:
- name: geode
platform: linux
run:
path: /bin/bash
args:
- -ec
- |
set -e
FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
url=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}
BINARY_EXTENSIONS="jar|war|class|exe|dll|o|so|obj|bin|out|pyc"
echo "Source artifacts should not contain any files ending in$(echo "|${BINARY_EXTENSIONS}"|sed 's/[^a-z]/ ./g')"
echo ""
function verifyNoBinaries {
file=$1
echo ""
echo Checking $file...
curl -s $url/$file | tar tvzf - | egrep '\.('"${BINARY_EXTENSIONS}"')$' | tee -a bins
}
verifyNoBinaries apache-geode-${VERSION}-src.tgz
verifyNoBinaries apache-geode-examples-${VERSION}.tar.gz
verifyNoBinaries apache-geode-native-${VERSION}-src.tar.gz
verifyNoBinaries apache-geode-benchmarks-${VERSION}-src.tgz
echo ""
echo ""
if grep -q . bins ; then
echo Binary files were found!
exit 1
else
echo All good
fi
EOF
fly -t concourse.apachegeode-ci.info-main login --team-name main --concourse-url https://concourse.apachegeode-ci.info/
fly -t concourse.apachegeode-ci.info-main set-pipeline -p apache-release-${VERSION//./-}-rc -c $PIPEYML
Expand Down
4 changes: 2 additions & 2 deletions dev-tools/release/finalize_release.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,8 @@ echo "============================================================"
set -x
cd $SVN_RELEASE_DIR
svn update --set-depth immediates
#identify the latest patch release for the latest 2 major.minor releases, remove anything else from mirrors (all releases remain available on non-mirrored archive site)
RELEASES_TO_KEEP=2
#identify the latest patch release for "N-2" (the latest 3 major.minor releases), remove anything else from mirrors (all releases remain available on non-mirrored archive site)
RELEASES_TO_KEEP=3
set +x
ls | awk -F. '/KEYS/{next}{print 1000000*$1+1000*$2+$3,$1"."$2"."$3}'| sort -n | awk '{mm=$2;sub(/\.[^.]*$/,"",mm);V[mm]=$2}END{for(v in V){print V[v]}}'|tail -$RELEASES_TO_KEEP > ../keep
echo Keeping releases: $(cat ../keep)
Expand Down

0 comments on commit 80ffe65

Please sign in to comment.