patch_tester.sh

#!/bin/sh

# Tests a set of patches from a directory.
# Copyright (C) 2007, 2008, 2011  Free Software Foundation, Inc.
# Contributed by Sebastian Pop <sebastian.pop@amd.com>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

cat <<EOF

WARNING: This script should only be fed with patches from known
         authorized and trusted sources.  Don't even think about
         hooking it up to a raw feed from the gcc-patches list or
         you'll regret it.

EOF

args=$@

svnpath=svn://gcc.gnu.org/svn/gcc
dashj=
default_standby=1
standby=$default_standby
default_watermark=0.60
watermark=$default_watermark
savecompilers=false
nopristinecache=false
nogpg=false
stop=false

usage() {
    cat <<EOF
patch_tester.sh [-j<N>] [-standby N] [-watermark N] [-savecompilers] [-nogpg]
                [-svnpath URL] [-stop] [-nopristinecache]
                <source_dir> [patches_dir [state_dir [build_dir]]]

    J is the flag passed to make.  Default is empty string.

    STANDBY is the number of minutes between checks for new patches in
    PATCHES_DIR.  Default is ${default_standby} minutes.

    WATERMARK is the 5 minute average system charge under which a new
    compile can start.  Default is ${default_watermark}.

    SAVECOMPILERS copies the compilers in the same directory as the
    test results for the non patched version.  Default is not copy.

    NOPRISTINECACHE prevents use of cached test results from any earlier
    test runs on the pristine version of the branch and revision under
    test (the default behaviour).  This should be used when testing the
    same revision and patch with multiple sets of configure options, as
    these may affect the set of baseline failures.

    NOGPG can be used to avoid checking the GPG signature of patches.

    URL is the location of the GCC SVN repository.  The default is
    ${svnpath}.

    STOP exits when PATCHES_DIR is empty.

    SOURCE_DIR is the directory containing GCC's toplevel configure.

    PATCHES_DIR is the directory containing the patches to be tested.
    Default is SOURCE_DIR/patches.

    STATE_DIR is where the tester maintains its internal state.
    Default is SOURCE_DIR/state.

    BUILD_DIR is the build tree, a temporary directory that this
    script will delete and recreate.  Default is SOURCE_DIR/obj.

EOF
    exit 1
}

makedir () {
    DIRNAME=$1
    mkdir -p $DIRNAME
    if [ $? -ne 0 ]; then
	echo "ERROR: could not make directory $DIRNAME"
	exit 1
    fi
}

while [ $# -ne 0 ]; do
    case $1 in
	-j*)
	    dashj=$1; shift
	    ;;
	-standby)
	    [[ $# > 2 ]] || usage
	    standby=$2; shift; shift
	    ;;
	-watermark)
	    [[ $# > 2 ]] || usage
	    watermark=$2; shift; shift
	    ;;
	-savecompilers)
	    savecompilers=true; shift
	    ;;
	-nopristinecache)
	    nopristinecache=true; shift
	    ;;
	-nogpg)
	    nogpg=true; shift
	    ;;
	-stop)
	    stop=true; shift
	    ;;
	-svnpath)
	    svnpath=$2; shift; shift
	    ;;
	-*) 
	    echo "Invalid option: $1"
	    usage
	    ;;
	*)
	    break
	    ;;
    esac
done

test $# -eq 0 && usage

SOURCE=$1
PATCHES=
STATE=
BUILD=

if [[ $# < 2 ]]; then
    PATCHES=$SOURCE/patches
else
    PATCHES=$2
fi
if [[ $# < 3 ]]; then
    STATE=$SOURCE/state
else
    STATE=$3
fi
if [[ $# < 4 ]]; then
    BUILD=$SOURCE/obj
else
    BUILD=$4
fi

[ -d $PATCHES ] || makedir $PATCHES
[ -d $STATE ] || makedir $STATE
[ -d $STATE/patched ] || makedir $STATE/patched
[ -d $SOURCE ] || makedir $SOURCE
[ -f $SOURCE/config.guess ] || {
    cd $SOURCE
    svn -q co $svnpath/trunk .
    if [ $? -ne 0 ]; then
	echo "ERROR: initial svn checkout failed"
	exit 1
    fi
}

# This can contain required local settings:
#  default_config  configure options, always passed
#  default_make    make bootstrap options, always passed
#  default_check   make check options, always passed
[ -f $STATE/defaults ] && . $STATE/defaults

VERSION=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`

exec >> $STATE/tester.log 2>&1 || exit 1
set -x

TESTING=$STATE/testing
REPORT=$TESTING/report
PRISTINE=$TESTING/pristine
PATCHED=$TESTING/patched
PATCH=
TARGET=`$SOURCE/config.guess || exit 1` 
TESTLOGS="gcc/testsuite/gcc/gcc.sum
gcc/testsuite/gfortran/gfortran.sum
gcc/testsuite/g++/g++.sum
gcc/testsuite/objc/objc.sum
$TARGET/libstdc++-v3/testsuite/libstdc++.sum
$TARGET/libffi/testsuite/libffi.sum
$TARGET/libgomp/testsuite/libgomp.sum
$TARGET/libmudflap/testsuite/libmudflap.sum"
COMPILERS="gcc/cc1
gcc/cc1obj
gcc/cc1plus
gcc/f951
gcc/jc1
gcc/gnat1
gcc/tree1"

now () {
    echo `TZ=UTC date +"%Y_%m_%d_%H_%M_%S"`
}

report () {
    echo "$@" >> $REPORT
}

freport () {
    if [ -s $1 ]; then
	report "(cat $1"
	cat $1 >> $REPORT
	report "tac)"
    fi
}

cleanup () {
    cd $SOURCE
    svn cleanup && svn revert -R . && svn st | cut -d' ' -f5- | xargs rm -v
}

selfexec () {
    exec ${CONFIG_SHELL-/bin/sh} $0 $args
}

update () {
    svn_branch=`grep "^branch:" $PATCH | sed -e "s/^branch://g" -e "s/ //g"`
    if [ x$svn_branch = x ]; then
	svn_branch=trunk
    fi

    svn_revision=`grep "^revision:" $PATCH | sed -e "s/^revision://g" -e "s/ //g"`
    if [ x$svn_revision = x ]; then
	svn_revision=HEAD
    fi

    cleanup
    cd $SOURCE
    case $svn_branch in
	trunk)
	    if ! svn switch -r $svn_revision $svnpath/trunk &> $TESTING/svn ; then
		report "failed to update svn sources with"
		report "svn switch -r $svn_revision $svnpath/trunk"
		freport $TESTING/svn
		return 1
	    fi
	    ;;

	${svnpath}*)
	    if ! svn switch -r $svn_revision $svn_branch &> $TESTING/svn ; then
		report "failed to update svn sources with"
		report "svn switch -r $svn_revision $svn_branch"
		freport $TESTING/svn
		return 1
	    fi
	    ;;

	*)
	    if ! svn switch -r $svn_revision $svnpath/branches/$svn_branch &> $TESTING/svn ; then
		report "failed to update svn sources with"
		report "svn switch -r $svn_revision $svnpath/branches/$svn_branch"
		freport $TESTING/svn
		return 1
	    fi
	    ;;
    esac
    contrib/gcc_update --touch

    current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
    if [[ $VERSION < $current_version ]]; then
	if [ -f $SOURCE/contrib/patch_tester.sh ]; then
	    selfexec
	fi
    fi

    return 0
}

apply_patch () {
    if [ $nogpg = false ]; then
	if ! gpg --batch --verify $PATCH &> $TESTING/gpgverify ; then
	    report "your patch failed to verify:"
	    freport $TESTING/gpgverify
	    return 1
	fi
    fi

    cd $SOURCE
    if ! patch -p0 < $PATCH &> $TESTING/patching ; then
	report "your patch failed to apply:"
	report "(check that the patch was created at the top level)"
	freport $TESTING/patching
	return 1
    fi

    # Just assume indexes for now -- not really great, but svn always
    # makes them.
    grep "^Index: " $PATCH | sed -e 's/Index: //' | while read file; do
	# If the patch resulted in an empty file, delete it.
	# This is how svn reports deletions.
	if [ ! -s $file ]; then
	    rm -f $file
	    report "Deleting empty file $file"
	fi
    done
}

save_compilers () {
    for COMPILER in $COMPILERS ; do
	if [ -f $BUILD/$COMPILER ]; then
	    cp $BUILD/$COMPILER $PRISTINE
	fi
    done
}

bootntest () {
    rm -rf $BUILD
    mkdir $BUILD
    cd $BUILD

    CONFIG_OPTIONS=`grep "^configure:" $PATCH | sed -e "s/^configure://g"`
    CONFIG_OPTIONS="$default_config $CONFIG_OPTIONS"
    if ! eval $SOURCE/configure $CONFIG_OPTIONS &> $1/configure ; then
	report "configure with `basename $1` version failed with:"
	freport $1/configure
	return 1
    fi

    MAKE_ARGS=`grep "^make:" $PATCH | sed -e "s/^make://g"`
    MAKE_ARGS="$default_make $MAKE_ARGS"
    if ! eval make $dashj $MAKE_ARGS &> $1/bootstrap ; then
	report "bootstrap with `basename $1` version failed with last lines:"
	tail -30 $1/bootstrap > $1/last_bootstrap
	freport $1/last_bootstrap
	report "grep --context=20 Error bootstrap:"
	grep --context=20 Error $1/bootstrap > $1/bootstrap_error
	freport $1/bootstrap_error
	return 1
    fi

    CHECK_OPTIONS=`grep "^check:" $PATCH | sed -e "s/^check://g"`
    CHECK_OPTIONS="$default_check $CHECK_OPTIONS"
    eval make $dashj $CHECK_OPTIONS -k check &> $1/check

    SUITESRUN="`grep 'Summary ===' $1/check | cut -d' ' -f 2 | sort`"
    if [ x$SUITESRUN = x ]; then
	report "check with `basename $1` version failed, no testsuites were run"
	return 1
    fi

    for LOG in $TESTLOGS ; do
	if [ -f $BUILD/$LOG ]; then
	    mv $BUILD/$LOG $1
	    mv `echo "$BUILD/$LOG" | sed -e "s/\.sum/\.log/g"` $1
	fi
    done

    return 0
}

bootntest_patched () {
    cleanup
    mkdir -p $PATCHED
    apply_patch && bootntest $PATCHED
    return $?
}

# Build the pristine tree with exactly the same options as the patch under test.
bootntest_pristine () {
    cleanup
    current_branch=`svn info $SOURCE | grep "^URL:" | sed -e "s/URL: //g" -e "s,${svnpath},,g"`
    current_version=`svn info $SOURCE | grep "^Revision:" | sed -e "s/^Revision://g" -e "s/ //g"`
    PRISTINE=$STATE/$current_branch/$current_version

    if [ $nopristinecache = true ]; then
      rm -rf $PRISTINE
    fi
    if [ -d $PRISTINE ]; then
	ln -s $PRISTINE $TESTING/pristine
	return 0
    else
	mkdir -p $PRISTINE
	ln -s $PRISTINE $TESTING/pristine
	bootntest $PRISTINE
	RETVAL=$?
	if [ $RETVAL = 0 -a $savecompilers = true ]; then
	    save_compilers
	fi
	return $RETVAL
    fi
}

regtest () {
    touch $1/report
    touch $1/passes
    touch $1/failed
    touch $1/regress

    for LOG in $TESTLOGS ; do
	NLOG=`basename $LOG`
	if [ -f $1/$NLOG ]; then
	    awk '/^FAIL: / { print "'$NLOG'",$2; }' $1/$NLOG
	fi
    done | sort | uniq > $1/failed

    comm -12 $1/failed $1/passes >> $1/regress
    NUMREGRESS=`wc -l < $1/regress | tr -d ' '`

    if [ $NUMREGRESS -eq 0 ] ; then
	for LOG in $TESTLOGS ; do
	    NLOG=`basename $LOG`
	    if [ -f $1/$NLOG ] ; then
		awk '/^PASS: / { print "'$NLOG'",$2; }' $1/$NLOG
	    fi
	done | sort | uniq | comm -23 - $1/failed > $1/passes
	echo "there are no regressions with your patch." >> $1/report
    else
	echo "with your patch there are $NUMREGRESS regressions." >> $1/report
	echo "list of regressions with your patch:" >> $1/report
	cat $1/regress >> $1/report
    fi
}

contrib_compare_tests () {
    report "comparing logs with contrib/compare_tests:"
    for LOG in $TESTLOGS ; do
 	NLOG=`basename $LOG`
 	if [ -f $PRISTINE/$NLOG -a -f $PATCHED/$NLOG ]; then
 	    $SOURCE/contrib/compare_tests $PRISTINE/$NLOG $PATCHED/$NLOG > $TESTING/compare_$NLOG
 	    freport $TESTING/compare_$NLOG
 	fi
    done
}

compare_passes () {
    regtest $PRISTINE
    cp $PRISTINE/passes $PATCHED
    regtest $PATCHED
    freport $PATCHED/report
    report "FAILs with patched version:"
    freport $PATCHED/failed
    report "FAILs with pristine version:"
    freport $PRISTINE/failed

    # contrib_compare_tests
}

write_report () {
    backup_patched=$STATE/patched/`now`
    report "The files used for the validation of your patch are stored in $backup_patched on the tester machine."

    EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
    if [ x$EMAIL != x ]; then
	mutt -s "[regtest] Results for `basename $PATCH` on $TARGET" -i $REPORT -a $PATCH $EMAIL
    fi

    mv $TESTING $backup_patched
}

announce () {
    EMAIL=`grep "^email:" $PATCH | sed -e "s/^email://g" -e "s/ //g"`
    if [ x$EMAIL != x ]; then

	START_REPORT=$TESTING/start_report
	echo "Hi, " >> $START_REPORT
	echo "I'm the automatic tester running on $TARGET." >> $START_REPORT
	echo "I just started to look at your patch `basename $PATCH`." >> $START_REPORT
	echo "Bye, your automatic tester." >> $START_REPORT
	mutt -s "[regtest] Starting bootstrap for `basename $PATCH` on $TARGET" -i $START_REPORT $EMAIL
    fi
}

# After selfexec, $TESTING is already set up.  
if [ -d $TESTING ]; then
    # The only file in $TESTING is the patch.
    PATCH=`ls -rt -1 $TESTING | head -1`
    PATCH=$TESTING/$PATCH
    if [ -f $PATCH ]; then
	bootntest_patched && bootntest_pristine && compare_passes
	write_report
    fi
fi

firstpatch=true
while true; do
    PATCH=`ls -rt -1 $PATCHES | head -1`
    if [ x$PATCH = x ]; then
	if [ $stop = true ]; then
	    if [ $firstpatch = true ]; then
		echo "No patches ready to test, quitting."
		exit 1
	    else
		echo "No more patches to test."
		exit 0
	    fi
	fi
	sleep ${standby}m
    else
	firstpatch=false
	sysload=`uptime | cut -d, -f 5`
	if [[ $sysload > $watermark ]]; then
	    # Wait a bit when system load is too high.
	    sleep ${standby}m
	else
	    mkdir -p $TESTING
	    mv $PATCHES/$PATCH $TESTING/
	    PATCH=$TESTING/$PATCH

	    announce
	    update && bootntest_patched && bootntest_pristine && compare_passes
	    write_report
	fi
    fi
done