build-and-test

#!/bin/bash
#
# Copyright (c) Microsoft. All rights reserved.
#
# Licensed under the MIT license. See LICENSE.md file in the project root
# for full license information.
# ==============================================================================
#
# TODO --math-library support on Windows

# Setting some default values
BUILD=1
RUN=1
CLEAN_AFTER=0
CLEAN_BEFORE=0
RANDOM_OUTPUT=0
CODE_COVERAGE=no
FLAVORS="debug:release"
TARGETS="cpu:gpu"
MATH_LIBRARY="mkl"
TESTTARGETS="cpu:gpu"
EXTRA_CONFIGURE_OPTIONS=

# parsing command line arguments:
while [[ $# > 0 ]]
do
key="$1"

case $key in
    -h|--help)
    echo "Usage: build-and-test [options]"
    echo "Options:"
    echo "  -q|--quiet-build - redirect build output to file"
    echo "  -r|--run-only - elides build step, runs the binaries that have already been built"
    echo "  -b|--build-only - just build, do not run"
    echo "  -f|--flavors <flavor1:flavor2...> - which flavor to build (by default $FLAVORS)"
    echo "  -t|--targets <target1:target2...> - which target to build (by default $TARGETS)"
    echo "  -m|--math-library <mathlibrary> - which math library to build with (by default $MATH_LIBRARY)"
    echo "  -tt|--test-targets <testtarget1:testtarget2...> - which target to test (by default $TESTTARGETS)"
    echo "  -cc|--code-coverage - build with support for code coverage (gcov)"
    echo "  -cb|--clean-build - clean up the enlistment binaries before build"
    echo "  -cba|--clean-build-after - clean up the enlistment binaries after build"
    echo "  -rnd|--random-output-suffix - add random suffix to output directory"
    echo "  -o|--output-directory <output_dir> - specify output directory to use (by default those will be in <cntk_root>.run-<operating_system>)"
    echo "  -x|--extra-configure-options <options> - extra options to pass to configure"
    echo "The root directory used to build and run CNTK is hosts the Scripts directory that contains this script"
    exit 1
    ;;
    -q|--quiet)
    QUIET_BUILD=1
    ;;
    -r|--run-only)
    BUILD=0
    RUN=1
    ;;
    -rnd|--random-output-suffix)
    RANDOM_OUTPUT=1
    ;;
    -b|--build-only)
    BUILD=1
    RUN=0
    ;;
    -cb|--clean-build)
    CLEAN_BEFORE=1
    BUILD=1
    ;;
    -cba|--clean-build-after)
    CLEAN_AFTER=1
    BUILD=1
    ;;
    -f|--flavors)
    FLAVORS="${2,,}"
    shift # past argument
    ;;
    -t|--targets)
    TARGETS="${2,,}"
    shift # past argument
    ;;
    -m|--math-library)
    case ${2,,} in
        mkl)
        MATH_LIBRARY_OPTION="--with-mkl=$MKL_PATH"
        ;;
        mkl-sequential)
        MATH_LIBRARY_OPTION="--with-mkl-sequential=$MKL_PATH"
        ;;
        *)
        echo Unknown math library $MATH_LIBRARY
        exit 1
        ;;
    esac
    shift # past argument
    ;;
    -tt|--test-targets)
    TESTTARGETS="${2,,}"
    shift # past argument
    ;;
    -cc|--code-coverage)
    CODE_COVERAGE=yes
    ;;
    -x|--extra-configure-options)
    EXTRA_CONFIGURE_OPTIONS="$2"
    shift # past argument
    ;;
    -o|--output-directory)
    OUTPUT_DIR="$2"
    shift # past argument
    ;;
    *)
    echo Unknown option $key
    exit 1
    ;;
esac
shift # past argument or value
done

# Step 0 -- Validate all necessary prerequisites and check for incompatible options
# It is possible to use this script on Windows to build CNTK
# from Cygwin window with Visual C++ environment loaded.
# In that case OS environment variable will be set and we
# can use it to differentiate from Linux.
if [[ $CLEAN_BEFORE == 1 && $RUN == 1 && $BUILD == 0 ]]; then
    echo "============ ERROR: Incompatible options RUN and CLEAN_BEFORE set without BUILD ============"
    exit 1
fi

if [[ $OS == "Windows_NT" && $OSTYPE == "cygwin" ]]; then
    DEBUG_DIR=Debug
    RELEASE_DIR=Release
    PREFIX_DIR=x64
    BIN_NAME=CNTK.exe
    BUILD_OS="windows"

    if [[ $VS120COMNTOOLS == "" ]]; then
        echo "============ Visual Studio 12.0 environment not properly setup or VS not installed ============"
        echo "============ Please find and run the appropriate vcvarsall.bat script ============"
        exit 1
    fi

    if [[ "${TARGETS,,}" =~ "1bitsgd" && "${TARGETS,,}" =~ "gpu" ]]; then
        echo "============ Cannot specify both GPU and 1bit-SGD as targets, please choose one ============"
        exit 1
    fi

elif [[ $OSTYPE == "linux-gnu" ]]; then
    DEBUG_DIR=build/$TARGET/debug
    RELEASE_DIR=build/$TARGET/release
    PREFIX_DIR=
    # Make sure no dependencies on current directory
    BIN_NAME=bin/cntk
    MAKEFILE=Makefile
    BUILD_OS="linux"
else
    echo "============ ERROR: Unsupported OS ============"
    echo "============ Script supports only building from Linux and Windows through Cygwin ============"
    exit 1
fi

# Step 1 -- Prepare temporary folders and files, tweak settings if necessary

# Get to the root path from which we know how to build and run
SCRIPT=`readlink -f $0`
SCRIPT_DIR=`dirname $SCRIPT`
CNTK_ROOT=`dirname $SCRIPT_DIR`

# Setup the output directory
if [[ $OUTPUT_DIR == "" ]]; then
    OUTPUT_DIR="$CNTK_ROOT/.run-$BUILD_OS"
fi

# Add random number at the end of the output directory to prevent overwriting previous results
if [[ $RANDOM_OUTPUT == 1 ]]; then
    OUTPUT_DIR="$OUTPUT_DIR-$RANDOM"
fi

echo "============ Creating CNTK temp directory in $OUTPUT_DIR ============"
mkdir -p $OUTPUT_DIR || exit $?

CONF_FILE="$OUTPUT_DIR/Simple.cntk"
BUILD_FILE="$OUTPUT_DIR/Build"
RUN_FILE="$OUTPUT_DIR/Result"

if ! [[ -d "$CNTK_ROOT/Source" ]]; then
    echo "============ ERROR: Build script located in the wrong directory ($SCRIPT_DIR) ============"
    exit 1
fi

cd $CNTK_ROOT

if ! [[ -f $CONF_FILE ]]; then
    cp Examples/Other/Simple2d/Config/Simple.cntk $CONF_FILE || exit $?

    # This chmod is necessary due to restrictive Cygwin interpretation of Windows permissions.
    # Cygwin interprets Windows permissions as ----rwx---, which lacks read permissions for user.
    chmod a+r $CONF_FILE || exit $?
fi

if [[ $QUIET_BUILD == 1 ]]; then
    echo "============ WARNING: You have selected quiet build. All build output will be placed in ($OUTPUT_DIR) ============"
fi

# Initialize lists of flavors and targets
flavorArray=(${FLAVORS//:/ })
targetArray=(${TARGETS//:/ })
testTargetArray=(${TESTTARGETS//:/ })

# Step 2 -- Build the project for all requested flavors and targets
if [[ $BUILD == 1 ]]; then
    for FLAVOR in "${flavorArray[@]}"
    do
        for TARGET in "${targetArray[@]}"
        do
            echo "============ Building CNTK $TARGET/$FLAVOR (clean=$CLEAN_BEFORE)  ============"

            # Our make is too noisy right now and it is difficult to spot
            # issues from stdout and stderr. In the quiet mode these are
            # redirected to a file where they could be examined after the fact
            if [[ $QUIET_BUILD == 1 ]]; then
                exec 6>$BUILD_FILE.$FLAVOR.out || exit $?
                exec 7>$BUILD_FILE.$FLAVOR.err || exit $?
            else
                exec 6>&1 || exit $?
                exec 7>&2 || exit $?
            fi

            if [[ $OS == "Windows_NT" ]]; then
                OneBitSGDOPT=/property:CNTK_ENABLE_1BitSGD=
                if [[ $TARGET == "cpu" ]]; then
                    CONFIGURATION=${FLAVOR}_CpuOnly
                else
                    CONFIGURATION=${FLAVOR}
                    if [[ $TARGET == "1bitsgd" ]]; then
                        OneBitSGDOPT=/property:CNTK_ENABLE_1BitSGD=true
                    fi
                fi

                if [[ $CLEAN_BEFORE == 1 ]]; then
                    msbuild.exe /nologo /verbosity:m /property:Configuration=$CONFIGURATION /t:Clean 1>&6 2>&7 || exit $?
                fi
                msbuild.exe /nologo /verbosity:m /m /property:Configuration=$CONFIGURATION $OneBitSGDOPT 1>&6 2>&7 || exit $?
            else
                OneBitSGDOPT=no
                BUILD_DIR=build/$TARGET/$FLAVOR
                if [[ $TARGET == "cpu" ]]; then
                    CUDAOPT=no
                else
                    CUDAOPT=yes
                    if [[ $TARGET == "1bitsgd" ]]; then
                        OneBitSGDOPT=yes
                    fi
                fi
                ./configure --with-build-top=$BUILD_DIR ${MATH_LIBRARY_OPTION} --with-buildtype=$FLAVOR --cuda=$CUDAOPT --with-code-coverage=$CODE_COVERAGE --1bitsgd=$OneBitSGDOPT $EXTRA_CONFIGURE_OPTIONS
                if [[ $CLEAN_BEFORE == 1 ]]; then
                    make -C $BUILD_DIR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
                fi
                make -C $BUILD_DIR -j `nproc` -f $MAKEFILE 1>&6 2>&7 || exit $?

            fi
            if [[ $QUIET_BUILD == 1 ]]; then
              chmod a+r $BUILD_FILE.*
            fi
        done
    done
fi

# Step 3 -- Run the project tests, both debug and release, if requested
if [[ $RUN == 1 ]]; then

    cd $PREFIX_DIR
    echo "============ cp Examples/Other/Simple2d/Config/Simple.cntk $CONF_FILE  ============"
    echo "============ cd $CNTK_ROOT/Examples/Other/Simple2d/Data  ============"

    for FLAVOR in "${flavorArray[@]}"
    do
        for TARGET in "${targetArray[@]}"
        do
            for TESTTARGET in "${testTargetArray[@]}"
            do
                # Determine how to set the deviceId parameter in the configuration.
                # 0 will pick the first GPU (or fail. -1 will pick the CPU.
                if [[ $TESTTARGET == gpu ]]; then
                    DEVICE_ID=0
                else
                    DEVICE_ID=-1
                fi

                if [[ $TESTTARGET == gpu && $TARGET == cpu ]]; then
                    # CPU-only builds cannot run GPU test targets
                    continue;
                fi

                if [[ $OSTYPE == "linux-gnu" ]]; then
                    FLAVOR_DIR=build/$TARGET/$FLAVOR
                else
                    if [[ $FLAVOR == "debug" ]]; then
                        FLAVOR_DIR="$DEBUG_DIR"
                    else
                        FLAVOR_DIR="$RELEASE_DIR"
                    fi
                    if [[ $TARGET == cpu ]]; then
                        FLAVOR_DIR="${FLAVOR}_CpuOnly"
                    fi
                fi
                OUT_FILE="$RUN_FILE.$FLAVOR.$TARGET.$TESTTARGET.out"

                BIN_PATH=$CNTK_ROOT/$PREFIX_DIR/$FLAVOR_DIR/$BIN_NAME
                if ! [[ -f $BIN_PATH ]]; then
                    echo "============ ERROR: CNTK did not build properly for $TARGET/$FLAVOR ============"
                    echo "Missing file: $BIN_PATH"
                    exit 1
                fi

                if [[ $OS == "Windows_NT" ]]; then
                    echo "============ Running $BIN_PATH configFile=`cygpath -w $CONF_FILE` for $TARGET/$FLAVOR (on $TESTTARGET) ============"
                else
                    echo "============ Running $BIN_PATH configFile=$CONF_FILE for $TARGET/$FLAVOR (on $TESTTARGET) ============"
                fi
                echo "============ output in ($OUT_FILE) ============"
                cd $CNTK_ROOT/Examples/Other/Simple2d/Data
                rm -rf "$OUTPUT_DIR/Models"
                if [[ $OS == "Windows_NT" ]]; then
                    # We have to use cygpath on Windows to modify the file paths into the format readable by cntk.
                    time $BIN_PATH configFile="`cygpath -w $CONF_FILE`" deviceId=$DEVICE_ID OutputDir="`cygpath -w $OUTPUT_DIR`" &>$OUT_FILE || exit $?
                else
                    time $BIN_PATH configFile=$CONF_FILE deviceId=$DEVICE_ID OutputDir="$OUTPUT_DIR" &>$OUT_FILE || exit $?
                fi
                chmod a+r $RUN_FILE.*

                # Check if execution was successful
                grep -q "Using ${TESTTARGET^^}" "$OUT_FILE" || {
                    echo "============ ERROR: Run output (in $OUT_FILE) did not contain information about target device ($TESTTARGET) ============"
                    exit 1
                }

                grep -q "EXCEPTION" "$OUT_FILE" && {
                    echo "============ ERROR: Run output in ($OUT_FILE) contains exceptions ============"
                    grep "EXCEPTION" "$OUT_FILE"
                    exit 1
                }
            done
        done
    done
fi

# Step 5 -- Optionally clean after builds and tests
if [[ $CLEAN_AFTER == 1 ]]; then
    cd $CNTK_ROOT
    for FLAVOR in "${flavorArray[@]}"
    do
        for TARGET in "${targetArray[@]}"
        do
            echo "============ Cleaning up CNTK $TARGET/$FLAVOR  ============"
            if [[ $OS == "Windows_NT" ]]; then
                if [[ $TARGET == "cpu" ]]; then
                    CONFIGURATION=${FLAVOR}_CpuOnly
                else
                    CONFIGURATION=${FLAVOR}
                fi
                msbuild.exe /nologo /verbosity:m /property:Configuration=$CONFIGURATION /t:clean 1>&6 2>&7 || exit $?
            else
                make -C build/$TARGET/$FLAVOR -f $MAKEFILE clean 1>&6 2>&7 || exit $?
            fi
        done
    done
    rm -rf "$OUTPUT_DIR"
fi

echo "============ Build and test of CNTK was successful!  ============"