From 5387f3b258e38d23b9d0ba2e52c0baebef47fafd Mon Sep 17 00:00:00 2001 From: Mike Richter Date: Mon, 18 Apr 2016 11:16:26 -0400 Subject: [PATCH] Improve integraton tests (multiple changes) #14682 Add test-suite.sh script to call individual test scripts. Create safe wrappers for destructive methods rm and rsync. Use common.sh in each of the test scripts. Verify the original app directory is clean before testing. --- test/tests/common.sh | 46 ++++++ .../test-double-obfuscation-protection.sh | 139 ++-------------- test/tests/test-filters-and-exclusions.sh | 150 +++--------------- test/tests/test-new-options.sh | 11 +- test/tests/test-suite.sh | 7 + 5 files changed, 97 insertions(+), 256 deletions(-) create mode 100755 test/tests/test-suite.sh diff --git a/test/tests/common.sh b/test/tests/common.sh index 9f08a906..269a7eae 100644 --- a/test/tests/common.sh +++ b/test/tests/common.sh @@ -185,3 +185,49 @@ toList() { cat "${source}" | sed 's|[",]||g' | awk '{ print $3; }' | sort | grep -v '^$' > "${destination}" fi } + +rsyncInSandbox() { + if test $# -lt 2 + then + echo "$(basename $0): rsyncInSandbox [options] " >&2 + echo " Review help documentation for rsync." >&2 + exit 1 + fi + + if [[ "${@: -1}" != */sandbox/* ]] + then + echo "$(basename $0): rsyncInSandbox: destination must contain 'sandbox' path part" >&2 + echo " destination: ${@: -1}" >&2 + exit 2 + fi + + rsync "$@" +} + +rmFromSandbox() { + if test $# -ne 1 + then + echo "$(basename $0): rmFromSandbox " >&2 + echo " Only supports removing one directory at a time from the sandbox." >&2 + exit 1 + fi + + if [[ "$1" != */sandbox/* ]] + then + echo "$(basename $0): rmFromSandbox: directory must contain 'sandbox' path part" >&2 + echo " directory: $1" >&2 + exit 2 + fi + + rm -r -- "$1" +} + +checkOriginalIsClean() { + if test "${original}" != "" \ + && test "${buildDir}" != "" \ + && test -e "${original}/${buildDir}" + then + echo "Original directory is not clean: ${original}/${buildDir}" >&2 + exit 1 + fi +} diff --git a/test/tests/test-double-obfuscation-protection.sh b/test/tests/test-double-obfuscation-protection.sh index 3bd25ad1..387a614f 100755 --- a/test/tests/test-double-obfuscation-protection.sh +++ b/test/tests/test-double-obfuscation-protection.sh @@ -1,142 +1,35 @@ #!/bin/bash -testRoot="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")" -sandbox="${testRoot}/sandbox" -apps="${testRoot}/apps" -results="${testRoot}/results" +targetAppName=BoxSim +thisDirectory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +testRoot="$(dirname "${thisDirectory}")" +. "${testRoot}/tests/common.sh" -#echo "testRoot=${testRoot}" -#echo "sandbox=${sandbox}" -#echo "apps=${apps}" -test -e "${sandbox}" || mkdir -p "${sandbox}" -test -e "${results}" || mkdir -p "${results}" - -targetName=BoxSim - -original="${apps}/${targetName}" -work="${sandbox}/${targetName}" -lastRun="${results}/run.log" +original="${apps}/${targetAppName}" +work="${sandbox}/${targetAppName}" buildLog="${results}/build.log" -testLog="${results}/test-suite.log" buildDir=build -testCount=0 -failureCount=0 -errorCount=0 -successCount=0 - -testName="" -error="" -firstSetup=yes - -TEST() { - if test "${firstSetup}" = "yes" - then - firstSetup="" - date > "${testLog}" - else - tearDown # between tests - fi - - testName="$1" - testCount=$((testCount + 1)) - - - echo "Setup:" >> "${testLog}" - rsync -a --delete "${original}/" "${work}" - - cd "${work}" - - - echo -n "Test: ${testName}: " - echo "Test: ${testName}: " >> "${testLog}" +oneTimeSetUp() { + checkOriginalIsClean } -tearDown() { - if test "${testName}" != "" - then - if test "${error}" != "" - then - echo "FAIL" - echo "error: ${error}" - failureCount=$((failureCount + 1)) - else - echo "PASS" - successCount=$((successCount + 1)) - fi - - testName="" - error="" - fi +oneTimeTearDown() { + rmFromSandbox "${work}" } -report() { - tearDown - - echo "Done." - echo "Tests run: ${testCount}, pass: ${successCount}, fail: ${failureCount}" - - if test "${testCount}" -eq 0 - then - echo "error: no tests were executed" >&2 - exit 2 - fi - - if test "${successCount}" -lt "${testCount}" \ - || test "${failureCount}" -gt 0 - then - exit 1 - fi +setUp() { + rsyncInSandbox -a --delete "${original}/" "${work}" + pushd "${work}" > /dev/null } -run() { - echo "$@" >> "${testLog}" - "$@" 2>&1 | tee "${lastRun}" >> "${testLog}" - return "${PIPESTATUS[0]}" -} - -verify() { - echo "verify $@" >> "${testLog}" - if test "${error}" = "" - then - "$@" &> /dev/null - result=$? - if test "${result}" -ne 0 - then - error="\"$@\" (return: ${result})" - fi - fi -} - -verifyFails() { - echo "verifyFails $@" >> "${testLog}" - if test "${error}" = "" - then - "$@" &> /dev/null - result=$? - if test "${result}" -eq 0 - then - error="\"$@\" (expected non-zero)" - fi - fi +tearDown() { + popd > /dev/null } -toList() { - if test $# -lt 2 - then - echo "$(basename $0): toList " >&2 - exit 1 - fi - - source="$1" - destination="$2" - - echo "Writing ${destination}" >> "${testLog}" - cat "${source}" | sed 's|[",]||g' | awk '{ print $3; }' | sort | grep -v '^$' > "${destination}" -} -program="${work}/build/Build/Products/Release-iphoneos/${targetName}.app/${targetName}" +program="${work}/build/Build/Products/Release-iphoneos/${targetAppName}.app/${targetAppName}" TEST "Normal obfuscated build" run make build diff --git a/test/tests/test-filters-and-exclusions.sh b/test/tests/test-filters-and-exclusions.sh index c9e07fb8..db2e4492 100755 --- a/test/tests/test-filters-and-exclusions.sh +++ b/test/tests/test-filters-and-exclusions.sh @@ -1,151 +1,45 @@ #!/bin/bash -testRoot="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")" -sandbox="${testRoot}/sandbox" -apps="${testRoot}/apps" -results="${testRoot}/results" +targetAppName=BoxSim +thisDirectory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +testRoot="$(dirname "${thisDirectory}")" +. "${testRoot}/tests/common.sh" -#echo "testRoot=${testRoot}" -#echo "sandbox=${sandbox}" -#echo "apps=${apps}" -test -e "${sandbox}" || mkdir -p "${sandbox}" -test -e "${results}" || mkdir -p "${results}" - -original="${apps}/BoxSim" -work="${sandbox}/BoxSim" -lastRun="${results}/run.log" +original="${apps}/${targetAppName}" +prepared="${sandbox}/${targetAppName}-pre" +work="${sandbox}/${targetAppName}" buildLog="${results}/build.log" -testLog="${results}/test-suite.log" buildDir=build -testCount=0 -failureCount=0 -errorCount=0 -successCount=0 - -testName="" -error="" -firstSetup=yes - -TEST() { - if test "${firstSetup}" = "yes" - then - firstSetup="" - date > "${testLog}" - rsync -a --delete "${original}/" "${work}" - else - tearDown # between tests - fi - testName="$1" - testCount=$((testCount + 1)) +oneTimeSetUp() { + checkOriginalIsClean + rsyncInSandbox -a --delete "${original}/" "${prepared}" - echo "Setup:" >> "${testLog}" - rsync -a --delete --exclude=build "${original}/" "${work}" + echo "Building ..." + ( cd "${prepared}" ; make build &> "${buildLog}" ) + echo "Done." +} - cd "${work}" +oneTimeTearDown() { + rmFromSandbox "${prepared}" + rmFromSandbox "${work}" +} - if ! test -e "${buildDir}" - then - echo "Building ..." - make build &> "${buildLog}" - echo "Done." - fi +setUp() { + rsyncInSandbox -a --delete "${prepared}/" "${work}" targetApp="$(ls -td $(find "${work}/${buildDir}" -name "*.app") | head -1)" targetAppName="$(echo "${targetApp}" | sed 's,.*/\([^/]*\)\.app,\1,')" program="$(ls -td $(find "${targetApp}" -type f -and -name "${targetAppName}") | head -1)" - #echo "targetApp=${targetApp}" - #echo "targetAppName=${targetAppName}" - #echo "program=${program}" - - echo -n "Test: ${testName}: " - echo "Test: ${testName}: " >> "${testLog}" + pushd "${work}" > /dev/null } tearDown() { - if test "${testName}" != "" - then - if test "${error}" != "" - then - echo "FAIL" - echo "error: ${error}" - failureCount=$((failureCount + 1)) - else - echo "PASS" - successCount=$((successCount + 1)) - fi - - testName="" - error="" - fi -} - -report() { - tearDown - - echo "Done." - echo "Tests run: ${testCount}, pass: ${successCount}, fail: ${failureCount}" - - if test "${testCount}" -eq 0 - then - echo "error: no tests were executed" >&2 - exit 2 - fi - - if test "${successCount}" -lt "${testCount}" \ - || test "${failureCount}" -gt 0 - then - exit 1 - fi -} - -run() { - echo "$@" >> "${testLog}" - "$@" 2>&1 | tee "${lastRun}" >> "${testLog}" -} - -verify() { - echo "verify $@" >> "${testLog}" - if test "${error}" = "" - then - "$@" &> /dev/null - result=$? - if test "${result}" -ne 0 - then - error="\"$@\" (return: ${result})" - fi - fi -} - -verifyFails() { - echo "verifyFails $@" >> "${testLog}" - if test "${error}" = "" - then - "$@" &> /dev/null - result=$? - if test "${result}" -eq 0 - then - error="\"$@\" (expected non-zero)" - fi - fi -} - -toList() { - if test $# -lt 2 - then - echo "$(basename $0): toList " >&2 - exit 1 - fi - - source="$1" - destination="$2" - - echo "Writing ${destination}" >> "${testLog}" - cat "${source}" | sed 's|[",]||g' | awk '{ print $3; }' | sort | grep -v '^$' > "${destination}" + popd > /dev/null } diff --git a/test/tests/test-new-options.sh b/test/tests/test-new-options.sh index 0fe0c140..743f83df 100755 --- a/test/tests/test-new-options.sh +++ b/test/tests/test-new-options.sh @@ -14,7 +14,9 @@ buildDir=build oneTimeSetUp() { - rsync -a --delete "${original}/" "${prepared}" + checkOriginalIsClean + + rsyncInSandbox -a --delete "${original}/" "${prepared}" echo "Building ..." ( cd "${prepared}" ; make build &> "${buildLog}" ) @@ -22,12 +24,12 @@ oneTimeSetUp() { } oneTimeTearDown() { - [[ "${prepared}" == */sandbox/* ]] && rm -r -- "${prepared}" - [[ "${work}" == */sandbox/* ]] && rm -r -- "${work}" + rmFromSandbox "${prepared}" + rmFromSandbox "${work}" } setUp() { - rsync -a --delete "${prepared}/" "${work}" + rsyncInSandbox -a --delete "${prepared}/" "${work}" targetApp="$(ls -td $(find "${work}/${buildDir}" -name "*.app") | head -1)" targetAppName="$(echo "${targetApp}" | sed 's,.*/\([^/]*\)\.app,\1,')" @@ -38,7 +40,6 @@ setUp() { tearDown() { popd > /dev/null - return 0 } checkVersion() { diff --git a/test/tests/test-suite.sh b/test/tests/test-suite.sh new file mode 100755 index 00000000..2bbdcf65 --- /dev/null +++ b/test/tests/test-suite.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +./test-double-obfuscation-protection.sh +./test-filters-and-exclusions.sh +./test-new-options.sh