Skip to content

Commit

Permalink
Integrate wilrich/Py3K-TestDriver into master
Browse files Browse the repository at this point in the history
  • Loading branch information
Project Philly committed Mar 17, 2016
2 parents 6e9ed7c + 1dcae06 commit f711b66
Showing 1 changed file with 55 additions and 42 deletions.
97 changes: 55 additions & 42 deletions Tests/EndToEndTests/TestDriver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,13 @@
# matching against all test-cases/pattern simulteneously
#

import sys, os, argparse, traceback, yaml, subprocess, random, re, time, sets
import sys, os, argparse, traceback, yaml, subprocess, random, re, time

try:
import six
except ImportError:
print("Python package 'six' not installed. Please run 'pip install six'.")
sys.exit(1)

thisDir = os.path.dirname(os.path.realpath(__file__))
windows = os.getenv("OS")=="Windows_NT"
Expand Down Expand Up @@ -128,13 +134,13 @@ def __init__(self, suite, name, pathToYmlFile):

# parsing test cases
self.testCases = []
if "testCases" in self.rawYamlData.keys():
if "testCases" in list(self.rawYamlData.keys()):
testCasesYaml = self.rawYamlData["testCases"]
for name in testCasesYaml.keys():
for name in list(testCasesYaml.keys()):
try:
self.testCases.append(TestCase(name, testCasesYaml[name]))
except Exception as e:
print >>sys.stderr, "ERROR registering test case: " + name
six.print_("ERROR registering test case: " + name, file=sys.stderr)
raise

# parsing all tags, example input:
Expand All @@ -157,7 +163,7 @@ def __init__(self, suite, name, pathToYmlFile):
try:
assert(type(predicate(flavor='foo', device='bar', os='foobar', build_sku='qux')) == bool)
except Exception as e:
print ("Can't parse tag predicate expression in {0} ({1}):\n{2}".format(pathToYmlFile, pythonExpr, e))
six.print_("Can't parse tag predicate expression in {0} ({1}):\n{2}".format(pathToYmlFile, pythonExpr, e))
raise e

# saving generated lambda into tags dictionary
Expand All @@ -178,7 +184,7 @@ def discoverAllTests():
test = Test(suiteName, testName, dirName + "/testcases.yml")
Test.allTestsIndexedByFullName[test.fullName.lower()] = test
except Exception as e:
print >>sys.stderr, "ERROR registering test: " + dirName
six.print_("ERROR registering test: " + dirName, file=sys.stderr)
traceback.print_exc()
sys.exit(1)

Expand Down Expand Up @@ -208,7 +214,7 @@ def runImpl(self, flavor, device, args):
with open(baselineFile, "r") as f:
baseline = f.read().split("\n")
if args.verbose:
print ("Baseline: " + baselineFile)
six.print_("Baseline: " + baselineFile)

# Before running the test, pre-creating TestCaseRunResult object for each test case
# and compute filtered lines from baseline file.
Expand Down Expand Up @@ -244,6 +250,8 @@ def runImpl(self, flavor, device, args):
break
os.environ["TEST_CNTK_BINARY"] = tempPath
os.environ["MPI_BINARY"] = "mpiexec"
if not os.path.exists(os.environ["TEST_CNTK_BINARY"]):
raise ValueError("the cntk executable does not exist at path '%s'"%os.environ["TEST_CNTK_BINARY"])
os.environ["TEST_DIR"] = self.testDir
os.environ["TEST_DATA_DIR"] = self.dataDir
os.environ["TEST_RUN_DIR"] = runDir
Expand All @@ -255,7 +263,7 @@ def runImpl(self, flavor, device, args):
logFile = os.path.join(runDir, "output.txt")
allLines = []
if args.verbose:
print (self.fullName + ":>" + logFile)
six.print_(self.fullName + ":>" + logFile)
with open(logFile, "w") as output:
cmdLine = ["bash", "-c", self.testDir + "/run-test 2>&1"]
process = subprocess.Popen(cmdLine, stdout=subprocess.PIPE)
Expand All @@ -269,13 +277,13 @@ def runImpl(self, flavor, device, args):
line=line[:len(line)-1]

if args.verbose:
print (self.fullName + ": " + line)
six.print_(self.fullName + ": " + line)

if args.dry_run:
print (line)
continue

print >>output, line
six.print_(line, file=output)
allLines.append(line)
output.flush()
for testCaseRunResult in result.testCaseRunResults:
Expand All @@ -290,7 +298,7 @@ def runImpl(self, flavor, device, args):
# checking exit code
if exitCode != 0:
if args.dry_run:
print "[SKIPPED]"
six.print_("[SKIPPED]")
return result
else:
return TestRunResult.fatalError("Exit code must be 0", "==> got exit code {0} when running: {1}".format(exitCode, " ".join(cmdLine)), logFile = logFile)
Expand All @@ -301,7 +309,7 @@ def runImpl(self, flavor, device, args):
if not testCaseRunResult.succeeded:
result.succeeded = False

if (self.testCases)>0 and args.update_baseline and result.succeeded:
if len(self.testCases)>0 and args.update_baseline and result.succeeded:
# When running in --update-baseline mode
# verifying that new output is succesfully matching every pattern in the testcases.yml
# If this is not the case then baseline update will be rejected
Expand All @@ -313,7 +321,7 @@ def runImpl(self, flavor, device, args):

if result.succeeded:
if args.verbose:
print ("Updating baseline file " + baselineFile)
six.print_("Updating baseline file " + baselineFile)
with open(baselineFile, "w") as f:
f.write("\n".join(allLines))

Expand Down Expand Up @@ -343,7 +351,7 @@ def findBaselineFile(self, flavor, device):
def matchesTag(self, tag, flavor, device, os, build_sku):
tagL = tag.lower() # normalizing the tag for comparison
# enumerating all the tags
for tag in self.tags.keys():
for tag in list(self.tags.keys()):
# match by direct string comparison or by prefix matching rule:
# e.g: 'bvt' matches 'bvt' 'bvt-a', 'bvt-b' but not 'bvtx'
if tag==tagL or tag.startswith(tagL + "-"):
Expand All @@ -362,7 +370,7 @@ def __init__(self, name, yamlNode):
try:
self.patterns.append(TestPattern(pattern))
except Exception as e:
print >>sys.stderr, "ERROR registering pattern: " + pattern
six.print_("ERROR registering pattern: " + pattern, file=sys.stderr)
raise

# Processes the baseline file and return an instance of TestCaseRunResult
Expand Down Expand Up @@ -403,8 +411,8 @@ def processLine(self, line, result, verbose):
"Output: {1}\n"
).format(expected, line)
if verbose:
print ("[FAILED]: Testcase " + self.name)
print ("Baseline: " + expected)
six.print_("[FAILED]: Testcase " + self.name)
six.print_("Baseline: " + expected)

# also show all failed patterns
for p in failedPatterns:
Expand Down Expand Up @@ -482,11 +490,16 @@ def patternParse(self, match):
# Checks wether given line matches this pattern
# returns True or False
def match(self, line):
if type(line) == bytes:
line = line.decode("utf-8")
return self.regex.match(line) != None

# Compares a line from baseline log and a line from real output against this pattern
# return true or false
def compare(self, expected, actual):
if type(actual) == bytes:
actual = actual.decode("utf-8")
#import pdb;pdb.set_trace()
em = self.regex.match(expected)
am = self.regex.match(actual)
if em == None and am == None:
Expand Down Expand Up @@ -531,7 +544,7 @@ def __init__(self, testCaseName, succeeded, diagnostics = None):
# Lists all available tests
def listCommand(args):
testsByTag = {}
for test in Test.allTestsIndexedByFullName.values():
for test in list(Test.allTestsIndexedByFullName.values()):
for flavor in args.flavors:
for device in args.devices:
for os in args.oses:
Expand All @@ -540,15 +553,15 @@ def listCommand(args):
continue
tag = test.matchesTag(args.tag, flavor, device, os, build_sku) if args.tag else '*'
if tag:
if tag in testsByTag.keys():
if tag in list(testsByTag.keys()):
testsByTag[tag].add(test.fullName)
else:
testsByTag[tag] = sets.Set([test.fullName])
testsByTag[tag] = set([test.fullName])
for tag in sorted(testsByTag.keys()):
if tag=="*":
print (' '.join(sorted(testsByTag[tag])))
six.print_(' '.join(sorted(testsByTag[tag])))
else:
print (tag + ": " + ' '.join(sorted(testsByTag[tag])))
six.print_(tag + ": " + ' '.join(sorted(testsByTag[tag])))

# Runs given test(s) or all tests
def runCommand(args):
Expand All @@ -560,23 +573,23 @@ def runCommand(args):
if name.lower() in Test.allTestsIndexedByFullName:
testsToRun.append(Test.allTestsIndexedByFullName[name.lower()])
else:
print >>sys.stderr, "ERROR: test not found", name
six.print_("ERROR: test not found", name, file=sys.stderr)
return 1
else:
testsToRun = Test.allTestsIndexedByFullName.values()
testsToRun = list(Test.allTestsIndexedByFullName.values())

devices = args.devices
flavors = args.flavors

os.environ["TEST_ROOT_DIR"] = os.path.dirname(os.path.realpath(sys.argv[0]))

print ("CNTK Test Driver is started")
print ("Running tests: " + " ".join([y.fullName for y in testsToRun]))
print ("Build location: " + args.build_location)
print ("Build SKU: " + args.build_sku)
print ("Run location: " + args.run_dir)
print ("Flavors: " + " ".join(flavors))
print ("Devices: " + " ".join(devices))
six.print_("Running tests: " + " ".join([y.fullName for y in testsToRun]))
six.print_("Build location: " + args.build_location)
six.print_("Build SKU: " + args.build_sku)
six.print_("Run location: " + args.run_dir)
six.print_("Flavors: " + " ".join(flavors))
six.print_("Devices: " + " ".join(devices))
if (args.update_baseline):
print ("*** Running in automatic baseline update mode ***")
print ("")
Expand Down Expand Up @@ -612,30 +625,30 @@ def runCommand(args):
if result.succeeded:
succeededCount = succeededCount + 1
# in no-verbose mode this will be printed in the same line as 'Running test...'
print ("[OK] {0:.2f} sec".format(result.duration))
six.print_("[OK] {0:.2f} sec".format(result.duration))
else:
print ("[FAILED] {0:.2f} sec".format(result.duration))
six.print_("[FAILED] {0:.2f} sec".format(result.duration))
# Showing per-test-case results:
for testCaseRunResult in result.testCaseRunResults:
if testCaseRunResult.succeeded:
# Printing 'OK' test cases only in verbose mode
if (args.verbose):
print(" [OK] " + testCaseRunResult.testCaseName);
six.print_(" [OK] " + testCaseRunResult.testCaseName)
else:
# 'FAILED' + detailed diagnostics with proper indendtation
print(" [FAILED] " + testCaseRunResult.testCaseName);
six.print_(" [FAILED] " + testCaseRunResult.testCaseName)
if testCaseRunResult.diagnostics:
for line in testCaseRunResult.diagnostics.split('\n'):
print " " + line;
six.print_(" " + line);
# In non-verbose mode log wasn't piped to the stdout, showing log file path for conveniencce

if not result.succeeded and not args.verbose and result.logFile:
print (" See log file for details: " + result.logFile)
six.print_(" See log file for details: " + result.logFile)

if args.update_baseline:
print ("{0}/{1} baselines updated, {2} failed".format(succeededCount, totalCount, totalCount - succeededCount))
six.print_("{0}/{1} baselines updated, {2} failed".format(succeededCount, totalCount, totalCount - succeededCount))
else:
print ("{0}/{1} tests passed, {2} failed".format(succeededCount, totalCount, totalCount - succeededCount))
six.print_("{0}/{1} tests passed, {2} failed".format(succeededCount, totalCount, totalCount - succeededCount))
if succeededCount != totalCount:
sys.exit(10)

Expand Down Expand Up @@ -684,23 +697,23 @@ def runCommand(args):
if (args.device):
args.device = args.device.lower()
if not args.device in args.devices:
print >>sys.stderr, "--device must be one of", args.devices
six.print_("--device must be one of", args.devices, file=sys.stderr)
sys.exit(1)
args.devices = [args.device]

args.flavors = ["debug", "release"]
if (args.flavor):
args.flavor = args.flavor.lower()
if not args.flavor in args.flavors:
print >>sys.stderr, "--flavor must be one of", args.flavors
six.print_("--flavor must be one of", args.flavors, file=sys.stderr)
sys.exit(1)
args.flavors = [args.flavor]

args.buildSKUs = ["cpu", "gpu", "1bitsgd"]
if (args.build_sku):
args.build_sku = args.build_sku.lower()
if not args.build_sku in args.buildSKUs:
print >>sys.stderr, "--build-sku must be one of", args.buildSKUs
six.print_("--build-sku must be one of", args.buildSKUs, file=sys.stderr)
sys.exit(1)
args.buildSKUs = [args.build_sku]
if args.build_sku == "cpu" and args.devices == ["gpu"]:
Expand All @@ -715,7 +728,7 @@ def runCommand(args):
if (args.os):
args.os = args.os.lower()
if not args.os in args.oses:
print >>sys.stderr, "--os must be one of", args.oses
six.print_("--os must be one of", args.oses, file=sys.stderr)
sys.exit(1)
args.oses = [args.os]

Expand Down

0 comments on commit f711b66

Please sign in to comment.