Added --build-sku option to TestDriver.py to enable filtering tests b…

…ased on build SKUs (aka CPUONLY vs GPU vs 1BitSGD)

Tests/EndToEndTests/Image/QuickE2E/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../Data
 tags:
      # running on every BVT job in 'I' (Image) leg:
-     - bvt-i  device=='gpu'
+     - bvt-i  (build_sku == 'default') and device=='gpu'
      # running every Nightly job in 'I' leg
-     - nightly-i device=='gpu'
+     - nightly-i (build_sku == 'default') and device=='gpu'
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/ParallelTraining/NoQuantization/DoublePrecision/testcases.yml

@@ -2,9 +2,9 @@ dataDir: ../../Data
 tags:
      # running on every BVT job in 'P' (Parallel) leg in Debug-GPU and Release-CPU configurations:
      # Also skipping Release-CPU on linux to save time (for now due to unknown reason it runs much longer comparing to windows)
-     - bvt-p  ((flavor=='debug') ^ (device=='cpu')) and not (os=='linux' and device=='cpu')
+     - bvt-p  ((build_sku == 'default') or (build_sku == '1bitsgd')) and ((flavor=='debug') ^ (device=='cpu')) and not (os=='linux' and device=='cpu')
      # running unconditionally on every Nightly job in 'P' leg
-     - nightly-p
+     - nightly-p ((build_sku == 'default') or (build_sku == '1bitsgd'))
 
 testCases:
   Must train epochs in exactly same order and parameters for each MPI Rank:

Tests/EndToEndTests/ParallelTraining/NoQuantization/SinglePrecision/testcases.yml

@@ -2,9 +2,9 @@ dataDir: ../../Data
 
 tags:
      # running on every BVT job in 'P' (Parallel) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-p  (flavor=='debug') ^ (device=='cpu')
+     - bvt-p  ((build_sku == 'default') or (build_sku == '1bitsgd')) and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'P' leg
-     - nightly-p
+     - nightly-p ((build_sku == 'default') or (build_sku == '1bitsgd'))
 
 testCases:
   Must train epochs in exactly same order and parameters for each MPI Rank:

Tests/EndToEndTests/Speech/DNN/DiscriminativePreTraining/testcases.yml

@@ -7,14 +7,14 @@ tags:
      # is being disabled on the CPU until the retirement of those machines
 
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     # - bvt-s  (flavor=='debug') ^ (device=='cpu')
+     # - bvt-s  (build_sku == 'default') and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'S' leg
-     # - nightly-s
+     # - nightly-s (build_sku == 'default')
 
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU configuration:
-     - bvt-s  (flavor=='debug') and (device=='gpu')
+     - bvt-s  (build_sku == 'default') and ((flavor=='debug') and (device=='gpu'))
      # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s (device=='gpu')
+     - nightly-s (build_sku == 'default') and (device=='gpu')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/Speech/DNN/Parallel1BitQuantization/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../../Data
 tags:
-     # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-s  (flavor=='debug') ^ (device=='cpu')
-     # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s
+     # running for 1bitsgd build SKU on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
+     - bvt-s  (build_sku == '1bitsgd') and ((flavor=='debug') ^ (device=='cpu'))
+     # running for 1bitsgd build SKU  on every Nightly job in 'S' leg
+     - nightly-s (build_sku == '1bitsgd')
 
 testCases:
   Must train epochs in exactly same order and parameters for each MPI Rank:

Tests/EndToEndTests/Speech/DNN/ParallelBufferedAsyncGradientAggregation/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../../Data
 tags:
-     # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-s  (flavor=='debug') ^ (device=='cpu')
-     # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s
+     # running for 1bitsgd build SKU on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
+     - bvt-s  (build_sku == '1bitsgd') and ((flavor=='debug') ^ (device=='cpu'))
+     # running for 1bitsgd build SKU on every Nightly job in 'S' leg
+     - nightly-s (build_sku == '1bitsgd')
 
 testCases:
   Must train epochs in exactly same order and parameters for each MPI Rank:

Tests/EndToEndTests/Speech/DNN/ParallelNoQuantization/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../../Data
 tags:
      # running on every BVT job in 'P' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-p  (flavor=='debug') ^ (device=='cpu')
+     - bvt-p  ((build_sku == 'default') or (build_sku == '1bitsgd')) and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'P' leg
-     - nightly-p
+     - nightly-p ((build_sku == 'default') or (build_sku == '1bitsgd'))
 
 testCases:
   Must train epochs in exactly same order and parameters for each MPI Rank:

Tests/EndToEndTests/Speech/DNN/ParallelNoQuantizationBufferedAsyncGradientAggregation/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../../Data
 tags:
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-s  (flavor=='debug') ^ (device=='cpu')
+     - bvt-s  ((build_sku == 'default') or (build_sku == '1bitsgd')) and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s
+     - nightly-s ((build_sku == 'default') or (build_sku == '1bitsgd'))
 
 testCases:
   Must train epochs in exactly same order and parameters for each MPI Rank:

Tests/EndToEndTests/Speech/DNN/SequenceTraining/testcases.yml

@@ -2,9 +2,9 @@ dataDir: ../../Data
 tags:
      # Note: Sequence training is currently not supported on the CPU
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU configuration:
-     - bvt-s  (flavor=='debug') and (device=='gpu')
+     - bvt-s  (build_sku == 'default') and ((flavor=='debug') and (device=='gpu'))
      # running GPU configuration on every Nightly job in 'S' leg
-     - nightly-s (device=='gpu')
+     - nightly-s (build_sku == 'default') and (device=='gpu')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/Speech/LSTM/FullUtterance/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../../Data
 tags:
      # running on every BVT job in 'L' (LSTM) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-l  (flavor=='debug') ^ (device=='cpu')
+     - bvt-l  (build_sku == 'default') and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'L' leg
-     - nightly-l
+     - nightly-l (build_sku == 'default')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/Speech/LSTM/Truncated/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../../Data
 tags:
      # running on every BVT job in 'L' (LSTM) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-l  (flavor=='debug') ^ (device=='cpu')
+     - bvt-l  (build_sku == 'default') and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'L' leg
-     - nightly-l
+     - nightly-l (build_sku == 'default')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/Speech/QuickE2E/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../Data
 tags:
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-s  (flavor=='debug') ^ (device=='cpu')
+     - bvt-s  (build_sku == 'default') and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s
+     - nightly-s (build_sku == 'default')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/Speech/SVD/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../Data
 tags:
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-s  (flavor=='debug') ^ (device=='cpu')
+     - bvt-s  (build_sku == 'default') and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s
+     - nightly-s (build_sku == 'default')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/Speech/Simple/testcases.yml

@@ -1,9 +1,9 @@
 dataDir: ../Data
 tags:
      # running on every BVT job in 'S' (Speech) leg in Debug-GPU and Release-CPU configurations:
-     - bvt-s  (flavor=='debug') ^ (device=='cpu')
+     - bvt-s  (build_sku == 'default') and ((flavor=='debug') ^ (device=='cpu'))
      # running unconditionally on every Nightly job in 'S' leg
-     - nightly-s
+     - nightly-s (build_sku == 'default')
 
 testCases:
   CNTK Run must be completed:

Tests/EndToEndTests/TestDriver.py

@@ -155,7 +155,7 @@ def __init__(self, suite, name, pathToYmlFile):
         # converting python expression into lambda and doing a smoke test by calling it with dummy parameters
         predicate = lambda pythonExpr=pythonExpr, **kwargs: eval(pythonExpr, kwargs)
         try:
-          assert(type(predicate(flavor='foo', device='var', os='foobar')) == bool)
+          assert(type(predicate(flavor='foo', device='bar', os='foobar', build_sku='qux')) == bool)
         except Exception as e:
           print "Can't parse tag predicate expression in {0} ({1}):\n{2}".format(pathToYmlFile, pythonExpr, e)
           raise e
@@ -331,15 +331,15 @@ def findBaselineFile(self, flavor, device):
 
   # Checks whether the test matches the specified tag,
   # returns matched tag name on succes, or None if there is no match(boolean, string) tuple
-  def matchesTag(self, tag, flavor, device, os):
+  def matchesTag(self, tag, flavor, device, os, build_sku):
     tagL = tag.lower() # normalizing the tag for comparison
     # enumerating all the tags
     for tag in self.tags.keys():
       # match by direct string comparison or by prefix matching rule: 
       # e.g: 'bvt' matches 'bvt' 'bvt-a', 'bvt-b' but not 'bvtx'
       if tag==tagL or tag.startswith(tagL + "-"):
         # evaluating tag's predicate
-        if self.tags[tag](flavor=flavor, device=device, os=os):
+        if self.tags[tag](flavor=flavor, device=device, os=os, build_sku=build_sku):
           return tag
     return None
 
@@ -526,12 +526,13 @@ def listCommand(args):
      for flavor in args.flavors:
         for device in args.devices:
            for os in args.oses:
-             tag = test.matchesTag(args.tag, flavor, device, os) if args.tag else '*'
-             if tag:
-               if tag in testsByTag.keys():
-                 testsByTag[tag].add(test.fullName)
-               else:
-                 testsByTag[tag] = sets.Set([test.fullName])
+             for build_sku in args.buildSKUs:
+               tag = test.matchesTag(args.tag, flavor, device, os, build_sku) if args.tag else '*'
+               if tag:
+                 if tag in testsByTag.keys():
+                   testsByTag[tag].add(test.fullName)
+                 else:
+                   testsByTag[tag] = sets.Set([test.fullName])
   for tag in sorted(testsByTag.keys()):
     if tag=="*":
       print ' '.join(sorted(testsByTag[tag]))
@@ -631,10 +632,13 @@ def runCommand(args):
                          "If not specified then all tests will be run.")
 defaultBuildLocation=os.path.realpath(os.path.join(thisDir, "../..", "x64" if windows else "build"))
 
+defaultBuildSKU = "default"
+
 runSubparser.add_argument("-b", "--build-location", default=defaultBuildLocation, help="location of the CNTK build to run")
 runSubparser.add_argument("-t", "--tag", help="runs tests which match the spacified tag")
 runSubparser.add_argument("-d", "--device", help="cpu|gpu - run on a specified device")
 runSubparser.add_argument("-f", "--flavor", help="release|debug - run only a specified flavor")
+runSubparser.add_argument("-s", "--build-sku", default=defaultBuildSKU, help="cpuonly|default|1bitsgd - run tests only for a specified build SKU")
 tmpDir = os.getenv("TEMP") if windows else "/tmp"
 defaultRunDir=os.path.join(tmpDir, "cntk-test-{0}.{1}".format(time.strftime("%Y%m%d%H%M%S"), random.randint(0,1000000)))
 runSubparser.add_argument("-r", "--run-dir", default=defaultRunDir, help="directory where to store test output, default: a random dir within /tmp")
@@ -648,6 +652,7 @@ def runCommand(args):
 listSubparser.add_argument("-t", "--tag", help="limits a resulting list to tests matching the spacified tag")
 listSubparser.add_argument("-d", "--device", help="cpu|gpu - tests for a specified device")
 listSubparser.add_argument("-f", "--flavor", help="release|debug - tests for specified flavor")
+listSubparser.add_argument("-s", "--build-sku", default=defaultBuildSKU, help="cpuonly|default|1bitsgd - list tests only for a specified build SKU")
 listSubparser.add_argument("--os", help="windows|linux - tests for a specified operating system")
 
 listSubparser.set_defaults(func=listCommand)
@@ -675,6 +680,14 @@ def runCommand(args):
     sys.exit(1)
   args.flavors = [args.flavor]
 
+args.buildSKUs = ["cpuonly", "default", "1bitsgd"]
+if (args.build_sku):
+  args.build_sku = args.build_sku.lower()
+  if not args.build_sku in args.buildSKUs:
+    print >>sys.stderr, "--build-sku must be one of", args.buildSKUs
+    sys.exit(1)
+  args.buildSKUs = [args.build_sku]
+
 if args.func == listCommand:
   args.oses = ["windows", "linux"]
   if (args.os):