ENH: Add script to create new tests

This script takes a configuration file, a test name (must be unique) and the test case to use. It then generates the output and stores this in the baseline files.

Update the test_features script to allow for tests that do not test all available features. (testing still fails if there are feature classes that are not tested).
When a feature class is tested, all features must be enabled.

tests/addTest.py

@@ -0,0 +1,74 @@
+import argparse
+import logging
+
+import six
+
+import radiomics
+from radiomics import featureextractor
+from testUtils import RadiomicsTestUtils
+
+
+def main(argv=None):
+  logger = logging.getLogger('radiomics.addTest')
+
+  logger.setLevel(logging.INFO)
+  radiomics.setVerbosity(logging.INFO)
+
+  handler = logging.FileHandler(filename='testLog.txt', mode='w')
+  formatter = logging.Formatter("%(levelname)s:%(name)s: %(message)s")
+  handler.setFormatter(formatter)
+
+  radiomics.logger.addHandler(handler)
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('TestName', type=str, help='Name for the new test, must not be already present in the baseline')
+  parser.add_argument('TestCase', type=str, choices=list(radiomics.testCases), help='Test image and segmentation to '
+                                                                                    'use in the new test')
+  parser.add_argument('Configuration', metavar='FILE', default=None,
+                      help='Parameter file containing the settings to be used in extraction')
+
+  args = parser.parse_args(argv)
+
+  logger.info('Input accepted, starting addTest...')
+
+  try:
+
+    testutils = RadiomicsTestUtils()
+
+    try:
+      assert args.TestName not in testutils.getTests()
+      assert args.TestCase in radiomics.testCases
+    except AssertionError as e:
+      logger.error('Input not valid, cancelling addTest! (%s)', e.message)
+      exit(1)
+
+    logger.debug('Initializing extractor')
+    extractor = featureextractor.RadiomicsFeaturesExtractor(args.Configuration)
+
+    logger.debug('Starting extraction')
+    featurevector = extractor.execute(*radiomics.getTestCase(args.TestCase))
+
+    configuration = {}
+    baselines = {}
+
+    for k, v in six.iteritems(featurevector):
+      if 'general_info' in k:
+        configuration[k] = v
+      else:
+        image_filter, feature_class, feature_name = k.split('_')
+        if feature_class not in baselines:
+          baselines[feature_class] = {}
+        baselines[feature_class][k] = v
+
+    configuration['general_info_TestCase'] = args.TestCase
+
+    testutils.addTest(args.TestName, configuration, baselines)
+
+    logger.info('addTest Done')
+
+  except Exception:
+    logger.error('Error running addTest!', exc_info=True)
+
+
+if __name__ == '__main__':
+  main()

tests/add_baseline.py

@@ -24,10 +24,10 @@ def __init__(self):
     self.baselineDir = os.path.join(dataDir, "baseline")
 
   def generate_scenarios(self):
-    for test in self.testCases:
-      for className, featureClass in six.iteritems(self.featureClasses):
-        if not os.path.exists(os.path.join(self.baselineDir, 'baseline_%s.csv' % (className))):
-          self.logger.debug('generate_scenarios: featureClass = %s', className)
+    for className, featureClass in six.iteritems(self.featureClasses):
+      if not os.path.exists(os.path.join(self.baselineDir, 'baseline_%s.csv' % className)):
+        self.logger.debug('generate_scenarios: featureClass = %s', className)
+        for test in self.testCases:
           yield test, className
 
   def process_testcase(self, test, featureClassName):

tests/testUtils.py

@@ -10,15 +10,12 @@
 import SimpleITK as sitk
 import six
 
-from radiomics import getTestCase, imageoperations
+from radiomics import getTestCase, imageoperations, testCases
 
 # Get the logger. This is done outside the class, as it is needed by both the class and the custom_name_func
 logger = logging.getLogger('radiomics.testing')
 
 
-TEST_CASES = ('brain1', 'brain2', 'breast1', 'lung1', 'lung2')
-
-
 def custom_name_func(testcase_func, param_num, param):
   """
   A custom test name function that will ensure that the tests are run such that they're batched with all tests for a
@@ -53,9 +50,9 @@ def custom_name_func(testcase_func, param_num, param):
 
 class RadiomicsTestUtils:
   """
-  This utility class reads in and stores the baseline files stored in 'data\baseline' (one per feature class)
-  It provides utility methods to get the baseline feature value for a feature class and compare it to the result generated
-  by the test.
+  This utility class reads in and stores the baseline files stored in 'data/baseline' (one per feature class)
+  It provides utility methods to get the baseline feature value for a feature class and compare it to the result
+  generated by the test.
   """
   def __init__(self):
     self._logger = logging.getLogger('radiomics.testing.utils')
@@ -120,7 +117,7 @@ def getFeatureNames(self, className, test):
     containing the feature names (without image type and feature class specifiers, i.e. just the feature name).
     """
     if className not in self._baseline:
-      return None  # No baseline available for specified class
+      raise AssertionError('No baseline available for class %s.' % className)
     return self._baseline[className].getTestFeatures(test)
 
   def setFeatureClassAndTestCase(self, className, test):
@@ -135,7 +132,6 @@ def setFeatureClassAndTestCase(self, className, test):
     If feature class and test case are unchanged, nothing is reloaded and function returns False. If either feature
     class or test case is changed, function returns True.
     """
-    global TEST_CASES
     if self._featureClassName == className and self._test == test:
       return False
 
@@ -158,7 +154,7 @@ class or test case is changed, function returns True.
     if self._testCase != self._current_config['TestCase']:
       self._testCase = self._current_config['TestCase']
       self._logger.info("Reading the image and mask for test case %s", self._testCase)
-      assert self._current_config['TestCase'] in TEST_CASES
+      assert self._current_config['TestCase'] in testCases
 
       imageName, maskName = getTestCase(self._testCase)
 
@@ -248,7 +244,7 @@ def checkResult(self, featureName, value):
     self._diffs[self._test][longName] = percentDiff
 
     # check for a less than three percent difference
-    if (percentDiff >= 0.03):
+    if percentDiff >= 0.03:
       self._logger.error('checkResult %s, baseline value = %f, calculated = %f, diff = %f%%', featureName,
                          float(baselineValue), value, percentDiff * 100)
     assert (percentDiff < 0.03)
@@ -289,6 +285,23 @@ def writeCSV(self, data, fileName):
     else:
       self._logger.info('No test cases run, aborting file write to %s', fileName)
 
+  def addTest(self, case, configuration, baselines):
+    if case in self._tests:
+      self._logger.warning('Test %s already present in the baseline, skipping addTest', case)
+      return
+
+    self._tests.add(case)
+    self._results[case] = {}
+    self._diffs[case] = {}
+
+    for featureClass in baselines:
+      if featureClass not in self._baseline:
+        self._logger.warning('Feature class %s does not yet have a baseline, creating a new one', featureClass)
+        self._baseline[featureClass] = PyRadiomicsBaseline(featureClass)
+
+      self._baseline[featureClass].addTest(case, configuration, baselines[featureClass])
+      self._baseline[featureClass].writeBaselineFile(self._baselineDir)
+
 
 class PyRadiomicsBaseline:
 
@@ -326,6 +339,15 @@ def readBaselineFile(cls, baselineFile):
       new_baseline.tests = set(tests)
     return new_baseline
 
+  def addTest(self, case, configuration, baseline):
+    if case in self.tests:
+      self.logger.warning('Test %s already present in the baseline, skipping addTest', case)
+      return
+
+    self.tests.add(case)
+    self.configuration[case] = configuration
+    self.baseline[case] = baseline
+
   def getTestConfig(self, test):
     if test not in self.configuration:
       return {}  # This test is not present in the baseline for this class
@@ -362,22 +384,22 @@ def getBaselineValue(self, test, featureName):
 
   def writeBaselineFile(self, baselineDir):
     baselineFile = os.path.join(baselineDir, 'baseline_%s.csv' % self.cls)
-    testCases = sorted(self.baseline.keys())
+    cases = sorted(list(self.baseline.keys()))
     with open(baselineFile, 'wb') as baseline:
       csvWriter = csv.writer(baseline)
-      header = ['featureName'] + testCases
+      header = ['featureName'] + cases
       csvWriter.writerow(header)
 
-      config = self.configuration[testCases[0]].keys()
+      config = self.configuration[cases[0]].keys()
       for c in config:
         row = [c]
-        for testCase in testCases:
+        for testCase in cases:
           row.append(str(self.configuration[testCase].get(c, '')))
         csvWriter.writerow(row)
 
-      features = self.baseline[testCases[0]].keys()
+      features = self.baseline[cases[0]].keys()
       for f in features:
         row = [f]
-        for testCase in testCases:
+        for testCase in cases:
           row.append(str(self.baseline[testCase].get(f, '')))
         csvWriter.writerow(row)

tests/test_features.py

@@ -28,9 +28,13 @@ def generate_scenarios():
     for test in tests:
       for featureClassName in extractor.getFeatureClassNames():
         # Get all feature names for which there is a baseline with current test case
+        # Raises an assertion error when the class is not yet present in the baseline
+        # Returns None if no baseline is present for this specific test case
+        # Returns a list of feature names for which baseline values are present for this test
         baselineFeatureNames = testUtils.getFeatureNames(featureClassName, test)
 
-        assert (baselineFeatureNames is not None)
+        if baselineFeatureNames is None:
+          continue
         assert (len(baselineFeatureNames) > 0)
 
         uniqueFeatures = set([f.split('_')[-1] for f in baselineFeatureNames])

tests/test_matrices.py

@@ -9,23 +9,23 @@
 import numpy
 import six
 
-from radiomics import getFeatureClasses
+from radiomics import getFeatureClasses, testCases
 from testUtils import custom_name_func, RadiomicsTestUtils
 
 
 testUtils = RadiomicsTestUtils()
 
-tests = sorted(testUtils.getTests())
-
 featureClasses = getFeatureClasses()
 
 
 class TestMatrices:
 
   def generate_scenarios():
-    global tests, featureClasses
+    global featureClasses
 
-    for testCase in tests:
+    for testCase in testCases:
+      if testCase.startswith('test'):
+        continue
       for className, featureClass in six.iteritems(featureClasses):
         assert(featureClass is not None)
         if "_calculateMatrix" in dir(featureClass):