Tests: switch to custom python runner from bats (simulationcraft#5382)

Out current tests are a bit arcane to setup and run because they use the
bats testing framework. While it's good and tested we use it only as
basically a glorified for loop, even calling to python for talent string
generation.

As we already use python for some parts of testing we may as well fully
switch to it. While it's true that it's another piece of code we have to
maintain, it's actually less code than we had with bats and has a bunch
of benefits:
* an actual language for setting up & running tests
* one less obscure dependency
* possibility to easily run tests locally

The "framework" is pretty limited & simple for now but seems to fully
replace what we have before. The main idea is that you simply add tests
into a list (with a possiblity to group them) and then simply run them.

.github/workflows/main.yml

@@ -153,17 +153,28 @@ jobs:
         run: ${{ runner.workspace }}/b/ninja/simc output=/dev/null html=/dev/null json2=/dev/null json3=/dev/null ${{ matrix.simc_flags }}
 
   simc-tests:
-    name: ${{ matrix.tier }}-${{ matrix.class }}-${{ matrix.fight_style }}
+    name: test-${{ matrix.tier }}-${{ matrix.spec }}
     runs-on: ubuntu-20.04
     needs: [ ubuntu-clang-10-build ]
 
     strategy:
       fail-fast: false
       matrix:
         tier: [ Tier25 ]
-        class: [ Death_Knight, Demon_Hunter, Druid, Hunter, Mage, Monk, Paladin, Priest, Rogue, Shaman, Warlock, Warrior ]
-        fight_style: [ Patchwerk, DungeonSlice, HeavyMovement ]
-
+        spec: [
+            Death_Knight_Blood, Death_Knight_Frost, Death_Knight_Unholy,
+            Demon_Hunter_Havoc, Demon_Hunter_Vengeance,
+            Druid_Balance, Druid_Feral, Druid_Guardian,
+            Hunter_Beast_Mastery, Hunter_Marksmanship, Hunter_Survival,
+            Mage_Arcane, Mage_Fire, Mage_Frost,
+            Monk_Brewmaster, Monk_Windwalker,
+            Paladin_Retribution,
+            Priest_Discipline, Priest_Shadow,
+            Rogue_Assassination, Rogue_Outlaw, Rogue_Subtlety,
+            Shaman_Elemental, Shaman_Enhancement, Shaman_Restoration,
+            Warlock_Affliction, Warlock_Demonology, Warlock_Destruction,
+            Warrior_Arms, Warrior_Fury,
+          ]
 
     steps:
       - uses: actions/cache@v2
@@ -174,21 +185,14 @@ jobs:
             tests
           key: ubuntu-clang-10-for_run-${{ github.sha }}
 
-      - name: Setup BATS
-        uses: mig4/[email protected]
-
       - name: Run
         env:
           UBSAN_OPTIONS: print_stacktrace=1
           SIMC_CLI_PATH: ${{ runner.workspace }}/b/ninja/simc
           SIMC_PROFILE_DIR: ${{ github.workspace }}/profiles/${{ matrix.tier }}
-          SIMC_FIGHT_STYLE: ${{ matrix.fight_style }}
-          SIMC_CLASS: ${{ matrix.class }}
           SIMC_THREADS: 2
           SIMC_ITERATIONS: 2
-        run: |
-          cd tests
-          bats --tap talents.bats covenants.bats
+        run: tests/run.py ${{ matrix.spec }}
 
   build-docker:
     name: docker

tests/helper.py

@@ -0,0 +1,118 @@
+import sys, os, shutil, subprocess, re, signal
+from pathlib import Path
+
+def __error_status(code):
+    if code and code < 0:
+        try:
+            return 'Process died with %r' % signal.Signals(-code)
+        except ValueError:
+            return 'Process died with unknown signal {}'.format(-code)
+    else:
+        return 'Process returned non-zero exit status {}'.format(code)
+
+def __simc_cli_path(var):
+    path = os.environ.get(var)
+    if path is None:
+        sys.exit(1)
+    path = shutil.which(path)
+    if path is None:
+        sys.exit(1)
+    return str(Path(path).resolve())
+
+IN_CI = 'CI' in os.environ
+
+SIMC_CLI_PATH = __simc_cli_path('SIMC_CLI_PATH')
+SIMC_ITERATIONS = int( os.environ.get('SIMC_ITERATIONS', '10') )
+SIMC_THREADS = int( os.environ.get('SIMC_THREADS', '2') )
+SIMC_FIGHT_STYLE = os.environ.get('SIMC_FIGHT_STYLE')
+SIMC_PROFILE_DIR = os.environ.get('SIMC_PROFILE_DIR', os.getcwd())
+
+def find_profiles(klass):
+    files = Path(SIMC_PROFILE_DIR).glob('*_{}*.simc'.format(klass))
+    return ( ( path.name, str(path.resolve()) ) for path in files )
+
+class TestGroup(object):
+    def __init__(self, name, **kwargs):
+        self.name = name
+        self.profile = kwargs.get('profile')
+        self.fight_style = kwargs.get('fight_style', SIMC_FIGHT_STYLE)
+        self.iterations = kwargs.get('iterations', SIMC_ITERATIONS)
+        self.threads = kwargs.get('threads', SIMC_THREADS)
+        self.tests = []
+
+class Test(object):
+    def __init__(self, name, **kwargs):
+        self.name = name
+
+        group = kwargs.get('group')
+        if group:
+            group.tests.append(self)
+
+        self._profile = kwargs.get('profile', group and group.profile)
+        self._fight_style = kwargs.get('fight_style', group and group.fight_style or SIMC_FIGHT_STYLE)
+        self._iterations = kwargs.get('iterations', group and group.iterations or SIMC_ITERATIONS)
+        self._threads = kwargs.get('threads', group and group.threads or SIMC_THREADS)
+        self._args = kwargs.get('args', [])
+
+    def args(self):
+        args = [
+            'iterations={}'.format(self._iterations),
+            'threads={}'.format(self._threads),
+            'cleanup_threads=1',
+            'default_actions=1',
+        ]
+        if self._fight_style:
+            args.append('fight_style={}'.format(self._fight_style))
+        args.append(self._profile)
+        for arg in self._args:
+            if isinstance(arg, tuple):
+                args.append('{}={}'.format(*arg))
+            else:
+                args.append(str(arg))
+        return args
+
+SIMC_WALL_SECONDS_RE = re.compile('WallSeconds\\s*=\\s*([0-9\\.]+)')
+def run_test(test):
+    args = [ SIMC_CLI_PATH ]
+    args.extend(test.args())
+
+    try:
+        res = subprocess.run(args, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='UTF-8', timeout=30)
+        wall_time = SIMC_WALL_SECONDS_RE.search(res.stdout)
+        return ( True, float(wall_time.group(1)), None )
+    except subprocess.CalledProcessError as err:
+        return ( False, 0, err )
+
+def run(tests):
+    success = 0
+    failure = 0
+    total = 0
+
+    def do_run(test):
+        nonlocal total, failure, success
+        total += 1
+        print('  {:<60}    '.format(subtest.name), end='', flush=True)
+        res, time, err = run_test(subtest)
+        if res:
+            print('[PASS] {:.5f}'.format(time))
+            success += 1
+        else:
+            print('[FAIL]')
+            print('-- {:<62} --------------'.format(__error_status(err.returncode)))
+            print(err.cmd)
+            if err.stderr:
+                print(err.stderr.rstrip('\r\n'))
+            print('-' * 80)
+            failure += 1
+
+    for test in tests:
+        if isinstance(test, TestGroup):
+            print(' {:<79}'.format(test.name))
+            for subtest in test.tests:
+                do_run(subtest)
+        else:
+            do_run(test)
+
+    print('Passed: {}/{}'.format(success, total))
+
+    return failure

tests/run.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import sys
+
+from helper import Test, TestGroup, run, find_profiles
+from talent_options import talent_combinations
+
+FIGHT_STYLES = ( 'Patchwerk', 'DungeonSlice', 'HeavyMovement' )
+COVENANTS = ( 'Kyrian', 'Venthyr', 'Night_Fae', 'Necrolord' )
+
+if len(sys.argv) < 2:
+    sys.exit(1)
+
+klass = sys.argv[1]
+
+print(' '.join(klass.split('_')))
+
+tests = []
+for profile, path in find_profiles(klass):
+    for fight_style in FIGHT_STYLES:
+        grp = TestGroup('{}/{}/talents'.format(profile, fight_style),
+                        fight_style=fight_style, profile=path)
+        tests.append(grp)
+        for talents in talent_combinations(klass):
+            Test(talents, group=grp, args=[ ('talents', talents) ])
+
+    for fight_style in FIGHT_STYLES:
+        grp = TestGroup('{}/{}/covenants'.format(profile, fight_style),
+                        fight_style=fight_style, profile=path)
+        tests.append(grp)
+        for covenant in COVENANTS:
+            Test(covenant, group=grp,
+                 args=[ ('covenant', covenant.lower()), ('level', 60) ])
+
+sys.exit( run(tests) )

tests/talent_options → tests/talent_options.py

@@ -1,7 +1,3 @@
-#!/usr/bin/env python3
-
-import sys
-
 __max_tiers = 7
 __talents_per_tier = 3
 __class_map = {
@@ -115,35 +111,28 @@
     ]
 }
 
-if len(sys.argv) < 2:
-    sys.exit(1)
-
-_class = None
-for class_name in __class_map.keys():
-    if class_name in sys.argv[1]:
-        _class = class_name
-        break
-
-if not _class:
-    sys.exit(1)
-
-# Check untalented
-combinations = [ '0' * __max_tiers ]
-
-for talents in __class_map[_class]:
-    talent_arr = [ '0' ] * __max_tiers
-    if talents[1] == -1:
-        for talent in range(0, __talents_per_tier):
-            talent_arr[talents[0]] = str(talent + 1)
-            talent_str = ''.join(talent_arr)
+def talent_combinations(klass):
+    if klass not in __class_map:
+        klass_split = klass.split('_')
+        for i in range(len(klass_split)):
+            klass = '_'.join(klass_split[0:i])
+            if klass in __class_map:
+                break
 
-            if talent_str not in combinations:
-                combinations.append(talent_str)
-    else:
-        talent_arr[talents[0]] = str(talents[1])
-        combinations.append(''.join(talent_arr))
+    # Check untalented
+    combinations = [ '0' * __max_tiers ]
 
-print(' '.join(combinations))
+    for talents in __class_map[klass]:
+        talent_arr = [ '0' ] * __max_tiers
+        if talents[1] == -1:
+            for talent in range(0, __talents_per_tier):
+                talent_arr[talents[0]] = str(talent + 1)
+                talent_str = ''.join(talent_arr)
 
-sys.exit(0)
+                if talent_str not in combinations:
+                    combinations.append(talent_str)
+        else:
+            talent_arr[talents[0]] = str(talents[1])
+            combinations.append(''.join(talent_arr))
 
+    return combinations