Run integration tests in CI

.gitlab-ci.yml

@@ -5,6 +5,7 @@ default:
 # See https://docs.gitlab.com/ee/ci/yaml/#needs
 stages:
   - build
+  - test
 
 # https://blog.nimbleways.com/let-s-make-faster-gitlab-ci-cd-pipelines/
 variables:
@@ -21,7 +22,6 @@ variables:
   image: ubuntu:focal
   rules:
     - if: $CI_PIPELINE_SOURCE == "push"
-
 
 .Ubuntu:
   extends: .Ubuntu_Image
@@ -300,6 +300,24 @@ Ubuntu_Clang_tests_Debug:
     reports:
       junit: build/tests.xml
 
+Ubuntu_Clang_integration_tests:
+  extends: .Ubuntu_Image
+  stage: test
+  needs:
+    - Ubuntu_Clang
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+  cache:
+    key: Ubuntu_Clang_integration_tests.v1
+    paths:
+      - .cache/pip
+      - apt-cache/
+  before_script:
+    - CI/install_debian_deps.sh openmw-integration-tests
+    - pip3 install --user numpy matplotlib termtables click
+  script:
+    - CI/run_integration_tests.sh
+
 .MacOS:
   image: macos-11-xcode-12
   tags:

CI/before_script.linux.sh

@@ -28,7 +28,6 @@ declare -a CMAKE_CONF_OPTS=(
     -DBUILD_SHARED_LIBS=OFF
     -DUSE_SYSTEM_TINYXML=ON
     -DOPENMW_USE_SYSTEM_RECASTNAVIGATION=ON
-    -DCMAKE_INSTALL_PREFIX=install
     -DOPENMW_CXX_FLAGS="-Werror -Werror=implicit-fallthrough"  # flags specific to OpenMW project
 )
 

CI/install_debian_deps.sh

@@ -42,6 +42,39 @@ declare -rA GROUPED_DEPS=(
   "
 
   [openmw-coverage]="gcovr"
+
+  [openmw-integration-tests]="
+    ca-certificates
+    git
+    git-lfs
+    libavcodec58
+    libavformat58
+    libavutil56
+    libboost-filesystem1.71.0
+    libboost-iostreams1.71.0
+    libboost-program-options1.71.0
+    libboost-system1.71.0
+    libbullet2.88
+    libcollada-dom2.4-dp0
+    libicu66
+    libjpeg8
+    libluajit-5.1-2
+    liblz4-1
+    libmyguiengine3debian1v5
+    libopenal1
+    libopenscenegraph161
+    libpng16-16
+    libqt5opengl5
+    librecast1
+    libsdl2-2.0-0
+    libsqlite3-0
+    libswresample3
+    libswscale5
+    libtinyxml2.6.2v5
+    libyaml-cpp0.6
+    python3-pip
+    xvfb
+  "
 )
 
 if [[ $# -eq 0 ]]; then

CI/run_integration_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+
+git clone --depth=1 https://gitlab.com/OpenMW/example-suite.git
+
+xvfb-run --auto-servernum --server-args='-screen 0 640x480x24x60' \
+    scripts/integration_tests.py --omw build/install/bin/openmw --workdir integration_tests_output example-suite/
+
+ls integration_tests_output/*.osg_stats.log | while read v; do
+    scripts/osg_stats.py --stats '.*' --regexp_match < "${v}"
+done

scripts/data/integration_tests/test_lua_api/player.lua

@@ -12,46 +12,56 @@ input.setControlSwitch(input.CONTROL_SWITCH.Magic, false)
 input.setControlSwitch(input.CONTROL_SWITCH.VanityMode, false)
 input.setControlSwitch(input.CONTROL_SWITCH.ViewMode, false)
 
-testing.registerLocalTest('playerMovement',
+testing.registerLocalTest('playerRotation',
     function()
-        local startTime = core.getSimulationTime()
-        local pos = self.position
-
-        while core.getSimulationTime() < startTime + 0.5 do
+        local endTime = core.getSimulationTime() + 1
+        while core.getSimulationTime() < endTime do
             self.controls.jump = false
             self.controls.run = true
             self.controls.movement = 0
             self.controls.sideMovement = 0
-            local progress = (core.getSimulationTime() - startTime) / 0.5
-            self.controls.yawChange = util.normalizeAngle(math.rad(90) * progress - self.rotation.z)
+            self.controls.yawChange = util.normalizeAngle(math.rad(90) - self.rotation.z) * 0.5
             coroutine.yield()
         end
         testing.expectEqualWithDelta(self.rotation.z, math.rad(90), 0.05, 'Incorrect rotation')
+    end)
 
-        while core.getSimulationTime() < startTime + 1.5 do
+testing.registerLocalTest('playerForwardRunning',
+    function()
+        local startPos = self.position
+        local endTime = core.getSimulationTime() + 1
+        while core.getSimulationTime() < endTime do
             self.controls.jump = false
             self.controls.run = true
             self.controls.movement = 1
             self.controls.sideMovement = 0
             self.controls.yawChange = 0
             coroutine.yield()
         end
-        direction = (self.position - pos) / types.Actor.runSpeed(self)
-        testing.expectEqualWithDelta(direction.x, 1, 0.1, 'Run forward, X coord')
-        testing.expectEqualWithDelta(direction.y, 0, 0.1, 'Run forward, Y coord')
+        local direction, distance = (self.position - startPos):normalize()
+        local normalizedDistance = distance / types.Actor.runSpeed(self)
+        testing.expectEqualWithDelta(normalizedDistance, 1, 0.2, 'Normalized forward runned distance')
+        testing.expectEqualWithDelta(direction.x, 0, 0.1, 'Run forward, X coord')
+        testing.expectEqualWithDelta(direction.y, 1, 0.1, 'Run forward, Y coord')
+    end)
 
-        pos = self.position
-        while core.getSimulationTime() < startTime + 2.5 do
+testing.registerLocalTest('playerDiagonalWalking',
+    function()
+        local startPos = self.position
+        local endTime = core.getSimulationTime() + 1
+        while core.getSimulationTime() < endTime do
             self.controls.jump = false
             self.controls.run = false
             self.controls.movement = -1
             self.controls.sideMovement = -1
             self.controls.yawChange = 0
             coroutine.yield()
         end
-        direction = (self.position - pos) / types.Actor.walkSpeed(self)
+        local direction, distance = (self.position - startPos):normalize()
+        local normalizedDistance = distance / types.Actor.walkSpeed(self)
+        testing.expectEqualWithDelta(normalizedDistance, 1, 0.2, 'Normalized diagonally walked distance')
         testing.expectEqualWithDelta(direction.x, -0.707, 0.1, 'Walk diagonally, X coord')
-        testing.expectEqualWithDelta(direction.y, 0.707, 0.1, 'Walk diagonally, Y coord')
+        testing.expectEqualWithDelta(direction.y, -0.707, 0.1, 'Walk diagonally, Y coord')
     end)
 
 return {
@@ -60,4 +70,3 @@ return {
     },
     eventHandlers = testing.eventHandlers
 }
-

scripts/data/integration_tests/test_lua_api/test.lua

@@ -29,10 +29,10 @@ local function testTimers()
 
     while not (ts1 and ts2 and th1 and th2) do coroutine.yield() end
 
-    testing.expectAlmostEqual(th1, 36, 'async:newGameTimer failed')
-    testing.expectAlmostEqual(ts1, 0.5, 'async:newSimulationTimer failed')
-    testing.expectAlmostEqual(th2, 72, 'async:newUnsavableGameTimer failed')
-    testing.expectAlmostEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed')
+    testing.expectGreaterOrEqual(th1, 36, 'async:newGameTimer failed')
+    testing.expectGreaterOrEqual(ts1, 0.5, 'async:newSimulationTimer failed')
+    testing.expectGreaterOrEqual(th2, 72, 'async:newUnsavableGameTimer failed')
+    testing.expectGreaterOrEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed')
 end
 
 local function testTeleport()
@@ -51,9 +51,25 @@ local function testTeleport()
     testing.expectEqualWithDelta(player.rotation.z, math.rad(-90), 0.05, 'teleporting changes rotation')
 end
 
+local function initPlayer()
+    player:teleport('', util.vector3(4096, 4096, 867.237), util.vector3(0, 0, 0))
+    coroutine.yield()
+end
+
 tests = {
     {'timers', testTimers},
-    {'playerMovement', function() testing.runLocalTest(player, 'playerMovement') end},
+    {'playerRotation', function()
+        initPlayer()
+        testing.runLocalTest(player, 'playerRotation')
+    end},
+    {'playerForwardRunning', function()
+        initPlayer()
+        testing.runLocalTest(player, 'playerForwardRunning')
+    end},
+    {'playerDiagonalWalking', function()
+        initPlayer()
+        testing.runLocalTest(player, 'playerDiagonalWalking')
+    end},
     {'teleport', testTeleport},
 }
 

scripts/data/integration_tests/test_lua_api/testing_util.lua

@@ -52,6 +52,12 @@ function M.expectAlmostEqual(v1, v2, msg)
     end
 end
 
+function M.expectGreaterOrEqual(v1, v2, msg)
+    if not (v1 >= v2) then
+        error(string.format('%s: %f >= %f', msg or '', v1, v2), 2)
+    end
+end
+
 local localTests = {}
 local localTestRunner = nil
 

scripts/integration_tests.py

@@ -51,14 +51,30 @@ def runTest(name):
         )
         if (test_dir / "test.omwscripts").exists():
             omw_cfg.write("content=test.omwscripts\n")
+    with open(config_dir / "settings.cfg", "a", encoding="utf-8") as settings_cfg:
+        settings_cfg.write(
+            "[Video]\n"
+            "resolution x = 640\n"
+            "resolution y = 480\n"
+            "framerate limit = 60\n"
+        )
+    stdout_lines = list()
+    exit_ok = True
+    test_success = True
     with subprocess.Popen(
-        [f"{openmw_binary}", "--replace=config", f"--config={config_dir}", "--skip-menu", "--no-grab"],
+        [openmw_binary, "--replace=config", "--config", config_dir, "--skip-menu", "--no-grab", "--no-sound"],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         encoding="utf-8",
+        env={
+            "OPENMW_OSG_STATS_FILE": work_dir / f"{name}.{time_str}.osg_stats.log",
+            "OPENMW_OSG_STATS_LIST": "times",
+            **os.environ,
+        },
     ) as process:
         quit_requested = False
         for line in process.stdout:
+            stdout_lines.append(line)
             words = line.split(" ")
             if len(words) > 1 and words[1] == "E]":
                 print(line, end="")
@@ -72,16 +88,30 @@ def runTest(name):
             elif "TEST_FAILED" in line:
                 w = line.split("TEST_FAILED")[1].split("\t")
                 print(f"FAILED {w[3]}\t\t")
+                test_success = False
         process.wait(5)
         if not quit_requested:
             print("ERROR: Unexpected termination")
-    shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log")
-    print(f"{name} finished")
+            exit_ok = False
+        if process.returncode != 0:
+            print(f"ERROR: openmw exited with code {process.returncode}")
+            exit_ok = False
+    if os.path.exists(config_dir / "openmw.log"):
+        shutil.copyfile(config_dir / "openmw.log", work_dir / f"{name}.{time_str}.log")
+    if not exit_ok:
+        sys.stdout.writelines(stdout_lines)
+    if test_success and exit_ok:
+        print(f"{name} succeeded")
+    else:
+        print(f"{name} failed")
+    return test_success and exit_ok
 
 
+status = 0
 for entry in tests_dir.glob("test_*"):
     if entry.is_dir():
-        runTest(entry.name)
+        if not runTest(entry.name):
+            status = -1
 shutil.rmtree(config_dir, ignore_errors=True)
 shutil.rmtree(userdata_dir, ignore_errors=True)
-
+exit(status)

scripts/osg_stats.py

@@ -76,7 +76,7 @@ def main(print_keys, regexp_match, timeseries, hist, hist_ratio, stdev_hist, plo
     def matching_keys(patterns):
         if regexp_match:
             return [key for pattern in patterns for key in keys if re.search(pattern, key)]
-        return keys
+        return patterns
     if timeseries:
         draw_timeseries(sources=frames, keys=matching_keys(timeseries), add_sum=timeseries_sum,
                         begin_frame=begin_frame, end_frame=end_frame)