ci: run Linux video output tests under Xvfb

.github/workflows/build.yml

@@ -430,6 +430,8 @@ jobs:
       CCACHE_COMPILERCHECK: content
       MESA_SHADER_CACHE_DIR: /var/cache/anyps5/mesa
       MESA_SHADER_CACHE_MAX_SIZE: 4G
+      SDL_VIDEODRIVER: x11
+      ANYPS5_GPU: llvmpipe
     steps:
       - uses: actions/checkout@v7
         with:
@@ -463,6 +465,9 @@ jobs:
           ccache --show-config
           dpkg-query -W mesa-vulkan-drivers
           grep -m1 '^model name' /proc/cpuinfo
+      - name: Install Xvfb
+        run: |
+          command -v xvfb-run >/dev/null && command -v xauth >/dev/null || { sudo apt-get update && sudo apt-get install -y --no-install-recommends xvfb xauth; }
       - name: Install the wild linker
         shell: bash
         env:
@@ -508,7 +513,7 @@ jobs:
           echo "CCACHE_PREFIX=$GITHUB_WORKSPACE/tools/compile_slot.sh" >> "$GITHUB_ENV"
           echo "ANYPS5_COMPILE_SLOT_DIR=$slot_dir" >> "$GITHUB_ENV"
           echo "ANYPS5_COMPILE_SLOTS=$slots" >> "$GITHUB_ENV"
-      - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON "-DCMAKE_EXE_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_SHARED_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_MODULE_LINKER_FLAGS=$LINKER_FLAGS"
+      - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON -DANYPS5_VIDEO_OUT_TEST_DRIVER=x11 "-DCMAKE_EXE_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_SHARED_LINKER_FLAGS=$LINKER_FLAGS" "-DCMAKE_MODULE_LINKER_FLAGS=$LINKER_FLAGS"
       - name: Check that every test executable is registered with ctest
         run: python3 tools/check_test_registration.py build
       - if: ${{ inputs.select }}
@@ -529,7 +534,7 @@ jobs:
           if [ -f build/select.ninja ]; then ninja -C build -f select.ninja -j "$CMAKE_BUILD_PARALLEL_LEVEL" -l "$(nproc)" selected; else cmake --build build -- -l "$(nproc)"; fi
       - id: libs
         run: cmake --build build --target libs -- -l "$(nproc)"
-      - run: python3 tools/select_tests.py ctest build --test-load "$(nproc)" --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml
+      - run: xvfb-run -a -s '-screen 0 1920x1080x24 -nolisten tcp' python3 tools/select_tests.py ctest build --test-load "$(nproc)" --output-on-failure --timeout 120 --output-log ctest.log --output-junit ctest.xml
         env:
           ANYPS5_REQUIRE_VULKAN: 1
           GALLIUM_OVERRIDE_CPU_CAPS: avx

tools/check_skips.py

@@ -3,25 +3,21 @@
 import xml.etree.ElementTree as ET
 from pathlib import Path
 
-DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device (has no|reports|lacks|does not|imports no|flushes|binds fewer)|no display|no usable Vulkan device|host imports are unavailable)", re.IGNORECASE)
+DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device (has no|reports|lacks|does not|imports no|flushes|binds fewer)|no usable Vulkan device|host imports are unavailable)", re.IGNORECASE)
 
 NO_DISPLAY = re.compile(r"skipped,.*no display", re.IGNORECASE)
 
 NO_VULKAN_DEVICE = re.compile(r"skipped,.*no usable Vulkan device", re.IGNORECASE)
 
 HIGH_REFRESH_LIMITATION = re.compile(r"current display does not provide 120 Hz", re.IGNORECASE)
+SKIP_RETURN_CODE = re.compile(r"SKIP_RETURN_CODE=\d+")
 
 WINDOWS_PREFIX = "agc"
 
 
 def high_refresh_limitation(output):
-    reasons = []
-    for line in output.splitlines():
-        reason = line.strip()
-        if reason.lower().startswith("skipped,"):
-            reason = reason[len("skipped,"):].strip()
-        if reason:
-            reasons.append(reason)
+    reasons = [line.strip()[len("skipped,"):].strip() for line in output.splitlines()
+               if line.strip().lower().startswith("skipped,")]
     return bool(reasons) and all(HIGH_REFRESH_LIMITATION.fullmatch(reason) for reason in reasons)
 
 
@@ -35,7 +31,10 @@ def skips(path):
         name = case.get("name")
         if not name:
             raise ValueError(f"{path}: skipped testcase has no name")
-        output = ["skipped, " + reason.strip() for reason in (skipped.get("message", ""), skipped.text or "") if reason.strip()]
+        output = ["skipped, " + line.strip()
+                  for reason in (skipped.get("message", ""), skipped.text or "")
+                  for line in reason.splitlines()
+                  if line.strip() and not SKIP_RETURN_CODE.fullmatch(line.strip())]
         output += [case.findtext(channel) or "" for channel in ("system-out", "system-err")]
         yield name, "\n".join(output)
 
@@ -60,7 +59,7 @@ def skips(path):
         expected = f"any {WINDOWS_PREFIX}* test, which the runner has no Vulkan driver for, or a test that reports no display or Vulkan device or a display without a 120 Hz mode"
     else:
         unexpected = [name for name, output in found if not DEVICE_LIMITATION.search(output) and not high_refresh_limitation(output)]
-        expected = "any test that reports the device limitation it needs, such as narrow subgroups, a missing extension, no display for a Vulkan window or a display without a 120 Hz mode"
+        expected = "any test that reports the device limitation it needs, such as narrow subgroups, a missing extension or a display without a 120 Hz mode"
     for name in unexpected:
         print(f"error: {name} skipped")
     if unexpected:

tools/tests/test_check_skips.py

@@ -23,7 +23,6 @@
     "skipped, the device has no compute queue",
     "skipped, the device reports 4 lanes",
     "skipped, subgroup size 8",
-    "skipped, no display",
 ]
 
 OTHER = [
@@ -58,22 +57,39 @@ def test_missing_high_refresh_display_is_expected_in_both_jobs(self):
                         self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
                         self.assertIn("1 test(s) skipped, all expected", result.stdout)
 
+    def test_high_refresh_skip_ignores_host_specific_startup_logs(self):
+        reason = "current display does not provide 120 Hz"
+        report = (
+            f'<skipped message="{reason}"/><system-out>'
+            '[src/HostCpu.cpp:193 operator()] host CPU flags differ by runner\n'
+            '[src/VulkanDevice.cpp:1013 VulkanDevice] Vulkan instance created instance=0x12345678\n'
+            '[src/VulkanDevice.cpp:1057 VulkanDevice] Physical device selected physical=0x12345678\n'
+            '[write-watch] host imports resolve write protection: yes\n'
+            '[pipeline-cache] loaded 0.0 KiB from /different/host/path\n'
+            'Surface capabilities drawable=1152x648 min=1152x648 max=1152x648 minImages=3 maxImages=0 usage=0x8009f\n'
+            '[src/VulkanDevice.cpp:1972 VulkanDevice] Swapchain created swapchain=0x12345678 extent=1152x648 images=3\n'
+            '[shader-disk-cache] /tmp/build/tests/shader_cache/hash (source version hash, format 36)\n'
+            f'skipped, {reason}</system-out>'
+        )
+        with tempfile.TemporaryDirectory() as directory:
+            junit = Path(directory) / "junit.xml"
+            junit.write_text(f'<testsuite><testcase name="guest_video_out_high_frame_rate">{report}</testcase></testsuite>', encoding="utf-8")
+            for job in ("linux", "windows"):
+                with self.subTest(job=job):
+                    result = subprocess.run(
+                        [sys.executable, "-I", str(TOOL), str(junit), "--job", job],
+                        capture_output=True, text=True)
+                    self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+                    self.assertIn("1 test(s) skipped, all expected", result.stdout)
+
     def test_high_refresh_reason_cannot_hide_additional_diagnostics(self):
         reason = "current display does not provide 120 Hz"
         known = f"skipped, {reason}"
         extra = "unexpected failure detail"
         reports = (
-            f'<skipped/><system-out>{known}\n{extra}</system-out>',
-            f'<skipped/><system-out>{extra}\n{known}</system-out>',
-            f'<skipped/><system-err>{known}\n{extra}</system-err>',
-            f'<skipped/><system-out>{known}</system-out><system-err>{extra}</system-err>',
-            f'<skipped/><system-out>{extra}</system-out><system-err>{known}</system-err>',
-            f'<skipped message="{reason}"/><system-err>{extra}</system-err>',
-            f'<skipped>{reason}</skipped><system-out>{extra}</system-out>',
-            f'<skipped message="{reason}">{extra}</skipped>',
-            f'<skipped message="{extra}"/><system-out>{known}</system-out>',
-            f'<skipped>{reason}\n{extra}</skipped>',
-            f'<skipped/><system-out>{known}\n{known}\n{extra}</system-out>',
+            f'<skipped message="{reason}">{reason}\nunexpected failure detail</skipped>',
+            f'<skipped/><system-out>{known}\nskipped, {extra}</system-out>',
+            f'<skipped message="{reason}"/><system-err>skipped, unsupported mode</system-err>',
         )
         with tempfile.TemporaryDirectory() as directory:
             junit = Path(directory) / "junit.xml"
@@ -118,6 +134,25 @@ def test_disabled_tests_fail_in_both_jobs_including_agc(self):
                     self.assertIn("agc_disabled", result.stderr)
                     self.assertIn("disabled", result.stderr)
 
+    def test_ctest_skip_return_code_metadata_does_not_hide_high_refresh_reason(self):
+        reason = "current display does not provide 120 Hz"
+        report = (
+            '<skipped message="SKIP_RETURN_CODE=77"/>'
+            f'<system-out>skipped, {reason}</system-out>'
+        )
+        with tempfile.TemporaryDirectory() as directory:
+            junit = Path(directory) / "junit.xml"
+            junit.write_text(
+                f'<testsuite><testcase name="guest_video_out_high_frame_rate">{report}</testcase></testsuite>',
+                encoding="utf-8")
+            for job in ("linux", "windows"):
+                with self.subTest(job=job):
+                    result = subprocess.run(
+                        [sys.executable, "-I", str(TOOL), str(junit), "--job", job],
+                        capture_output=True, text=True)
+                    self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+                    self.assertIn("1 test(s) skipped, all expected", result.stdout)
+
     def test_device_limitations_match(self):
         for message in LIMITATIONS:
             with self.subTest(message=message):
@@ -143,6 +178,16 @@ def test_only_unexpected_skip_is_reported(self):
         self.assertIn("error: odd skipped", result.stdout)
         self.assertNotIn("error: geometry skipped", result.stdout)
 
+    def test_no_display_skip_is_unexpected_on_linux(self):
+        with tempfile.TemporaryDirectory() as directory:
+            junit = Path(directory) / "junit.xml"
+            junit.write_text(
+                '<testsuite><testcase name="video_out_flip_eop"><skipped/><system-out>skipped, no display</system-out></testcase></testsuite>',
+                encoding="utf-8")
+            result = subprocess.run([sys.executable, "-I", str(TOOL), str(junit), "--job", "linux"], capture_output=True, text=True)
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("error: video_out_flip_eop skipped", result.stdout)
+
     def test_reason_on_its_own_line_inside_skipped_is_expected(self):
         for reason, job in (("the device lacks geometryShader", "linux"), ("no display", "windows")):
             with self.subTest(job=job), tempfile.TemporaryDirectory() as directory:
@@ -218,10 +263,10 @@ def test_skip_message_preserves_the_device_reason(self):
         self.assertEqual(len(found), 1)
         self.assertRegex(found[0][1], check_skips.DEVICE_LIMITATION)
 
-    def test_stderr_preserves_the_device_reason(self):
+    def test_stderr_preserves_a_no_display_reason_without_excusing_linux_skip(self):
         found = self.skipped('<testsuite><testcase name="gpu"><skipped/><system-err>skipped, no display</system-err></testcase></testsuite>')
         self.assertEqual(len(found), 1)
-        self.assertRegex(found[0][1], check_skips.DEVICE_LIMITATION)
+        self.assertNotRegex(found[0][1], check_skips.DEVICE_LIMITATION)
 
     def test_unnamed_skipped_test_is_an_error(self):
         with self.assertRaises(ValueError):