From a421c2458d3b877e4d62e0c3f5910171c3225117 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 19:21:29 -0400
Subject: [PATCH 01/29] ci: add metadrive simulation workflow

---
 .github/workflows/metadrive-simulation.yml | 192 +++++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 .github/workflows/metadrive-simulation.yml

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
new file mode 100644
index 00000000000000..3d03b59aa2ca4e
--- /dev/null
+++ b/.github/workflows/metadrive-simulation.yml
@@ -0,0 +1,192 @@
+name: MetaDrive Simulation Test
+
+on:
+  pull_request:
+    branches: [master]
+    paths:
+      - ".github/workflows/metadrive-simulation.yml"
+      - "tools/sim/**"
+      - "selfdrive/**"
+      - "system/**"
+      - "cereal/**"
+      - "openpilot/**"
+      - "pyproject.toml"
+      - "uv.lock"
+  workflow_dispatch:
+
+concurrency:
+  group: metadrive-simulation-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  metadrive-simulation:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+
+    env:
+      DISPLAY: ":99"
+      LIBGL_ALWAYS_SOFTWARE: "1"
+      GALLIUM_DRIVER: "llvmpipe"
+      LP_NUM_THREADS: "4"
+      MESA_GL_VERSION_OVERRIDE: "3.3"
+      MESA_GLSL_VERSION_OVERRIDE: "330"
+      SIMULATION: "1"
+      PASSIVE: "0"
+      SKIP_FW_QUERY: "1"
+      CI: "true"
+      PYTHONUNBUFFERED: "1"
+
+    steps:
+      - name: Checkout openpilot
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: false
+
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet /opt/ghc "$AGENT_TOOLSDIRECTORY" || true
+          df -h
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            git \
+            curl \
+            ca-certificates \
+            capnproto \
+            libcapnp-dev \
+            ffmpeg \
+            libavcodec-dev \
+            libavformat-dev \
+            libavutil-dev \
+            libswscale-dev \
+            libegl1 \
+            libegl1-mesa-dev \
+            libgles2-mesa-dev \
+            libgl1-mesa-dev \
+            libgl1-mesa-dri \
+            libglx-mesa0 \
+            libglu1-mesa-dev \
+            mesa-utils \
+            mesa-common-dev \
+            xvfb \
+            x11-utils \
+            pulseaudio \
+            portaudio19-dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Start Xvfb
+        run: |
+          Xvfb :99 -screen 0 1920x1080x24 -ac +extension GLX +render -noreset > /tmp/xvfb.log 2>&1 &
+          echo "XVFB_PID=$!" >> "$GITHUB_ENV"
+          sleep 5
+
+          if ! xdpyinfo -display :99 >/dev/null 2>&1; then
+            echo "Xvfb failed to start"
+            cat /tmp/xvfb.log
+            exit 1
+          fi
+
+      - name: Verify software OpenGL
+        run: |
+          glxinfo | grep -E "OpenGL renderer|OpenGL version" || true
+
+      - name: Start virtual audio
+        run: |
+          pulseaudio --start --exit-idle-time=-1 || true
+          sleep 2
+          pactl load-module module-null-sink sink_name=virtual-speaker || true
+          pactl set-default-sink virtual-speaker || true
+          pactl list sinks short || true
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install -e .
+          python -m pip install metadrive-simulator pytest-timeout
+
+      - name: Pull MetaDrive assets
+        run: |
+          python -m metadrive.pull_asset
+
+      - name: Build openpilot
+        run: |
+          scons -u -j$(nproc)
+
+      - name: Run MetaDrive simulation test
+        id: metadrive_test
+        timeout-minutes: 15
+        run: |
+          python -m pytest tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving \
+            -v \
+            --tb=short \
+            --timeout=900 \
+            --log-cli-level=INFO
+
+      - name: Collect simulation artifacts
+        if: always()
+        run: |
+          mkdir -p simulation_artifacts
+
+          for location in /tmp /tmp/comma "$HOME/.comma" "$GITHUB_WORKSPACE"; do
+            if [ -d "$location" ]; then
+              find "$location" \( \
+                -name "*.qlog" -o \
+                -name "*.rlog" -o \
+                -name "*.hevc" -o \
+                -name "fcamera.hevc" -o \
+                -name "dcamera.hevc" -o \
+                -name "ecamera.hevc" -o \
+                -name "error.log" -o \
+                -name "crash.log" -o \
+                -name "*.log" \
+              \) -type f -exec cp --parents {} simulation_artifacts/ \; 2>/dev/null || true
+            fi
+          done
+
+          cat > simulation_artifacts/metadata.txt << EOF
+          MetaDrive Simulation Test
+          =========================
+          Run ID: ${{ github.run_id }}
+          Run attempt: ${{ github.run_attempt }}
+          Commit: ${{ github.sha }}
+          Test outcome: ${{ steps.metadrive_test.outcome }}
+          Date: $(date -Iseconds)
+
+          System
+          ======
+          OS: $(uname -a)
+          Python: $(python --version 2>&1)
+          OpenGL:
+          $(glxinfo | grep -E "OpenGL renderer|OpenGL version" || true)
+
+          Files
+          =====
+          EOF
+
+          find simulation_artifacts -type f -maxdepth 10 -print >> simulation_artifacts/metadata.txt
+          cat simulation_artifacts/metadata.txt
+
+      - name: Upload simulation artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metadrive-simulation-${{ github.run_id }}-${{ github.run_attempt }}
+          path: simulation_artifacts/
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Cleanup
+        if: always()
+        run: |
+          pulseaudio --kill || true
+          if [ -n "${XVFB_PID:-}" ]; then
+            kill "$XVFB_PID" || true
+          fi
\ No newline at end of file

From 93e9b64ddd06468bd40f709061765d92cd7e0fcd Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 19:31:33 -0400
Subject: [PATCH 02/29] ci: use python 3.12 for metadrive workflow

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 3d03b59aa2ca4e..6052dbe0114454 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -80,7 +80,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Start Xvfb
         run: |

From 66a4fc9a72db53b6713151b504dd2724243f0098 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 19:39:40 -0400
Subject: [PATCH 03/29] ci: install comma metadrive fork

---
 .github/workflows/metadrive-simulation.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 6052dbe0114454..8da29e21bcc477 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -110,11 +110,20 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install -e .
-          python -m pip install metadrive-simulator pytest-timeout
+          python -m pip install \
+            "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal" \
+            pytest-timeout
+
 
       - name: Pull MetaDrive assets
         run: |
-          python -m metadrive.pull_asset
+          python - <<'PY'
+          try:
+            from metadrive.pull_asset import pull_asset
+            pull_asset()
+          except ImportError:
+            import metadrive.pull_asset
+          PY
 
       - name: Build openpilot
         run: |

From 4d4d60959e3a392c95929a429f500b19aefa5281 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 19:45:16 -0400
Subject: [PATCH 04/29] ci: fix metadrive asset pull

---
 .github/workflows/metadrive-simulation.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 8da29e21bcc477..ec32b750054bde 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -118,11 +118,8 @@ jobs:
       - name: Pull MetaDrive assets
         run: |
           python - <<'PY'
-          try:
-            from metadrive.pull_asset import pull_asset
-            pull_asset()
-          except ImportError:
-            import metadrive.pull_asset
+          from metadrive.pull_asset import pull_asset
+          pull_asset(False)
           PY
 
       - name: Build openpilot

From cf7852d2971266981b1c8e0aab328742ac0cc339 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 19:50:09 -0400
Subject: [PATCH 05/29] ci: install imgui for metadrive build

---
 .github/workflows/metadrive-simulation.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index ec32b750054bde..24fc1d2d8291be 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -112,6 +112,7 @@ jobs:
           python -m pip install -e .
           python -m pip install \
             "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal" \
+            imgui \
             pytest-timeout
 
 

From 50a1bb4876e2b5c00650170e52a419713ae13b72 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:05:31 -0400
Subject: [PATCH 06/29] ci: use comma imgui dependency

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 24fc1d2d8291be..65603f748b55b1 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -112,7 +112,7 @@ jobs:
           python -m pip install -e .
           python -m pip install \
             "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal" \
-            imgui \
+            "imgui @ git+https://github.com/commaai/dependencies.git@release-imgui#subdirectory=imgui" \
             pytest-timeout
 
 

From f6b4cf1a4f72f71c6ecc141642efe8091347b1d8 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:13:01 -0400
Subject: [PATCH 07/29] ci: pull lfs assets for metadrive build

---
 .github/workflows/metadrive-simulation.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 65603f748b55b1..3a66917ba9f954 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -41,7 +41,12 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: recursive
-          lfs: false
+          lfs: true
+
+      - name: Pull Git LFS files
+        run: |
+          git lfs install
+          git lfs pull
 
       - name: Free disk space
         run: |

From 90a0b6e3e8ddb43f466a6d6290553349ae5a96ff Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:21:31 -0400
Subject: [PATCH 08/29] ci: pull only font lfs assets

---
 .github/workflows/metadrive-simulation.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 3a66917ba9f954..b7d8005193229c 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -41,12 +41,12 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: recursive
-          lfs: true
+          lfs: false
 
       - name: Pull Git LFS files
         run: |
           git lfs install
-          git lfs pull
+          git lfs pull --include="openpilot/selfdrive/assets/fonts/*.ttf" --exclude=""
 
       - name: Free disk space
         run: |

From 5696e7022c22f66c72ce8ac629fdbbdfa1a51b41 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:29:36 -0400
Subject: [PATCH 09/29] ci: pull all font lfs assets

---
 .github/workflows/metadrive-simulation.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index b7d8005193229c..19b70de58cc6b1 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -43,10 +43,10 @@ jobs:
           submodules: recursive
           lfs: false
 
-      - name: Pull Git LFS files
+      - name: Pull font LFS assets
         run: |
           git lfs install
-          git lfs pull --include="openpilot/selfdrive/assets/fonts/*.ttf" --exclude=""
+          git lfs pull --include="openpilot/selfdrive/assets/fonts/*" --exclude=""
 
       - name: Free disk space
         run: |

From 6c03b6c73979e1bebc2a31eb71e14116a7f8a04f Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:36:09 -0400
Subject: [PATCH 10/29] ci: pull model lfs assets for metadrive build

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 19b70de58cc6b1..b88b0e84f697c3 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Pull font LFS assets
         run: |
           git lfs install
-          git lfs pull --include="openpilot/selfdrive/assets/fonts/*" --exclude=""
+          git lfs pull --include="openpilot/selfdrive/assets/fonts/*,openpilot/selfdrive/modeld/models/*" --exclude=""
 
       - name: Free disk space
         run: |

From c4ac7cf0b1609e77efbddf05acfac4b4be6a6ea0 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:46:15 -0400
Subject: [PATCH 11/29] ci: install pytest-xdist for metadrive test

---
 .github/workflows/metadrive-simulation.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index b88b0e84f697c3..8ec0c5c953b97f 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -118,8 +118,8 @@ jobs:
           python -m pip install \
             "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal" \
             "imgui @ git+https://github.com/commaai/dependencies.git@release-imgui#subdirectory=imgui" \
-            pytest-timeout
-
+            pytest-timeout \
+            pytest-xdist
 
       - name: Pull MetaDrive assets
         run: |

From df6281a088f700ef7db0225003309aa60bce2815 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 20:57:05 -0400
Subject: [PATCH 12/29] ci: ignore repo pytest config for metadrive test

---
 .github/workflows/metadrive-simulation.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 8ec0c5c953b97f..abeb494a6a7e77 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -137,6 +137,7 @@ jobs:
         timeout-minutes: 15
         run: |
           python -m pytest tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving \
+            -c /dev/null \
             -v \
             --tb=short \
             --timeout=900 \

From 247f971c33137acbb9d27b40ca8e2b4650bea297 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 21:11:42 -0400
Subject: [PATCH 13/29] ci: set pytest rootdir for metadrive test

---
 .github/workflows/metadrive-simulation.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index abeb494a6a7e77..5b11d4eec7aad4 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -136,8 +136,10 @@ jobs:
         id: metadrive_test
         timeout-minutes: 15
         run: |
-          python -m pytest tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving \
+          python -m pytest \
+            --rootdir="$GITHUB_WORKSPACE" \
             -c /dev/null \
+            "$GITHUB_WORKSPACE/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \
             -v \
             --tb=short \
             --timeout=900 \

From 25da09fcdb30f8cde91c460d71e0501793fddd33 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 21:23:37 -0400
Subject: [PATCH 14/29] ci: fix metadrive test path

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 5b11d4eec7aad4..8dba983dc1fdfd 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -139,7 +139,7 @@ jobs:
           python -m pytest \
             --rootdir="$GITHUB_WORKSPACE" \
             -c /dev/null \
-            "$GITHUB_WORKSPACE/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \
+            "$GITHUB_WORKSPACE/openpilot/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \
             -v \
             --tb=short \
             --timeout=900 \

From 246a2b8f1e34ccb1f55bd5adae32f9c72548cfb1 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 21:39:55 -0400
Subject: [PATCH 15/29] ci: pull sound lfs assets for metadrive test

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 8dba983dc1fdfd..3a4723797c8ad1 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Pull font LFS assets
         run: |
           git lfs install
-          git lfs pull --include="openpilot/selfdrive/assets/fonts/*,openpilot/selfdrive/modeld/models/*" --exclude=""
+          git lfs pull --include="openpilot/selfdrive/assets/fonts/*,openpilot/selfdrive/assets/sounds/*,openpilot/selfdrive/modeld/models/*" --exclude=""
 
       - name: Free disk space
         run: |

From efb40d911e8ce89dbab14ca314295ebb6784bc1c Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 22:21:56 -0400
Subject: [PATCH 16/29] ci: print metadrive failure logs

---
 .github/workflows/metadrive-simulation.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 3a4723797c8ad1..c592fd4f7cf97d 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -145,6 +145,28 @@ jobs:
             --timeout=900 \
             --log-cli-level=INFO
 
+      - name: Print simulation logs on failure
+        if: failure()
+        run: |
+          echo "=== Searching for crash/error logs ==="
+          find /tmp "$HOME/.comma" "$GITHUB_WORKSPACE" -type f \( \
+            -name "*.log" -o \
+            -name "crash*" -o \
+            -name "error*" -o \
+            -name "*.txt" \
+          \) 2>/dev/null | sort | head -200
+
+          echo "=== Showing likely crash/error logs ==="
+          for f in $(find /tmp "$HOME/.comma" "$GITHUB_WORKSPACE" -type f \( \
+            -name "*.log" -o \
+            -name "crash*" -o \
+            -name "error*" \
+          \) 2>/dev/null | sort | head -50); do
+            echo ""
+            echo "----- $f -----"
+            tail -200 "$f" || true
+          done
+
       - name: Collect simulation artifacts
         if: always()
         run: |

From 2910793113c24c85a730feb468c39b20ea0e0e69 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 22:36:26 -0400
Subject: [PATCH 17/29] ci: print metadrive failure diagnostics

---
 .github/workflows/metadrive-simulation.yml | 33 ++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index c592fd4f7cf97d..a5713de0832ae9 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -144,7 +144,40 @@ jobs:
             --tb=short \
             --timeout=900 \
             --log-cli-level=INFO
+      - name: Print failure diagnostics
+        if: failure()
+        run: |
+          echo "=== Disk ==="
+          df -h || true
+
+          echo "=== Memory ==="
+          free -h || true
+
+          echo "=== OOM / kernel messages ==="
+          dmesg -T | grep -Ei "killed process|out of memory|oom|modeld|python" | tail -100 || true
+
+          echo "=== Running processes ==="
+          ps aux | sort -nrk 4 | head -40 || true
+
+          echo "=== Find logs ==="
+          find /tmp "$HOME/.comma"* "$GITHUB_WORKSPACE" -type f \( \
+            -name "*.log" -o \
+            -name "crash*" -o \
+            -name "error*" -o \
+            -name "*.txt" \
+          \) 2>/dev/null | sort | head -200
 
+          echo "=== Tail logs ==="
+          for f in $(find /tmp "$HOME/.comma"* "$GITHUB_WORKSPACE" -type f \( \
+            -name "*.log" -o \
+            -name "crash*" -o \
+            -name "error*" \
+          \) 2>/dev/null | sort | head -80); do
+            echo ""
+            echo "----- $f -----"
+            tail -200 "$f" || true
+          done
+          
       - name: Print simulation logs on failure
         if: failure()
         run: |

From ca8fe20b00a3e66b32e780d9acd67f53ea78fb4f Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 22:55:20 -0400
Subject: [PATCH 18/29] ci: run metadrive test with ONNX CPU

---
 .github/workflows/metadrive-simulation.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index a5713de0832ae9..c84f56847138fb 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -33,6 +33,7 @@ jobs:
       SIMULATION: "1"
       PASSIVE: "0"
       SKIP_FW_QUERY: "1"
+      ONNXCPU: 1
       CI: "true"
       PYTHONUNBUFFERED: "1"
 
@@ -177,7 +178,7 @@ jobs:
             echo "----- $f -----"
             tail -200 "$f" || true
           done
-          
+
       - name: Print simulation logs on failure
         if: failure()
         run: |

From 4a1fe2696e9fd683d216eb5c01011b67e5611875 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 23:19:19 -0400
Subject: [PATCH 19/29] ci: reduce cpu contention in metadrive test

---
 .github/workflows/metadrive-simulation.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index c84f56847138fb..be087da13628b2 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -27,13 +27,13 @@ jobs:
       DISPLAY: ":99"
       LIBGL_ALWAYS_SOFTWARE: "1"
       GALLIUM_DRIVER: "llvmpipe"
-      LP_NUM_THREADS: "4"
+      LP_NUM_THREADS: "2"
       MESA_GL_VERSION_OVERRIDE: "3.3"
       MESA_GLSL_VERSION_OVERRIDE: "330"
       SIMULATION: "1"
       PASSIVE: "0"
       SKIP_FW_QUERY: "1"
-      ONNXCPU: 1
+      ONNXCPU: "1"
       CI: "true"
       PYTHONUNBUFFERED: "1"
 

From e2fcc42a81e1d05fc8c9802e2210860ae199d9ed Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 23:34:16 -0400
Subject: [PATCH 20/29] ci: prioritize metadrive test process

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index be087da13628b2..803da80846e4bf 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -137,7 +137,7 @@ jobs:
         id: metadrive_test
         timeout-minutes: 15
         run: |
-          python -m pytest \
+          sudo -E nice -n -10 python -m pytest \
             --rootdir="$GITHUB_WORKSPACE" \
             -c /dev/null \
             "$GITHUB_WORKSPACE/openpilot/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \

From 46af3400ac4b7305a49abe24f663298c48035ced Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Wed, 24 Jun 2026 23:44:58 -0400
Subject: [PATCH 21/29] ci: use actions python for prioritized metadrive test

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 803da80846e4bf..ac5bbdc5384b7c 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -137,7 +137,7 @@ jobs:
         id: metadrive_test
         timeout-minutes: 15
         run: |
-          sudo -E nice -n -10 python -m pytest \
+          sudo -E nice -n -10 "$pythonLocation/bin/python" -m pytest \
             --rootdir="$GITHUB_WORKSPACE" \
             -c /dev/null \
             "$GITHUB_WORKSPACE/openpilot/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \

From 1b6888e21c9703d42ec834679f0199037579faed Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 00:01:21 -0400
Subject: [PATCH 22/29] ci: preserve python path for metadrive test

---
 .github/workflows/metadrive-simulation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index ac5bbdc5384b7c..b73aef9e0d76e3 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -137,7 +137,7 @@ jobs:
         id: metadrive_test
         timeout-minutes: 15
         run: |
-          sudo -E nice -n -10 "$pythonLocation/bin/python" -m pytest \
+          PYTHONPATH="$GITHUB_WORKSPACE:$GITHUB_WORKSPACE/openpilot:${PYTHONPATH:-}" "$pythonLocation/bin/python" -m pytest \
             --rootdir="$GITHUB_WORKSPACE" \
             -c /dev/null \
             "$GITHUB_WORKSPACE/openpilot/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \

From 523251ad9b28e19f9b88c3b85624864f9b726e28 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 00:12:55 -0400
Subject: [PATCH 23/29] ci: show metadrive runtime output

---
 .github/workflows/metadrive-simulation.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index b73aef9e0d76e3..222a34b93d7086 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -144,7 +144,8 @@ jobs:
             -v \
             --tb=short \
             --timeout=900 \
-            --log-cli-level=INFO
+            --log-cli-level=INFO \
+            -s
       - name: Print failure diagnostics
         if: failure()
         run: |

From ec60d28e21836fef22c4c7388c838e047ceb7b57 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 00:27:03 -0400
Subject: [PATCH 24/29] ci: use openpilot testing dependencies for metadrive

---
 .github/workflows/metadrive-simulation.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index 222a34b93d7086..bf73d7b28399e5 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -115,12 +115,11 @@ jobs:
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
-          python -m pip install -e .
+          python -m pip install -e ".[testing]"
           python -m pip install \
             "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal" \
             "imgui @ git+https://github.com/commaai/dependencies.git@release-imgui#subdirectory=imgui" \
-            pytest-timeout \
-            pytest-xdist
+            pytest-timeout
 
       - name: Pull MetaDrive assets
         run: |
@@ -137,10 +136,9 @@ jobs:
         id: metadrive_test
         timeout-minutes: 15
         run: |
-          PYTHONPATH="$GITHUB_WORKSPACE:$GITHUB_WORKSPACE/openpilot:${PYTHONPATH:-}" "$pythonLocation/bin/python" -m pytest \
-            --rootdir="$GITHUB_WORKSPACE" \
-            -c /dev/null \
+          "$pythonLocation/bin/python" -m pytest \
             "$GITHUB_WORKSPACE/openpilot/tools/sim/tests/test_metadrive_bridge.py::TestMetaDriveBridge::test_driving" \
+            -o addopts="" \
             -v \
             --tb=short \
             --timeout=900 \

From dd70cbc60c0478e8ed13cbdb69787bbabf741561 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 01:17:08 -0400
Subject: [PATCH 25/29] ci: extend metadrive test duration on actions

---
 openpilot/tools/sim/tests/test_metadrive_bridge.py | 3 ++-
 openpilot/tools/sim/tests/test_sim_bridge.py       | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/openpilot/tools/sim/tests/test_metadrive_bridge.py b/openpilot/tools/sim/tests/test_metadrive_bridge.py
index 9be640d736e159..c11bcb55e6d081 100644
--- a/openpilot/tools/sim/tests/test_metadrive_bridge.py
+++ b/openpilot/tools/sim/tests/test_metadrive_bridge.py
@@ -1,3 +1,4 @@
+import os
 import pytest
 import warnings
 
@@ -11,7 +12,7 @@
 class TestMetaDriveBridge(TestSimBridgeBase):
   @pytest.fixture(autouse=True)
   def setup_create_bridge(self, test_duration):
-    self.test_duration = 30
+    self.test_duration = 120 if os.getenv("GITHUB_ACTIONS") == "true" else 30
 
   def create_bridge(self):
     return MetaDriveBridge(False, False, self.test_duration, True)
diff --git a/openpilot/tools/sim/tests/test_sim_bridge.py b/openpilot/tools/sim/tests/test_sim_bridge.py
index f93cc2ef5059f1..6763fa8df19464 100644
--- a/openpilot/tools/sim/tests/test_sim_bridge.py
+++ b/openpilot/tools/sim/tests/test_sim_bridge.py
@@ -31,7 +31,7 @@ def test_driving(self):
     p_bridge = bridge.run(q, retries=10)
     self.processes.append(p_bridge)
 
-    max_time_per_step = 60
+    max_time_per_step = 120 if os.getenv("GITHUB_ACTIONS") == "true" else 60
 
     # Wait for bridge to startup
     start_waiting = time.monotonic()

From b07a53d34fd59b4223abcc1b894c4f267129623a Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 01:28:58 -0400
Subject: [PATCH 26/29] ci: extend metadrive bridge duration on actions

---
 openpilot/tools/sim/tests/test_metadrive_bridge.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openpilot/tools/sim/tests/test_metadrive_bridge.py b/openpilot/tools/sim/tests/test_metadrive_bridge.py
index c11bcb55e6d081..de6735e99c3ecd 100644
--- a/openpilot/tools/sim/tests/test_metadrive_bridge.py
+++ b/openpilot/tools/sim/tests/test_metadrive_bridge.py
@@ -12,7 +12,7 @@
 class TestMetaDriveBridge(TestSimBridgeBase):
   @pytest.fixture(autouse=True)
   def setup_create_bridge(self, test_duration):
-    self.test_duration = 120 if os.getenv("GITHUB_ACTIONS") == "true" else 30
+    self.test_duration = 300 if os.getenv("GITHUB_ACTIONS") == "true" else 30
 
   def create_bridge(self):
     return MetaDriveBridge(False, False, self.test_duration, True)

From f3939e455272e11e4c0498452b91cdb736431786 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 14:44:03 -0400
Subject: [PATCH 27/29] ci: revert metadrive test timeout changes

---
 openpilot/tools/sim/tests/test_metadrive_bridge.py | 3 +--
 openpilot/tools/sim/tests/test_sim_bridge.py       | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/openpilot/tools/sim/tests/test_metadrive_bridge.py b/openpilot/tools/sim/tests/test_metadrive_bridge.py
index de6735e99c3ecd..9be640d736e159 100644
--- a/openpilot/tools/sim/tests/test_metadrive_bridge.py
+++ b/openpilot/tools/sim/tests/test_metadrive_bridge.py
@@ -1,4 +1,3 @@
-import os
 import pytest
 import warnings
 
@@ -12,7 +11,7 @@
 class TestMetaDriveBridge(TestSimBridgeBase):
   @pytest.fixture(autouse=True)
   def setup_create_bridge(self, test_duration):
-    self.test_duration = 300 if os.getenv("GITHUB_ACTIONS") == "true" else 30
+    self.test_duration = 30
 
   def create_bridge(self):
     return MetaDriveBridge(False, False, self.test_duration, True)
diff --git a/openpilot/tools/sim/tests/test_sim_bridge.py b/openpilot/tools/sim/tests/test_sim_bridge.py
index 6763fa8df19464..f93cc2ef5059f1 100644
--- a/openpilot/tools/sim/tests/test_sim_bridge.py
+++ b/openpilot/tools/sim/tests/test_sim_bridge.py
@@ -31,7 +31,7 @@ def test_driving(self):
     p_bridge = bridge.run(q, retries=10)
     self.processes.append(p_bridge)
 
-    max_time_per_step = 120 if os.getenv("GITHUB_ACTIONS") == "true" else 60
+    max_time_per_step = 60
 
     # Wait for bridge to startup
     start_waiting = time.monotonic()

From d3f71502e2ce206cb3ee23af898a86c0ea8b840c Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 14:49:57 -0400
Subject: [PATCH 28/29] ci: configure virtual audio for metadrive

---
 .github/workflows/metadrive-simulation.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index bf73d7b28399e5..b1415c4e93c92e 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -35,6 +35,7 @@ jobs:
       SKIP_FW_QUERY: "1"
       ONNXCPU: "1"
       CI: "true"
+      CI_DISABLE_SOUNDD: "1"
       PYTHONUNBUFFERED: "1"
 
     steps:
@@ -110,6 +111,7 @@ jobs:
           sleep 2
           pactl load-module module-null-sink sink_name=virtual-speaker || true
           pactl set-default-sink virtual-speaker || true
+          pactl info || true
           pactl list sinks short || true
 
       - name: Install Python dependencies

From 5af455e41e68224c2c7bbc5d4e1661f392a2c961 Mon Sep 17 00:00:00 2001
From: dhruv pandoh <dhruvpandoh29@gmail.com>
Date: Thu, 25 Jun 2026 15:16:55 -0400
Subject: [PATCH 29/29] ci: remove extra cpu limits for metadrive

---
 .github/workflows/metadrive-simulation.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/metadrive-simulation.yml b/.github/workflows/metadrive-simulation.yml
index b1415c4e93c92e..316bc20b41011c 100644
--- a/.github/workflows/metadrive-simulation.yml
+++ b/.github/workflows/metadrive-simulation.yml
@@ -27,7 +27,7 @@ jobs:
       DISPLAY: ":99"
       LIBGL_ALWAYS_SOFTWARE: "1"
       GALLIUM_DRIVER: "llvmpipe"
-      LP_NUM_THREADS: "2"
+      LP_NUM_THREADS: "4"
       MESA_GL_VERSION_OVERRIDE: "3.3"
       MESA_GLSL_VERSION_OVERRIDE: "330"
       SIMULATION: "1"
@@ -35,7 +35,6 @@ jobs:
       SKIP_FW_QUERY: "1"
       ONNXCPU: "1"
       CI: "true"
-      CI_DISABLE_SOUNDD: "1"
       PYTHONUNBUFFERED: "1"
 
     steps: