fix(videoout): leave a host thread that cannot unwind shutdown

core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp

@@ -39,5 +39,6 @@ Graphics::DccKeys DisplayBufferKeys(const DisplayBuffer& buffer);
 }
 
 extern "C" std::size_t AgcDriverDisplayBufferSize_nid_postfix(const AgcDriver::DisplayBuffer& buffer);
+extern "C" void AgcDriverDumpExitFrame_nid_postfix(const AgcDriver::DisplayBuffer& buffer);
 
 #endif

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp

@@ -54,6 +54,7 @@ struct QueueWorker {
     std::atomic<std::uint64_t> queued{0};
     std::unordered_map<std::uint64_t, std::uint32_t> unfinishedWrites;
     std::thread thread;
+    std::atomic<bool> exited{false};
 };
 
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp

@@ -16,7 +16,9 @@
 #include "ShaderDiskCache.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/GuestWriteWatch.hpp"
+#include "prx/libc/include/Shutdown.hpp"
 #include <atomic>
+#include <chrono>
 #include <cstdint>
 #include <cstdio>
 #include <cerrno>
@@ -143,12 +145,14 @@ Driver::~Driver() {
 
 void Driver::Shutdown() {
     stop();
+    if (LibcHostShutdownAbandoned_nid_no_patch()) LibcTerminate_nid_no_patch(LibcExitCode_nid_no_patch());
     std::lock_guard lock(mutex);
     rethrowFailure();
 }
 
 void Driver::stop() {
     require(!onWorkerThread(), "worker cannot stop itself");
+    if (LibcHostShutdownAbandoned_nid_no_patch()) LibcTerminate_nid_no_patch(LibcExitCode_nid_no_patch());
     std::lock_guard shutdownLock(shutdownMutex);
     if (stopped) return;
     LibcRequestShutdown_nid_postfix();
@@ -164,7 +168,15 @@ void Driver::stop() {
     }
     changed.notify_all();
     for (auto& [queue, worker] : workers) {
-        if (worker.thread.joinable()) worker.thread.join();
+        if (!worker.thread.joinable()) continue;
+        while (!worker.exited.load(std::memory_order_acquire)) {
+            if (LibcHostShutdownAbandoned_nid_no_patch()) {
+                worker.thread.detach();
+                LibcTerminate_nid_no_patch(LibcExitCode_nid_no_patch());
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(20));
+        }
+        worker.thread.join();
     }
     try {
         DrawFinisher::Get().Drain();
@@ -176,7 +188,11 @@ void Driver::stop() {
     StopWorkerSampler();
     CloseRegistrationPreparation();
     Graphics::StopPipelineLibraryWorker();
-    std::lock_guard gpuLock(GuestMemory::GpuMutex());
+    while (!GuestMemory::GpuMutex().try_lock()) {
+        if (LibcHostShutdownAbandoned_nid_no_patch()) LibcTerminate_nid_no_patch(LibcExitCode_nid_no_patch());
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    std::lock_guard gpuLock(GuestMemory::GpuMutex(), std::adopt_lock);
     Graphics::ReportHostImports();
     device.Reset();
     replacedDevices.clear();

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp

@@ -136,6 +136,15 @@ void Driver::reapCompletionLabels() {
 
 void Driver::run(std::uint32_t id) noexcept {
     onWorkerThread() = true;
+    std::atomic<bool>* exitedFlag = nullptr;
+    {
+        std::lock_guard lock(mutex);
+        exitedFlag = &workers.at(id).exited;
+    }
+    struct Exited {
+        std::atomic<bool>* flag;
+        ~Exited() { if (flag) flag->store(true, std::memory_order_release); }
+    } exited{exitedFlag};
     char role[32];
     std::snprintf(role, sizeof role, "queue worker 0x%x", id);
     PinWorkerThread(role);

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -53,6 +53,7 @@
 #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "prx/libc/include/General.hpp"
+#include "prx/libc/include/Shutdown.hpp"
 #ifdef _WIN32
 #include <windows.h>
 #endif
@@ -3435,6 +3436,31 @@ FrameDumps& Dumps() {
     return dumps;
 }
 
+void DumpExitFrame(const DisplayBuffer& buffer) {
+    auto& dumps = Dumps();
+    if (!frameDumpOutput.Enabled_nid_no_patch() || dumps.dumped >= dumps.limit) return;
+    const int index = dumps.dumped++;
+    try {
+        std::vector<std::byte> pixels;
+        if (LibcHostShutdownAbandoned_nid_no_patch()) {
+            const auto size = DisplayBufferSize(buffer);
+            pixels = DecodeDisplayBuffer(buffer, {reinterpret_cast<const std::byte*>(buffer.address), size});
+        } else {
+            pixels = ReadDisplayBuffer(buffer);
+        }
+        WriteFrameBmp(index, buffer.width, buffer.height, pixels);
+    } catch (const std::exception& error) {
+        if (dumps.dumped == index + 1) --dumps.dumped;
+        std::fprintf(stderr, "[gpu] exit frame dump failed: %s\n", error.what());
+    } catch (...) {
+        if (dumps.dumped == index + 1) --dumps.dumped;
+    }
+}
+
+}
+
+extern "C" void AgcDriverDumpExitFrame_nid_postfix(const AgcDriver::DisplayBuffer& buffer) {
+    DumpExitFrame(buffer);
 }
 
 bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer, const DisplayBuffer* plane) {

core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp

@@ -219,6 +219,7 @@ class VideoOutDriver {
     VideoOutDriver();
     ~VideoOutDriver();
     void Shutdown();
+    void RememberExitFrames(const VideoOutConfig& cfg);
 
     VideoOutDriver(const VideoOutDriver&) = delete;
     VideoOutDriver& operator=(const VideoOutDriver&) = delete;
@@ -246,8 +247,18 @@ class VideoOutDriver {
     std::optional<AgcDriver::DisplayBuffer> overlayPlane();
     void triggerEvents(VideoOutConfig& cfg, int eventKind, void* triggerData);
     void pollHostHdr();
+    void dumpExitFrames();
+
+    struct ExitFrameNote {
+        const VideoOutConfig* config = nullptr;
+        std::vector<AgcDriver::DisplayBuffer> buffers;
+    };
 
     std::mutex mutex;
+    std::mutex exitFrameMutex;
+    std::vector<ExitFrameNote> exitFrames;
+    std::atomic<bool> presentReturned{false};
+    std::atomic<bool> vblankReturned{false};
     std::mutex shutdownMutex;
     bool stopped = false;
     std::array<std::shared_ptr<VideoOutConfig>, VIDEO_OUT_NUM_MAX> contexts;

core/libs/prx/libSceVideoOut/src/Buffer.cpp

@@ -68,6 +68,7 @@ int CommitBufferGroup(const std::string& function, VideoOutConfig& cfg, int setI
         AgcDriverStoreGuestLabel_nid_postfix(&label, 0);
     }
     if (group.ps4) cfg.ps4 = true;
+    VideoOutDriver::Get().RememberExitFrames(cfg);
     return setIndex;
 }
 
@@ -237,6 +238,7 @@ int APS5_VABI sceVideoOutSubmitChangeBufferAttribute2(int handle, int setIndex,
         AgcDriverCheckGuestMemory_nid_postfix(reinterpret_cast<const void*>(display.address), AgcDriverDisplayBufferSize_nid_postfix(display), display.alignment);
     }
     cfg->groups[setIndex] = group;
+    VideoOutDriver::Get().RememberExitFrames(*cfg);
     return 0;
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();
@@ -267,6 +269,7 @@ int APS5_VABI sceVideoOutUnregisterBuffers(int handle, int setIndex) try {
             buf = VideoOutBuffer{};
         }
     }
+    VideoOutDriver::Get().RememberExitFrames(*cfg);
     return 0;
 } catch (const ProcessShutdown&) {
     LibcAwaitExit_nid_postfix();

core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp

@@ -348,29 +348,102 @@ VideoOutDriver::~VideoOutDriver() {
     if (!stopped) Shutdown();
 }
 
+namespace {
+
+[[noreturn]] void EndIfHostAbandoned() {
+    LibcTerminate_nid_no_patch(LibcExitCode_nid_no_patch());
+}
+
+template <typename Mutex>
+void LockOrEndIfAbandoned(Mutex& mutex) {
+    while (!mutex.try_lock()) {
+        if (LibcHostShutdownAbandoned_nid_no_patch()) EndIfHostAbandoned();
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+}
+
+template <typename Thread>
+void JoinUnlessAbandoned(Thread& thread, const std::atomic<bool>& returned) {
+    if (!thread.joinable()) return;
+    while (!returned.load(std::memory_order_acquire)) {
+        if (LibcHostShutdownAbandoned_nid_no_patch()) {
+            thread.detach();
+            EndIfHostAbandoned();
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(20));
+    }
+    thread.join();
+}
+
+}
+
+void VideoOutDriver::RememberExitFrames(const VideoOutConfig& cfg) {
+    ExitFrameNote noted;
+    noted.config = &cfg;
+    for (const auto& buffer : cfg.buffers) {
+        if (!buffer.Occupied()) continue;
+        const auto groupIndex = buffer.groupIndex;
+        if (groupIndex < 0 || groupIndex >= VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX || !cfg.groups[groupIndex].occupied) continue;
+        try {
+            noted.buffers.push_back(DescribeVideoOutBuffer(buffer, cfg.groups[groupIndex]));
+        } catch (const std::exception&) {
+        }
+    }
+    std::lock_guard lock(exitFrameMutex);
+    for (auto& existing : exitFrames) {
+        if (existing.config != &cfg) continue;
+        existing.buffers = std::move(noted.buffers);
+        return;
+    }
+    exitFrames.push_back(std::move(noted));
+}
+
+void VideoOutDriver::dumpExitFrames() {
+    if (!LibcExitRequested_nid_no_patch()) return;
+    std::vector<AgcDriver::DisplayBuffer> buffers;
+    {
+        std::lock_guard lock(exitFrameMutex);
+        for (const auto& note : exitFrames) buffers.insert(buffers.end(), note.buffers.begin(), note.buffers.end());
+    }
+    for (const auto& buffer : buffers) {
+        try {
+            AgcDriverDumpExitFrame_nid_postfix(buffer);
+        } catch (const std::exception&) {
+        } catch (...) {
+        }
+    }
+}
+
 void VideoOutDriver::Shutdown() {
+    dumpExitFrames();
+    if (LibcHostShutdownAbandoned_nid_no_patch()) EndIfHostAbandoned();
     require(std::this_thread::get_id() != presentThread.get_id(), "presentation thread cannot stop itself");
-    std::lock_guard shutdownLock(shutdownMutex);
+    LockOrEndIfAbandoned(shutdownMutex);
+    std::lock_guard shutdownLock(shutdownMutex, std::adopt_lock);
     if (stopped) return;
     LibcRequestShutdown_nid_postfix();
+    LockOrEndIfAbandoned(flipQueue->mutex);
     {
-        std::lock_guard queueLock(flipQueue->mutex);
+        std::lock_guard queueLock(flipQueue->mutex, std::adopt_lock);
         flipQueue->stopping = true;
     }
     presentThread.request_stop();
     vblankThread.request_stop();
     flipQueue->changed.NotifyAll();
-    presentThread.join();
-    vblankThread.join();
+    JoinUnlessAbandoned(presentThread, presentReturned);
+    JoinUnlessAbandoned(vblankThread, vblankReturned);
+    if (LibcHostShutdownAbandoned_nid_no_patch()) EndIfHostAbandoned();
     {
-        std::lock_guard lock(mutex);
+        LockOrEndIfAbandoned(mutex);
+        std::lock_guard lock(mutex, std::adopt_lock);
         for (int handle = 1; handle < VIDEO_OUT_NUM_MAX; ++handle) {
             if (outputs[handle]) close(handle);
         }
     }
     OnMainThread([] { SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER); });
     stopped = true;
-    std::lock_guard lock(flipQueue->mutex);
+    LockOrEndIfAbandoned(flipQueue->mutex);
+    std::lock_guard lock(flipQueue->mutex, std::adopt_lock);
     if (flipQueue->failure) std::rethrow_exception(flipQueue->failure);
 }
 
@@ -682,6 +755,10 @@ std::optional<AgcDriver::DisplayBuffer> VideoOutDriver::overlayPlane() {
 }
 
 void VideoOutDriver::presentLoop(std::stop_token token, std::promise<void>& started) {
+    struct Returned {
+        std::atomic<bool>& flag;
+        ~Returned() { flag.store(true, std::memory_order_release); }
+    } returned{presentReturned};
     try {
         OnMainThread([&] {
             window.Ensure(VIDEO_OUT_DEFAULT_WIDTH, VIDEO_OUT_DEFAULT_HEIGHT);
@@ -825,6 +902,10 @@ void VideoOutDriver::presentLoop(std::stop_token token, std::promise<void>& star
 }
 
 void VideoOutDriver::vblankLoop(std::stop_token token) {
+    struct Returned {
+        std::atomic<bool>& flag;
+        ~Returned() { flag.store(true, std::memory_order_release); }
+    } returned{vblankReturned};
     using Frame = std::chrono::duration<int64_t, std::ratio<1001, 120000>>;
     const auto start = std::chrono::steady_clock::now();
     try {

core/libs/prx/libc/include/Shutdown.hpp

@@ -15,6 +15,11 @@ extern "C" void LibcStartExit_nid_postfix();
 extern "C" [[noreturn]] void LibcExit_nid_no_patch(int code);
 extern "C" [[noreturn]] void LibcExitWithoutHandlers_nid_no_patch(int code);
 extern "C" [[noreturn]] void LibcTerminate_nid_no_patch(int code);
+extern "C" int LibcExitCode_nid_no_patch();
+extern "C" bool LibcExitRequested_nid_no_patch();
+extern "C" void LibcNoteHostShutdownAbandoned_nid_no_patch();
+extern "C" bool LibcHostShutdownAbandoned_nid_no_patch();
+extern "C" [[noreturn]] void LibcFinishProcessExit_nid_no_patch(int code);
 extern "C" void LibcSetEndFlush_nid_no_patch(void (*flush)());
 extern "C" void LibcRunEndFlush_nid_no_patch();
 extern "C" void LibcSetExitDrain_nid_no_patch(void (*drain)());