feat(libkernel): start the guest off the main thread on macOS

core/libs/prx/libSceVideoOut/include/MainThread.hpp

@@ -0,0 +1,38 @@
+#ifndef CORE_LIBS_PRX_LIBSCEVIDEOOUT_INCLUDE_MAINTHREAD_HPP
+#define CORE_LIBS_PRX_LIBSCEVIDEOOUT_INCLUDE_MAINTHREAD_HPP
+
+#include <exception>
+#include <optional>
+#include <type_traits>
+#include <utility>
+
+#ifdef __APPLE__
+extern "C" void Aps5RunOnMainThread_nid_no_patch(void (*function)(void*), void* context);
+#endif
+
+template <class Function>
+std::invoke_result_t<Function&> OnMainThread(Function&& function) {
+#ifdef __APPLE__
+    using Result = std::invoke_result_t<Function&>;
+    struct Call {
+        std::remove_reference_t<Function>* function;
+        std::exception_ptr error;
+        std::conditional_t<std::is_void_v<Result>, bool, std::optional<Result>> result {};
+    } call {&function, nullptr};
+    Aps5RunOnMainThread_nid_no_patch([](void* context) {
+        auto& call = *static_cast<Call*>(context);
+        try {
+            if constexpr (std::is_void_v<Result>) (*call.function)();
+            else call.result.emplace((*call.function)());
+        } catch (...) {
+            call.error = std::current_exception();
+        }
+    }, &call);
+    if (call.error) std::rethrow_exception(call.error);
+    if constexpr (!std::is_void_v<Result>) return std::move(*call.result);
+#else
+    return function();
+#endif
+}
+
+#endif

core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp

@@ -13,6 +13,7 @@
 
 #include "SDL.h"
 #include "SDL_vulkan.h"
+#include "prx/libSceVideoOut/include/MainThread.hpp"
 #include "prx/libSceVideoOut/include/PadInput.hpp"
 #include "prx/libSceVideoOut/include/MouseInput.hpp"
 #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp"
@@ -64,31 +65,37 @@ std::optional<std::uint32_t> configuredResolution() {
 }
 
 std::vector<const char*> windowExtensions(SDL_Window* window) {
-    unsigned extensionCount = 0;
-    if (!SDL_Vulkan_GetInstanceExtensions(window, &extensionCount, nullptr)) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError());
-    std::vector<const char*> extensions(extensionCount);
-    if (!SDL_Vulkan_GetInstanceExtensions(window, &extensionCount, extensions.data())) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError());
-    extensions.resize(extensionCount);
-    return extensions;
+    return OnMainThread([&] {
+        unsigned extensionCount = 0;
+        if (!SDL_Vulkan_GetInstanceExtensions(window, &extensionCount, nullptr)) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError());
+        std::vector<const char*> extensions(extensionCount);
+        if (!SDL_Vulkan_GetInstanceExtensions(window, &extensionCount, extensions.data())) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError());
+        extensions.resize(extensionCount);
+        return extensions;
+    });
 }
 
 VkSurfaceKHR createWindowSurface(void* context, VkInstance instance) {
-    VkSurfaceKHR surface = VK_NULL_HANDLE;
-    if (!SDL_Vulkan_CreateSurface(static_cast<SDL_Window*>(context), instance, &surface)) throw std::runtime_error(std::string("SDL_Vulkan_CreateSurface failed: ") + SDL_GetError());
-    return surface;
+    return OnMainThread([&] {
+        VkSurfaceKHR surface = VK_NULL_HANDLE;
+        if (!SDL_Vulkan_CreateSurface(static_cast<SDL_Window*>(context), instance, &surface)) throw std::runtime_error(std::string("SDL_Vulkan_CreateSurface failed: ") + SDL_GetError());
+        return surface;
+    });
 }
 
 void windowDrawableSize(void* context, std::uint32_t* width, std::uint32_t* height) {
-    if ((SDL_GetWindowFlags(static_cast<SDL_Window*>(context)) & SDL_WINDOW_MINIMIZED) != 0) {
-        *width = 0;
-        *height = 0;
-        return;
-    }
-    int drawableWidth = 0;
-    int drawableHeight = 0;
-    SDL_Vulkan_GetDrawableSize(static_cast<SDL_Window*>(context), &drawableWidth, &drawableHeight);
-    *width = drawableWidth > 0 ? static_cast<std::uint32_t>(drawableWidth) : 0;
-    *height = drawableHeight > 0 ? static_cast<std::uint32_t>(drawableHeight) : 0;
+    OnMainThread([&] {
+        if ((SDL_GetWindowFlags(static_cast<SDL_Window*>(context)) & SDL_WINDOW_MINIMIZED) != 0) {
+            *width = 0;
+            *height = 0;
+            return;
+        }
+        int drawableWidth = 0;
+        int drawableHeight = 0;
+        SDL_Vulkan_GetDrawableSize(static_cast<SDL_Window*>(context), &drawableWidth, &drawableHeight);
+        *width = drawableWidth > 0 ? static_cast<std::uint32_t>(drawableWidth) : 0;
+        *height = drawableHeight > 0 ? static_cast<std::uint32_t>(drawableHeight) : 0;
+    });
 }
 
 void checkConfig(const VideoOutConfig& cfg) {
@@ -291,17 +298,19 @@ VideoOutDriver& VideoOutDriver::Get() {
 }
 
 VideoOutDriver::VideoOutDriver() {
-    SDL_SetHint(SDL_HINT_WINDOWS_DPI_AWARENESS, "permonitorv2");
+    OnMainThread([] {
+        SDL_SetHint(SDL_HINT_WINDOWS_DPI_AWARENESS, "permonitorv2");
 #ifdef _WIN32
-    SDL_SetHintWithPriority(SDL_HINT_DIRECTINPUT_ENABLED, "0", SDL_HINT_DEFAULT);
-    if (std::getenv("APS5_NO_WGI") != nullptr) SDL_SetHintWithPriority(SDL_HINT_JOYSTICK_WGI, "0", SDL_HINT_DEFAULT);
+        SDL_SetHintWithPriority(SDL_HINT_DIRECTINPUT_ENABLED, "0", SDL_HINT_DEFAULT);
+        if (std::getenv("APS5_NO_WGI") != nullptr) SDL_SetHintWithPriority(SDL_HINT_JOYSTICK_WGI, "0", SDL_HINT_DEFAULT);
 #endif
 #ifdef __linux__
-    SDL_SetHint(SDL_HINT_VIDEODRIVER, "wayland,x11");
+        SDL_SetHint(SDL_HINT_VIDEODRIVER, "wayland,x11");
 #endif
-    if (SDL_InitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER) < 0) {
-        throw std::runtime_error(std::string("SDL_InitSubSystem(VIDEO | GAMECONTROLLER) failed: ") + SDL_GetError());
-    }
+        if (SDL_InitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER) < 0) {
+            throw std::runtime_error(std::string("SDL_InitSubSystem(VIDEO | GAMECONTROLLER) failed: ") + SDL_GetError());
+        }
+    });
     try {
         AgcDriverWaitIdle_nid_postfix();
         std::promise<void> started;
@@ -329,7 +338,7 @@ VideoOutDriver::VideoOutDriver() {
             flipQueue->changed.NotifyAll();
             presentThread.join();
         }
-        SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER);
+        OnMainThread([] { SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER); });
         throw;
     }
 }
@@ -358,7 +367,7 @@ void VideoOutDriver::Shutdown() {
             if (outputs[handle]) close(handle);
         }
     }
-    SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER);
+    OnMainThread([] { SDL_QuitSubSystem(SDL_INIT_VIDEO | SDL_INIT_GAMECONTROLLER); });
     stopped = true;
     std::lock_guard lock(flipQueue->mutex);
     if (flipQueue->failure) std::rethrow_exception(flipQueue->failure);
@@ -598,7 +607,7 @@ void VideoOutDriver::processFlip(FlipRequest& req) {
             req.gpuComplete = true;
         }
     } else {
-        window.Ensure(req.width, req.height);
+        OnMainThread([&] { window.Ensure(req.width, req.height); });
         const auto extensions = windowExtensions(window.Handle());
         const AgcDriver::PresentationWindow target{window.Handle(), extensions, &createWindowSurface, &windowDrawableSize, req.width, req.height, req.timing};
         timing.Mark("window_prepare");
@@ -618,7 +627,7 @@ void VideoOutDriver::processFlip(FlipRequest& req) {
             AgcDriverPresentClear_nid_postfix(target, req.unregistered || req.index == VIDEO_OUT_BUFFER_INDEX_BLACK, gpuReady, &req);
         }
         timing.Mark("present");
-        window.UpdateTitle();
+        OnMainThread([&] { window.UpdateTitle(); });
         timing.Mark("window_title");
     }
     std::lock_guard lock(req.cfg->mutex);
@@ -658,31 +667,39 @@ void VideoOutDriver::processFlip(FlipRequest& req) {
 
 void VideoOutDriver::presentLoop(std::stop_token token, std::promise<void>& started) {
     try {
-        window.Ensure(VIDEO_OUT_DEFAULT_WIDTH, VIDEO_OUT_DEFAULT_HEIGHT);
-        updateDisplayRefreshRate();
+        OnMainThread([&] {
+            window.Ensure(VIDEO_OUT_DEFAULT_WIDTH, VIDEO_OUT_DEFAULT_HEIGHT);
+            updateDisplayRefreshRate();
+        });
         hdrProbe.Poll(window.Handle());
         const auto extensions = windowExtensions(window.Handle());
         AgcDriverAttachWindow_nid_postfix({window.Handle(), extensions, &createWindowSurface, &windowDrawableSize, VIDEO_OUT_DEFAULT_WIDTH, VIDEO_OUT_DEFAULT_HEIGHT, nullptr});
         hdrGeneration = AgcDriver::CurrentHostHdr().generation;
     } catch (...) {
         hdrProbe.Release();
-        window.Destroy();
+        OnMainThread([&] { window.Destroy(); });
         started.set_exception(std::current_exception());
         return;
     }
     started.set_value();
     std::shared_ptr<FlipRequest> current;
+    std::optional<PadInput> padInput;
+    std::optional<MouseInput> mouseInput;
+    std::optional<KeyboardInput> keyboardInput;
+    std::optional<ImeTextInput> imeTextInput;
 #if APS5_ENABLE_TIMING_LOG
     std::unique_ptr<FrameTimingLog> timingLog;
 #endif
     try {
 #if APS5_ENABLE_TIMING_LOG
         timingLog = std::make_unique<FrameTimingLog>();
 #endif
-        PadInput padInput;
-        MouseInput mouseInput;
-        KeyboardInput keyboardInput;
-        ImeTextInput imeTextInput;
+        OnMainThread([&] {
+            padInput.emplace();
+            mouseInput.emplace();
+            keyboardInput.emplace();
+            imeTextInput.emplace();
+        });
         while (!token.stop_requested()) {
             {
                 std::unique_lock lock(flipQueue->mutex);
@@ -695,23 +712,27 @@ void VideoOutDriver::presentLoop(std::stop_token token, std::promise<void>& star
                 }
             }
 
-            SDL_Event event;
-            while (SDL_PollEvent(&event)) {
-                if (event.type == SDL_QUIT) {
-                    LibcRequestExit_nid_postfix(0);
-                    throw ProcessShutdown{};
-                }
-                padInput.HandleEvent(event, window);
-                if (window.Handle() != nullptr) {
-                    mouseInput.HandleEvent(event, SDL_GetWindowID(window.Handle()));
-                    keyboardInput.HandleEvent(event, SDL_GetWindowID(window.Handle()));
-                    imeTextInput.HandleEvent(event, SDL_GetWindowID(window.Handle()));
+            const bool quit = OnMainThread([&] {
+                SDL_Event event;
+                while (SDL_PollEvent(&event)) {
+                    if (event.type == SDL_QUIT) return true;
+                    padInput->HandleEvent(event, window);
+                    if (window.Handle() != nullptr) {
+                        mouseInput->HandleEvent(event, SDL_GetWindowID(window.Handle()));
+                        keyboardInput->HandleEvent(event, SDL_GetWindowID(window.Handle()));
+                        imeTextInput->HandleEvent(event, SDL_GetWindowID(window.Handle()));
+                    }
                 }
+                padInput->Update(window);
+                updateDisplayRefreshRate();
+                return false;
+            });
+            if (quit) {
+                LibcRequestExit_nid_postfix(0);
+                throw ProcessShutdown{};
             }
-            padInput.Update(window);
-            updateDisplayRefreshRate();
             pollHostHdr();
-            mouseInput.UpdateCapture(!padInput.MouseModeEnabled());
+            OnMainThread([&] { mouseInput->UpdateCapture(!padInput->MouseModeEnabled()); });
             if (current) {
                 require(current->timing != nullptr, "missing presentation timing");
                 const auto dequeued = AgcDriver::FrameTiming::Clock::now();
@@ -722,7 +743,7 @@ void VideoOutDriver::presentLoop(std::stop_token token, std::promise<void>& star
                 if (window.Handle() != nullptr) SDL_GetWindowSize(window.Handle(), &windowWidth, &windowHeight);
                 if (windowWidth > 0 && windowHeight > 0) {
                     const auto presented = AgcDriver::ComputeContainSize_nid_postfix(current->width, current->height, static_cast<std::uint32_t>(windowWidth), static_cast<std::uint32_t>(windowHeight), true);
-                    mouseInput.SetPresentation(current->width, current->height, presented.width, presented.height);
+                    mouseInput->SetPresentation(current->width, current->height, presented.width, presented.height);
                 }
                 const auto finished = AgcDriver::FrameTiming::Clock::now();
                 AgcDriver::FrameTiming::Clock::duration interval{};
@@ -775,7 +796,13 @@ void VideoOutDriver::presentLoop(std::stop_token token, std::promise<void>& star
         if (!flipQueue->failure) flipQueue->failure = std::current_exception();
     }
     hdrProbe.Release();
-    window.Destroy();
+    OnMainThread([&] {
+        padInput.reset();
+        mouseInput.reset();
+        keyboardInput.reset();
+        imeTextInput.reset();
+        window.Destroy();
+    });
 }
 
 void VideoOutDriver::vblankLoop(std::stop_token token) {

core/libs/prx/libSceVideoOut/tests/Flip.cpp

@@ -1095,7 +1095,11 @@ void testOutputResolutionOverride() {
 
 }
 
-int main(int argc, char** argv) {
+#ifdef __APPLE__
+extern "C" [[noreturn]] void Aps5StartGuest_nid_no_patch(void (*entry)(void*, void*), void* block);
+#endif
+
+int runTests(int argc, char** argv) {
     VideoOutTestEnvironment environment;
     try {
         if (argc == 2 && std::string(argv[1]) == "decode") testDecode();
@@ -1136,3 +1140,15 @@ int main(int argc, char** argv) {
         return 1;
     }
 }
+
+int main(int argc, char** argv) {
+#ifdef __APPLE__
+    static int arguments;
+    static char** values;
+    arguments = argc;
+    values = argv;
+    Aps5StartGuest_nid_no_patch([](void*, void*) { std::exit(runTests(arguments, values)); }, nullptr);
+#else
+    return runTests(argc, argv);
+#endif
+}

core/libs/prx/libkernel/CMakeLists.txt

@@ -35,6 +35,7 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL
         ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/src/Tsd.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/src/Rwlock.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/src/Sem.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/src/MainThreadMacOs.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/Posix/Atfork.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/Posix/Attr.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Pthread/Posix/Barrier.cpp
@@ -84,6 +85,9 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL
         ${CMAKE_CURRENT_SOURCE_DIR}/AppMetadata/src/ParamJsonParser.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/AppMetadata/src/ParamSfoParser.cpp
 )
+if(APPLE)
+    target_link_libraries(libkernel PRIVATE "-framework CoreFoundation")
+endif()
 
 target_include_directories(libkernel PRIVATE ${LIBS_INCLUDE_DIR})
 target_include_directories(libkernel PRIVATE ${LIBS_INCLUDE_DIR}/nid/include)

core/libs/prx/libkernel/Pthread/src/MainThreadMacOs.cpp

@@ -0,0 +1,62 @@
+#ifdef __APPLE__
+
+#include <CoreFoundation/CoreFoundation.h>
+#include <dispatch/dispatch.h>
+#include <pthread.h>
+#include <algorithm>
+#include <atomic>
+#include <cstdio>
+#include <cstdlib>
+
+namespace {
+
+struct GuestStart {
+    void (*entry)(void*, void*);
+    void* block;
+};
+
+std::atomic<bool> mainLoopRunning {false};
+
+void* RunGuest(void* argument) {
+    const auto start = *static_cast<const GuestStart*>(argument);
+    start.entry(start.block, nullptr);
+    std::fprintf(stderr, "guest entry returned\n");
+    std::abort();
+}
+
+void KeepAlive(CFRunLoopTimerRef, void*) {}
+
+}
+
+extern "C" {
+
+[[noreturn]] void Aps5StartGuest_nid_no_patch(void (*entry)(void*, void*), void* block) {
+    static GuestStart start;
+    start = {entry, block};
+    pthread_attr_t attributes;
+    if (pthread_attr_init(&attributes) != 0) std::abort();
+    const auto stackSize = std::max<std::size_t>(pthread_get_stacksize_np(pthread_self()), 8u << 20);
+    if (pthread_attr_setstacksize(&attributes, stackSize) != 0) std::abort();
+    mainLoopRunning.store(true, std::memory_order_release);
+    pthread_t thread;
+    if (pthread_create(&thread, &attributes, RunGuest, &start) != 0) {
+        std::fprintf(stderr, "cannot start the guest thread\n");
+        std::abort();
+    }
+    pthread_attr_destroy(&attributes);
+    CFRunLoopTimerRef timer = CFRunLoopTimerCreate(nullptr, 1.0e10, 1.0e10, 0, 0, KeepAlive, nullptr);
+    CFRunLoopAddTimer(CFRunLoopGetMain(), timer, kCFRunLoopCommonModes);
+    for (;;) CFRunLoopRun();
+}
+
+void Aps5RunOnMainThread_nid_no_patch(void (*function)(void*), void* context) {
+    if (pthread_main_np() != 0 || !mainLoopRunning.load(std::memory_order_acquire)) {
+        function(context);
+        return;
+    }
+    dispatch_sync_f(dispatch_get_main_queue(), context, function);
+}
+
+}
+
+#endif

docs/dev/debt/libkernel.md

@@ -182,6 +182,7 @@
 - [sceKernelRaiseException](../../../core/libs/prx/libkernel/System/src/Exception.cpp) builds the handler's `ucontext` with FreeBSD's amd64 layout (16-byte sigmask, `mcontext` at 0x40, `mc_rsp` at 0xf8); the console's layout is not verified beyond the fields PS5Util reads
 - [sceKernelRaiseException](../../../core/libs/prx/libkernel/System/src/Exception.cpp) does not consult the target thread's signal mask
 - [pthread_sigmask](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) shares the process-wide mask of `sigprocmask` and `_sigprocmask`; FreeBSD keeps a mask per thread
+- On macOS the guest runs on a second thread and the main thread serves the main dispatch queue from its run loop: AppKit (SDL's Cocoa windows, Metal surfaces, event pumping and game controllers) only works on the main thread, and a guest never hands its thread back. The macOS entry stub of the relinker starts the guest through `Aps5StartGuest` ([MainThreadMacOs.cpp](../../../core/libs/prx/libkernel/Pthread/src/MainThreadMacOs.cpp)), on a thread with the stack size of the main thread (at least 8 MiB), and the guest ends the process with `exit`; a guest entry that returns aborts. libSceVideoOut sends its SDL calls to the main thread with `Aps5RunOnMainThread` (`OnMainThread` in [MainThread.hpp](../../../core/libs/prx/libSceVideoOut/include/MainThread.hpp)), which waits for them and rethrows their exceptions; a host program that never called `Aps5StartGuest` runs them in place
 - [sceKernelRaiseException](../../../core/libs/prx/libkernel/System/src/Exception.cpp) reaches a thread that waits in an alertable `TimedWait` wait through an APC. A thread blocked anywhere else, including all waits under `APS5_COARSE_TIMED_WAITS` and host waits inside the emulator, is redirected and runs the handler only when that wait returns
 - [sceKernelRaiseException](../../../core/libs/prx/libkernel/System/src/Exception.cpp) on Windows runs the handler wherever the target thread was interrupted. It waits only while the thread is inside winpthreads or restoring a context through `NtContinue`, and throws if that lasts 1 s, so a handler that calls into other host code the thread was interrupted in, such as the C runtime's heap or a lock of the emulator, can deadlock on it or reenter it. A thread that holds the address-wait lock of `sceKernelSyncOnAddressWait`, `sceKernelSyncOnAddressWake` and `_umtx_op`, or the park queue lock of the guest mutexes, gets the raise as an APC instead, which runs once it has released the lock or while it waits; the other locks of the emulator are not deferred this way
 - [sigaltstack](../../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) records each thread's alternate signal stack and reports it back, following FreeBSD's `kern_sigaltstack` (`EINVAL` for flags other than `SS_DISABLE`, `ENOMEM` below `MINSIGSTKSZ` 2048), but handlers never run on it, so `SS_ONSTACK` and `EPERM` are never reported