fix(libkernel): start thread stacks on a 16 KiB page boundary on Linux

core/libs/prx/libkernel/CMakeLists.txt

@@ -965,6 +965,14 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_stack PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        add_test_executable(guest_thread_stack_guard_tests ${LIBS_INCLUDE_DIR}/tests/GuestThreadStackGuard.cpp)
+        target_include_directories(guest_thread_stack_guard_tests PRIVATE ${LIBS_INCLUDE_DIR})
+        target_link_libraries(guest_thread_stack_guard_tests PRIVATE libkernel libc)
+        add_test(NAME guest_thread_stack_guard COMMAND guest_thread_stack_guard_tests)
+        set_tests_properties(guest_thread_stack_guard PROPERTIES TIMEOUT 20)
+        add_asan_test_options(guest_thread_stack_guard detect_stack_use_after_return=0)
+    endif()
     add_test_executable(guest_cond_timedwait_tests ${LIBS_INCLUDE_DIR}/tests/GuestCondTimedwait.cpp)
     target_include_directories(guest_cond_timedwait_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_cond_timedwait_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -33,6 +33,7 @@
 #include <system_error>
 
 #ifndef _WIN32
+#include <alloca.h>
 #include <cerrno>
 #include <csetjmp>
 #include <csignal>
@@ -411,8 +412,10 @@ static void SetStackFromHost(PthreadPrivate* thread) {
     std::uintptr_t low = 0;
     std::uintptr_t high = 0;
     if (!HostStackLimits(&low, &high)) throw std::runtime_error("Cannot query the host thread stack");
-    thread->stackAddress = reinterpret_cast<void*>(low);
-    thread->stackSize = static_cast<std::size_t>(high - low);
+    const auto start = (low + PS5_PAGE_SIZE - 1) & ~std::uintptr_t{PS5_PAGE_SIZE - 1};
+    if (start >= high) throw std::runtime_error("The host thread stack holds no whole guest page");
+    thread->stackAddress = reinterpret_cast<void*>(start);
+    thread->stackSize = static_cast<std::size_t>(high - start);
 }
 
 static bool CurrentStack(std::uintptr_t* low, std::uintptr_t* high) {
@@ -1085,9 +1088,9 @@ static void* StartNativeThread(void* opaque) {
         std::uintptr_t low = 0;
         std::uintptr_t high = 0;
         const auto reserve = HostStackReserve();
-        if (!HostStackLimits(&low, &high) || high - low < reserve + self->stackSize)
+        if (!HostStackLimits(&low, &high) || high - low < reserve + self->stackSize + PS5_PAGE_SIZE - 1)
             throw std::runtime_error("Cannot query guest thread stack");
-        self->stackAddress = reinterpret_cast<void*>(high - reserve - self->stackSize);
+        self->stackAddress = reinterpret_cast<void*>((high - reserve - self->stackSize) & ~std::uintptr_t{PS5_PAGE_SIZE - 1});
         self->threadId = std::this_thread::get_id();
 #if defined(__linux__)
         self->hostTid = static_cast<int>(gettid());
@@ -1109,6 +1112,9 @@ static void* StartNativeThread(void* opaque) {
             ReleaseThread(self);
         }
     } guard{self};
+    const auto stackEnd = reinterpret_cast<std::uintptr_t>(self->stackAddress) + self->stackSize;
+    if (const auto frame = reinterpret_cast<std::uintptr_t>(__builtin_frame_address(0)); frame > stackEnd)
+        static_cast<volatile unsigned char*>(alloca(frame - stackEnd))[0] = 0;
     std::jmp_buf exitJump;
     if (_setjmp(exitJump) == 0) {
         threadExitJump = &exitJump;

core/libs/tests/GuestThreadStackGuard.cpp

@@ -0,0 +1,46 @@
+#include "prx/libkernel/include/Api.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+#include <initializer_list>
+
+static constexpr int SCE_OK = 0;
+static constexpr std::uintptr_t GUEST_PAGE = 0x4000;
+static constexpr std::size_t GUARD_SIZE = 3 * GUEST_PAGE;
+static constexpr std::size_t STACK_SIZES[] = {65536, 1u << 20, 16u << 20};
+
+static void Require(bool value) { if (!value) std::abort(); }
+
+static void* APS5_VABI GuardStackBottom(void* requested) {
+    PthreadAttr attr = nullptr;
+    Require(scePthreadAttrInit(&attr) == SCE_OK);
+    Require(scePthreadAttrGet(scePthreadSelf(), &attr) == SCE_OK);
+    void* address = nullptr;
+    std::size_t size = 0;
+    Require(scePthreadAttrGetstack(&attr, &address, &size) == SCE_OK);
+    Require(scePthreadAttrDestroy(&attr) == SCE_OK);
+    const auto start = reinterpret_cast<std::uintptr_t>(address);
+    Require(start % GUEST_PAGE == 0);
+    if (requested != nullptr) Require(size == *static_cast<const std::size_t*>(requested));
+    int local = 0;
+    Require(reinterpret_cast<std::uintptr_t>(&local) > start + GUARD_SIZE && reinterpret_cast<std::uintptr_t>(&local) < start + size);
+    void* low = nullptr;
+    void* high = nullptr;
+    Require(sceKernelIsStack(&local, &low, &high) == SCE_OK);
+    Require(low == address && reinterpret_cast<std::uintptr_t>(high) == start + size);
+    for (const int protection : {3, 0, 3}) Require(mprotect_nid_postfix(address, GUARD_SIZE, protection) == 0);
+    return nullptr;
+}
+
+int main() {
+    GuardStackBottom(nullptr);
+    for (auto stackSize : STACK_SIZES) {
+        PthreadAttr attr = nullptr;
+        Require(scePthreadAttrInit(&attr) == SCE_OK);
+        Require(scePthreadAttrSetstacksize(&attr, stackSize) == SCE_OK);
+        Pthread thread = nullptr;
+        Require(scePthreadCreate(&thread, &attr, GuardStackBottom, &stackSize, nullptr) == SCE_OK);
+        Require(scePthreadAttrDestroy(&attr) == SCE_OK);
+        Require(scePthreadJoin(thread, nullptr) == SCE_OK);
+    }
+}

docs/dev/debt/libkernel.md

@@ -229,7 +229,7 @@
 - [getcontext and setcontext](../../../core/libs/prx/libkernel/System/src/UserContext.cpp) (libkernel) keep the 512-byte FXSAVE image in `mc_fpstate` (x87, MXCSR, XMM0-15), so the upper halves of YMM0-15 are neither saved nor restored, and `getcontext` leaves `mc_xfpustate` and `mc_xfpustate_len` 0. `getcontext` sets `mc_fpformat` and `mc_ownedfp` to FreeBSD's `_MC_FPFMT_XMM` and `_MC_FPOWNED_FPU`, `mc_cs` and `mc_ss` to the host selectors, and `mc_flags`, `mc_fsbase` and `mc_gsbase` to 0. `setcontext` loads `mc_fpstate` whatever `mc_fpformat` and `mc_ownedfp` say, and restores neither the segment selectors, `mc_fsbase` and `mc_gsbase` nor the extended state, which FreeBSD restores when `mc_flags` asks for them
 - [setcontext](../../../core/libs/prx/libkernel/System/src/UserContext.cpp) (libkernel) and [longjmp](../../../core/libs/prx/libc/src/JumpBuffer.cpp) (libc) on Windows set the thread's stack bounds in the TEB to the stack they switch to: a `makecontext` stack, or else the committed region around the target stack pointer. A switch to any other guest stack, such as a context built by hand, gets the bounds of that region only
 - [scePthreadAttrGet](../../../core/libs/prx/libkernel/Pthread/src/Attr.cpp) and [pthread_attr_get_np](../../../core/libs/prx/libkernel/Pthread/Posix/Attr.cpp) (libkernel) - the answer for a thread that has ended is not measured on the console; the implementation follows FreeBSD 11.4's [`_pthread_attr_get_np`](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libthr/thread/thr_attr.c#L139-L172) through `_thr_find_thread`. On Windows a host thread adopted by `scePthreadSelf` that ends without `scePthreadExit` keeps answering 0. Monster Hunter Wilds (PPSA24191) reuses a slot of its thread table only when `scePthreadAttrGet` returns `0x80020003`
-- [scePthreadAttrGet, pthread_attr_get_np and sceKernelIsStack](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) report the stack of a host thread the library did not create, on Windows, from one allocation granule (64 KiB) above the bottom of its reservation when everything above that is committed read-write, as in a relinked executable's main thread and its default-size host threads (0x200000 committed of 0x210000 reserved), so the guest can protect the lowest reported pages, as HotSpot does for its stack guard pages in Java 25 LAN / DNS (PPSA99869). Any other such thread, and every such thread on Linux, gets its whole host stack: on Windows `mprotect` commits its reserved lower pages before protecting them and drops the guard flag of Windows' guard pages, and on Linux it cannot change the unmapped lower pages
+- [scePthreadAttrGet, pthread_attr_get_np and sceKernelIsStack](../../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) report the stack of a host thread the library did not create, on Windows, from one allocation granule (64 KiB) above the bottom of its reservation when everything above that is committed read-write, as in a relinked executable's main thread and its default-size host threads (0x200000 committed of 0x210000 reserved), so the guest can protect the lowest reported pages, as HotSpot does for its stack guard pages in Java 25 LAN / DNS (PPSA99869). Any other such thread gets its whole host stack, and every such thread on Linux its host stack from the first 16 KiB boundary up: on Windows `mprotect` commits its reserved lower pages before protecting them and drops the guard flag of Windows' guard pages
 - [sceKernelInstallExceptionHandler](../../../core/libs/prx/libkernel/System/src/Exception.cpp) (libkernel) delivers a page fault on a mapped page with the wrong protection as SIGSEGV (`mc_trapno` 12), FreeBSD's choice under the default `machdep.prot_fault_translation`; [fpPS4](https://github.com/red-prig/fpPS4/blob/036a611bef2ca35da838c4d57d8502907ed5e4a1/sys/kern/trap.pas#L630-L667) delivers SIGBUS there, and the console's choice is not verified. The other traps follow FreeBSD 11's amd64 [`trap()`](https://github.com/freebsd/freebsd-src/blob/stable/11/sys/amd64/amd64/trap.c). Only faults in title code are delivered: the guest arena, on Windows the `.elf` sections of relinked images (the whole executable when it is not relinked), on Linux the executable and `.guest.prx` modules. Unity's PS5Util module installs a SIGUSR1 handler that reads `uc_mcontext.mc_rsp` (offset 0xf8 of the context) and raises SIGUSR1 on other threads to stop them for garbage collection
 - [sceKernelInstallExceptionHandler](../../../core/libs/prx/libkernel/System/src/Exception.cpp) (libkernel) leaves faults in the system libraries, the host runtime and drivers to the crash report, though on the console a fault inside libc or libkernel reaches the handler: here that code is host code a handler cannot resume. A stack overflow ends the process, and an in-page I/O error (`EXCEPTION_IN_PAGE_ERROR` on Windows) is not delivered; the console's signals for them are not verified
 - [sceKernelInstallExceptionHandler](../../../core/libs/prx/libkernel/System/src/Exception.cpp) (libkernel) delivers a fault raised inside a running handler again, as with `SA_NODEFER`; FreeBSD blocks the signal while its handler runs and ends the process on a second synchronous fault. A handler that returns without changing the context runs again on the same fault, as FreeBSD's `sigreturn` resumes the faulting instruction; the console's behaviour there is not verified