fix(libkernel): honor caller-provided pthread stacks

.github/sanitizer-known-failures.txt

@@ -5,4 +5,5 @@ agc_performance_overlay address,undefined Runs past its 30 s ctest TIMEOUT in th
 guest_backtrace address Fails only in ASan builds, with 'guest backtrace: first frame is not in the calling module', because the sanitizer adds a frame to backtrace(); #9240 starts the frames at the caller.
 guest_getrlimit address The test lowers RLIMIT_DATA to 1 TiB, below the private memory ASan has already mapped, so ASan's next internal mapping fails with ENOMEM ('failed to allocate ... DTLS_NextBlock').
 guest_math address,undefined ubuntu-latest's GCC 15 targets x86-64-v3 and fuses libc's float multiply-adds into FMA, so atanf misses the console bits by one ulp; #11059 builds libc with -ffp-contract=off.
+guest_pthread_provided_stack address A thread on a caller stack returns through uc_link from a makecontext context on a larger stack that contains its own, and ASan, which still takes the caller stack for the current one, ignores the __asan_handle_no_return in setcontext_nid_postfix and reports a stack-buffer-overflow on the stale redzones, because makecontext and setcontext have no fiber annotations.
 guest_vorbis_dec undefined The bundled libvorbis left-shifts a negative value (3rdparty/vorbis/lib/psy.c:319); #11480 updates libvorbis to the upstream fix.

core/libs/prx/libc/include/GuestStacks.hpp

@@ -10,6 +10,8 @@ namespace GuestStacks {
 extern "C" {
 
 void GuestStackRegister_nid_no_patch(const void* pointer, std::size_t bytes);
+void GuestStackEnter_nid_no_patch(const void* pointer, std::size_t bytes);
+void GuestStackLeave_nid_no_patch();
 void APS5_VABI GuestStackSwitch_nid_no_patch(std::uintptr_t stackPointer);
 
 }

core/libs/prx/libc/src/GuestStacks.cpp

@@ -22,6 +22,8 @@ constexpr std::size_t DeallocationStackOffset = 0x1478;
 
 std::mutex rangesLock;
 std::map<std::uintptr_t, std::uintptr_t> ranges;
+thread_local StackBounds threadStack{};
+thread_local StackBounds hostStack{};
 
 StackBounds CurrentBounds() {
     auto* tib = reinterpret_cast<NT_TIB*>(NtCurrentTeb());
@@ -38,6 +40,12 @@ void ApplyBounds(const StackBounds& bounds) {
     *reinterpret_cast<std::uintptr_t*>(teb + DeallocationStackOffset) = bounds.deallocation;
 }
 
+bool ThreadBounds(std::uintptr_t stackPointer, StackBounds& bounds) {
+    if (stackPointer < threadStack.limit || stackPointer >= threadStack.base) return false;
+    bounds = threadStack;
+    return true;
+}
+
 bool RegisteredBounds(std::uintptr_t stackPointer, StackBounds& bounds) {
     std::lock_guard lock(rangesLock);
     const auto next = ranges.upper_bound(stackPointer);
@@ -80,11 +88,36 @@ void GuestStackRegister_nid_no_patch(const void* pointer, std::size_t bytes) {
 #endif
 }
 
+void GuestStackEnter_nid_no_patch(const void* pointer, std::size_t bytes) {
+#ifdef _WIN32
+    const auto low = reinterpret_cast<std::uintptr_t>(pointer);
+    hostStack = CurrentBounds();
+    threadStack = {low + bytes, low, low};
+    GuestArena::GuestArenaPinWritable_nid_postfix(pointer, bytes);
+    ApplyBounds(threadStack);
+#else
+    (void)pointer;
+    (void)bytes;
+#endif
+}
+
+void GuestStackLeave_nid_no_patch() {
+#ifdef _WIN32
+    ApplyBounds(hostStack);
+    GuestArena::GuestArenaUnpinWritable_nid_postfix(reinterpret_cast<const void*>(threadStack.limit), threadStack.base - threadStack.limit);
+    threadStack = {};
+#endif
+}
+
 void APS5_VABI GuestStackSwitch_nid_no_patch(std::uintptr_t stackPointer) {
 #ifdef _WIN32
+    StackBounds target{};
+    if (ThreadBounds(stackPointer, target)) {
+        ApplyBounds(target);
+        return;
+    }
     const auto current = CurrentBounds();
     if (stackPointer >= current.deallocation && stackPointer < current.base) return;
-    StackBounds target{};
     if (RegisteredBounds(stackPointer, target) || HostBounds(stackPointer, target)) ApplyBounds(target);
 #else
     (void)stackPointer;

core/libs/prx/libkernel/CMakeLists.txt

@@ -887,6 +887,21 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_stack PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    if(WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        add_test_executable(guest_pthread_provided_stack_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadProvidedStack.cpp)
+        target_include_directories(guest_pthread_provided_stack_tests PRIVATE ${LIBS_INCLUDE_DIR})
+        target_compile_options(guest_pthread_provided_stack_tests PRIVATE -fno-omit-frame-pointer)
+        target_link_libraries(guest_pthread_provided_stack_tests PRIVATE libSceLibcInternal libkernel libc)
+        configure_windows_unwind(guest_pthread_provided_stack_tests)
+        add_test(NAME guest_pthread_provided_stack COMMAND guest_pthread_provided_stack_tests)
+        set_tests_properties(guest_pthread_provided_stack PROPERTIES TIMEOUT 20)
+        add_asan_test_options(guest_pthread_provided_stack detect_stack_use_after_return=0)
+        if(WIN32)
+            target_compile_definitions(guest_pthread_provided_stack_tests PRIVATE _WIN32_WINNT=0x0602)
+            set_tests_properties(guest_pthread_provided_stack PROPERTIES ENVIRONMENT_MODIFICATION
+                "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+        endif()
+    endif()
     add_test_executable(guest_cond_timedwait_tests ${LIBS_INCLUDE_DIR}/tests/GuestCondTimedwait.cpp)
     target_include_directories(guest_cond_timedwait_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_cond_timedwait_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/Pthread/Posix/Attr.cpp

@@ -27,6 +27,7 @@ int APS5_VABI pthread_attr_get_np_nid_postfix(Pthread thread, PthreadAttr* attr)
     auto* p = *attr;
     p->_stacksize = thread->stackSize;
     p->stackAddress = thread->stackAddress;
+    if (thread->callerStack) p->_guardsize = 0;
     p->_detachstate = thread->_detached ? 1 : 0;
     p->_schedpriority = thread->priority.load(std::memory_order_relaxed);
     p->_schedpolicy = thread->schedPolicy.load(std::memory_order_relaxed);

core/libs/prx/libkernel/Pthread/include/Pthread.hpp

@@ -145,6 +145,7 @@ struct PthreadPrivate {
     void* wakeEvent = nullptr;
     void* stackAddress = nullptr;
     std::size_t stackSize = 0;
+    bool callerStack = false;
     std::atomic<int> waitCount{0};
     std::atomic<KernelCpumask> affinity{DEFAULT_THREAD_AFFINITY};
     std::atomic<int> priority{DEFAULT_THREAD_PRIORITY};

core/libs/prx/libkernel/Pthread/src/Attr.cpp

@@ -94,6 +94,7 @@ int APS5_VABI scePthreadAttrGet(Pthread thread, PthreadAttr* attr) {
     if (!live) return SCE_KERNEL_ERROR_ESRCH;
     (*attr)->_stacksize = thread->stackSize;
     (*attr)->stackAddress = thread->stackAddress;
+    if (thread->callerStack) (*attr)->_guardsize = 0;
     (*attr)->_detachstate = thread->_detached ? DETACH_DETACHED : DETACH_JOINABLE;
     (*attr)->_schedpriority = thread->priority.load(std::memory_order_relaxed);
     (*attr)->_schedpolicy = thread->schedPolicy.load(std::memory_order_relaxed);

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -13,6 +13,7 @@
 #include "prx/libkernel/KernelErrors.hpp"
 #include "prx/libc/include/CpuTopology.hpp"
 #include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestStacks.hpp"
 #include "prx/libkernel/DirectMemory/DirectMemory.hpp"
 #include <algorithm>
 #include <cstdio>
@@ -53,6 +54,10 @@ extern "C" bool IsInitialPthread_nid_no_patch() {
 #endif
 }
 
+#ifdef ANYPS5_ENABLE_ASAN
+#include <sanitizer/common_interface_defs.h>
+#endif
+
 static constexpr int SCE_OK = 0;
 static constexpr std::size_t GUEST_THREAD_NAME_CAPACITY = 32;
 static constexpr int GUEST_EINVAL = 22;
@@ -94,6 +99,9 @@ struct GuestThreadState {
     int cancelState = GUEST_CANCEL_ENABLE;
     int cancelType = GUEST_CANCEL_DEFERRED;
     CleanupHandler* cleanupHandlers = nullptr;
+#ifdef _WIN32
+    std::uint64_t* exitJump = nullptr;
+#endif
 };
 static GuestThreadState& CurrentGuestState() {
 #ifdef _WIN32
@@ -117,9 +125,11 @@ static auto& guestStackMappings = *new std::map<std::uintptr_t, GuestStackMappin
 
 static void RegisterGuestStackMapping(PthreadPrivate* thread) {
     const auto mappingStart = reinterpret_cast<std::uintptr_t>(thread->guestStack);
-    const auto stackStart = reinterpret_cast<std::uintptr_t>(thread->stackAddress);
+    const auto mappingEnd = mappingStart + thread->guestStackBytes;
+    const auto stackStart = thread->callerStack ? mappingEnd : reinterpret_cast<std::uintptr_t>(thread->stackAddress);
+    const auto stackEnd = thread->callerStack ? mappingEnd : stackStart + thread->stackSize;
     std::lock_guard lock(guestStackMappingsLock);
-    guestStackMappings[mappingStart] = {mappingStart + thread->guestStackBytes, stackStart, stackStart + thread->stackSize};
+    guestStackMappings[mappingStart] = {mappingEnd, stackStart, stackEnd};
 }
 
 static void UnregisterGuestStackMapping(void* mapping) {
@@ -431,7 +441,7 @@ static void UnregisterStack(PthreadPrivate* self) {
         const auto [low, high] = found->second;
         for (auto it = protectedStackPages.begin(); it != protectedStackPages.end();) {
             if (it->first < high && it->first + it->second > low) {
-                RestoreStackPages(it->first, it->second);
+                if (!self->callerStack) RestoreStackPages(it->first, it->second);
                 it = protectedStackPages.erase(it);
             } else {
                 ++it;
@@ -793,24 +803,32 @@ void ThreadCancel::EndWait() {
     BeginWait(nullptr);
 }
 
-extern "C" void* APS5_VABI Aps5CallGuestEntry_nid_no_patch(PthreadEntry entry, void* arg);
+extern "C" void* APS5_VABI Aps5CallGuestEntry_nid_no_patch(PthreadEntry entry, void* arg, std::uintptr_t stackTop);
 
 asm(".text\n"
     APS5_ASM_FUNCTION("Aps5CallGuestEntry_nid_no_patch") R"(
     .cfi_startproc
     push %rbp
     .cfi_adjust_cfa_offset 8
     .cfi_rel_offset %rbp, 0
+    mov %rsp, %rbp
+    .cfi_def_cfa_register %rbp
+    test %rdx, %rdx
+    jz 1f
+    mov %rdx, %rsp
+    and $-16, %rsp
+1:
+    push %rbp
+    push %rbp
     push $0
-    .cfi_adjust_cfa_offset 8
     push $0
-    .cfi_adjust_cfa_offset 8
     mov %rsp, %rbp
+    .cfi_escape 0x0f, 0x05, 0x76, 0x10, 0x06, 0x23, 0x10
     mov %rdi, %rax
     mov %rsi, %rdi
     call *%rax
-    add $16, %rsp
-    .cfi_adjust_cfa_offset -16
+    mov 16(%rbp), %rsp
+    .cfi_def_cfa %rsp, 16
     pop %rbp
     .cfi_adjust_cfa_offset -8
     .cfi_restore %rbp
@@ -844,7 +862,7 @@ static void RunThread(std::unique_ptr<ThreadArgs> args) {
     pthread_sigmask(SIG_SETMASK, &signalMask, nullptr);
 #endif
 #endif
-    finishThread(self, Aps5CallGuestEntry_nid_no_patch(entry, arg));
+    finishThread(self, Aps5CallGuestEntry_nid_no_patch(entry, arg, 0));
     CurrentGuestState().current = nullptr;
 }
 
@@ -970,18 +988,30 @@ static std::wstring ThreadDescription(const std::string& name) {
     return description;
 }
 
+static void* APS5_VABI RunGuestThreadOnCallerStack_nid_no_patch(void* opaque) {
+    std::uint64_t exitJump[12];
+    if (setjmp_nid_postfix(exitJump) == 0) {
+        CurrentGuestState().exitJump = exitJump;
+        RunThread(std::unique_ptr<ThreadArgs>(static_cast<ThreadArgs*>(opaque)));
+    }
+    CurrentGuestState().exitJump = nullptr;
+    return nullptr;
+}
+
 static unsigned __stdcall StartNativeThread(void* opaque) {
     std::unique_ptr<NativeThreadArgs> args(static_cast<NativeThreadArgs*>(opaque));
     auto* self = args->guest->self;
     try {
-        ULONG_PTR low = 0;
-        ULONG_PTR high = 0;
-        GetCurrentThreadStackLimits(&low, &high);
-        if (high <= low || high - low < self->stackSize)
-            throw std::runtime_error("Cannot query guest thread stack");
-        self->stackAddress = reinterpret_cast<void*>(high - self->stackSize);
-        if (!CommittedReadWrite(high - self->stackSize, high))
-            throw std::runtime_error("Guest thread stack is not fully committed");
+        if (!self->callerStack) {
+            ULONG_PTR low = 0;
+            ULONG_PTR high = 0;
+            GetCurrentThreadStackLimits(&low, &high);
+            if (high <= low || high - low < self->stackSize)
+                throw std::runtime_error("Cannot query guest thread stack");
+            self->stackAddress = reinterpret_cast<void*>(high - self->stackSize);
+            if (!CommittedReadWrite(high - self->stackSize, high))
+                throw std::runtime_error("Guest thread stack is not fully committed");
+        }
         self->threadId = std::this_thread::get_id();
         CurrentGuestState().current = self;
         if (!self->name.empty()) {
@@ -997,22 +1027,51 @@ static unsigned __stdcall StartNativeThread(void* opaque) {
         return 0;
     auto guest = std::move(args->guest);
     args.reset();
-    RunThread(std::move(guest));
+    if (self->callerStack) {
+        GuestStacks::GuestStackEnter_nid_no_patch(self->stackAddress, self->stackSize);
+        Aps5CallGuestEntry_nid_no_patch(RunGuestThreadOnCallerStack_nid_no_patch, guest.release(), reinterpret_cast<std::uintptr_t>(self->stackAddress) + self->stackSize);
+        GuestStacks::GuestStackLeave_nid_no_patch();
+    } else {
+        RunThread(std::move(guest));
+    }
     CurrentGuestState().current = nullptr;
     ReleaseThread(self);
     return 0;
 }
 #else
+static void* APS5_VABI RunGuestThread_nid_no_patch(void* opaque) {
+    std::jmp_buf exitJump;
+    if (_setjmp(exitJump) == 0) {
+        threadExitJump = &exitJump;
+        RunThread(std::unique_ptr<ThreadArgs>(static_cast<ThreadArgs*>(opaque)));
+    }
+    threadExitJump = nullptr;
+    return nullptr;
+}
+
+#ifdef ANYPS5_ENABLE_ASAN
+static void* APS5_VABI RunGuestThreadOnCallerStack_nid_no_patch(void* opaque) {
+    const void* nativeStack = nullptr;
+    std::size_t nativeStackSize = 0;
+    __sanitizer_finish_switch_fiber(nullptr, &nativeStack, &nativeStackSize);
+    RunGuestThread_nid_no_patch(opaque);
+    __sanitizer_start_switch_fiber(nullptr, nativeStack, nativeStackSize);
+    return nullptr;
+}
+#endif
+
 static void* StartNativeThread(void* opaque) {
     std::unique_ptr<NativeThreadArgs> args(static_cast<NativeThreadArgs*>(opaque));
     auto* self = args->guest->self;
     try {
-        std::uintptr_t low = 0;
-        std::uintptr_t high = 0;
-        const auto reserve = HostStackReserve();
-        if (!HostStackLimits(&low, &high) || high - low < reserve + self->stackSize)
-            throw std::runtime_error("Cannot query guest thread stack");
-        self->stackAddress = reinterpret_cast<void*>(high - reserve - self->stackSize);
+        if (!self->callerStack) {
+            std::uintptr_t low = 0;
+            std::uintptr_t high = 0;
+            const auto reserve = HostStackReserve();
+            if (!HostStackLimits(&low, &high) || high - low < reserve + self->stackSize)
+                throw std::runtime_error("Cannot query guest thread stack");
+            self->stackAddress = reinterpret_cast<void*>(high - reserve - self->stackSize);
+        }
         self->threadId = std::this_thread::get_id();
 #if defined(__linux__)
         self->hostTid = static_cast<int>(gettid());
@@ -1034,12 +1093,19 @@ static void* StartNativeThread(void* opaque) {
             ReleaseThread(self);
         }
     } guard{self};
-    std::jmp_buf exitJump;
-    if (_setjmp(exitJump) == 0) {
-        threadExitJump = &exitJump;
-        RunThread(std::unique_ptr<ThreadArgs>(guest));
+    if (self->callerStack) {
+        const auto top = reinterpret_cast<std::uintptr_t>(self->stackAddress) + self->stackSize;
+#ifdef ANYPS5_ENABLE_ASAN
+        void* fakeStack = nullptr;
+        __sanitizer_start_switch_fiber(&fakeStack, self->stackAddress, self->stackSize);
+        Aps5CallGuestEntry_nid_no_patch(RunGuestThreadOnCallerStack_nid_no_patch, guest, top);
+        __sanitizer_finish_switch_fiber(fakeStack, nullptr, nullptr);
+#else
+        Aps5CallGuestEntry_nid_no_patch(RunGuestThread_nid_no_patch, guest, top);
+#endif
+    } else {
+        RunGuestThread_nid_no_patch(guest);
     }
-    threadExitJump = nullptr;
     return nullptr;
 }
 #endif
@@ -1070,14 +1136,20 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
     p->createdSuspended = attr && (*attr)->_createSuspended;
     p->suspended.store(p->createdSuspended, std::memory_order_relaxed);
     p->pendingException.store(0, std::memory_order_relaxed);
+    if (attr && (*attr)->stackAddress) {
+        p->stackAddress = (*attr)->stackAddress;
+        p->callerStack = true;
+    }
+    if (p->callerStack && reinterpret_cast<std::uintptr_t>(p->stackAddress) > std::numeric_limits<std::uintptr_t>::max() - p->stackSize)
+        throw std::runtime_error("scePthreadCreate: caller stack range overflows the address space");
     std::promise<bool> start;
     auto args = std::make_unique<ThreadArgs>(ThreadArgs{entry, arg, p.get()});
 #ifdef _WIN32
     SYSTEM_INFO system{};
     GetSystemInfo(&system);
-    if (p->stackSize < 16384 || p->stackSize > std::numeric_limits<unsigned>::max() - HOST_STACK_MARGIN - system.dwPageSize)
+    if (p->stackSize < 16384 || (!p->callerStack && p->stackSize > std::numeric_limits<unsigned>::max() - HOST_STACK_MARGIN - system.dwPageSize))
         throw std::runtime_error("scePthreadCreate: invalid Windows stack size");
-    const std::size_t nativeStack = (p->stackSize + HOST_STACK_MARGIN + system.dwPageSize - 1) / system.dwPageSize * system.dwPageSize;
+    const std::size_t nativeStack = ((p->callerStack ? 0 : p->stackSize) + HOST_STACK_MARGIN + system.dwPageSize - 1) / system.dwPageSize * system.dwPageSize;
     auto native = std::make_unique<NativeThreadArgs>(NativeThreadArgs{std::move(args), start.get_future(), {}});
     auto initialized = native->initialized.get_future();
     const auto handle = _beginthreadex(nullptr, static_cast<unsigned>(nativeStack), StartNativeThread, native.get(), 0, nullptr);
@@ -1105,7 +1177,7 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
     const auto page = static_cast<std::size_t>(sysconf(_SC_PAGESIZE));
     if (p->stackSize < 16384 || p->stackSize > std::numeric_limits<std::size_t>::max() - reserve - HOST_STACK_MARGIN - page)
         throw std::runtime_error("scePthreadCreate: invalid stack size");
-    const std::size_t nativeStack = (p->stackSize + reserve + HOST_STACK_MARGIN + page - 1) / page * page;
+    const std::size_t nativeStack = ((p->callerStack ? 0 : p->stackSize) + reserve + HOST_STACK_MARGIN + page - 1) / page * page;
     sigset_t creatorMask;
     pthread_sigmask(SIG_SETMASK, nullptr, &creatorMask);
     args->signalMask = creatorMask;
@@ -1266,6 +1338,8 @@ void APS5_VABI scePthreadExit(void* retval) {
     finishThread(self, retval);
     CurrentGuestState().current = nullptr;
 #ifdef _WIN32
+    if (auto* jump = CurrentGuestState().exitJump)
+        longjmp_nid_postfix(jump, 1);
     ReleaseThread(self);
     _endthreadex(0);
 #else