fix(libkernel): give Linux guest threads the stack size they request

core/libs/prx/libkernel/CMakeLists.txt

@@ -293,6 +293,16 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_start PROPERTIES TIMEOUT 60 ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_pthread_stack_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadStack.cpp)
+    target_include_directories(guest_pthread_stack_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_stack_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_stack_tests)
+    add_test(NAME guest_pthread_stack COMMAND guest_pthread_stack_tests)
+    set_tests_properties(guest_pthread_stack PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_stack PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
     add_test_executable(guest_cond_timedwait_tests ${LIBS_INCLUDE_DIR}/tests/GuestCondTimedwait.cpp)
     target_include_directories(guest_cond_timedwait_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_cond_timedwait_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/Pthread/include/Pthread.hpp

@@ -15,6 +15,10 @@
 #include <string>
 #include <thread>
 
+#ifndef _WIN32
+#include <pthread.h>
+#endif
+
 enum class MutexType : std::uint32_t {
     ErrorCheck = 1,
     Recursive = 2,
@@ -97,7 +101,7 @@ struct PthreadPrivate {
 #ifdef _WIN32
     void* nativeHandle = nullptr;
 #else
-    std::thread _thr;
+    pthread_t hostThread{};
 #endif
     std::thread::id threadId;
     std::atomic<unsigned> references{2};

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -8,6 +8,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <functional>
+#include <limits>
 #include <map>
 #include <utility>
 #include <string>
@@ -21,6 +22,7 @@
 #include <cerrno>
 #include <csetjmp>
 #include <pthread.h>
+#include <unistd.h>
 #endif
 
 static constexpr int SCE_OK = 0;
@@ -32,7 +34,6 @@ static constexpr std::size_t THREAD_NAME_CAPACITY = 32;
 #ifdef _WIN32
 #include <windows.h>
 #include <process.h>
-#include <limits>
 #endif
 
 struct ThreadArgs {
@@ -222,6 +223,29 @@ static thread_local std::unique_ptr<PthreadPrivate> adoptedThread;
 // and aborts. scePthreadExit instead jumps back to the thread start routine, which,
 // like _endthreadex on Windows, skips the guest frames without unwinding them.
 static thread_local std::jmp_buf* threadExitJump = nullptr;
+
+static void* MeasureStackReserve(void* reserve) {
+    std::uintptr_t low = 0;
+    std::uintptr_t high = 0;
+    if (HostStackLimits(&low, &high))
+        *static_cast<std::size_t*>(reserve) = high - reinterpret_cast<std::uintptr_t>(&low);
+    return nullptr;
+}
+
+static std::size_t HostStackReserve() {
+    static const std::size_t reserve = [] {
+        std::size_t measured = 0;
+        pthread_t probe{};
+        if (const int result = pthread_create(&probe, nullptr, MeasureStackReserve, &measured); result != 0)
+            throw std::system_error(result, std::generic_category(), "Measuring the host thread stack reserve");
+        if (const int result = pthread_join(probe, nullptr); result != 0)
+            throw std::system_error(result, std::generic_category(), "Measuring the host thread stack reserve");
+        if (measured == 0)
+            throw std::runtime_error("Cannot measure the host thread stack reserve");
+        return measured;
+    }();
+    return reserve;
+}
 #endif
 
 void ThreadLifecycle::SetThreadDtors(thread_dtors_func_t callback) {
@@ -272,7 +296,6 @@ static void RunThread(std::unique_ptr<ThreadArgs> args) {
     PthreadPrivate* self = args->self;
     BindHostCpuClock(self);
     TimedWait::BindThreadWaitState(&self->waitCount);
-    if (!self->stackAddress) SetStackFromHost(self);
     CurrentGuestState().current = self;
     RegisterStack(self);
     args.reset();
@@ -324,13 +347,15 @@ static void ApplyJobAffinity(PthreadPrivate& thread, void* handle) {
     });
     CpuTopology::PinTraced(thread.name.c_str(), handle, mask);
 }
+#endif
 
 struct NativeThreadArgs {
     std::unique_ptr<ThreadArgs> guest;
     std::future<bool> start;
     std::promise<void> initialized;
 };
 
+#ifdef _WIN32
 static unsigned __stdcall StartNativeThread(void* opaque) {
     std::unique_ptr<NativeThreadArgs> args(static_cast<NativeThreadArgs*>(opaque));
     auto* self = args->guest->self;
@@ -368,6 +393,42 @@ static unsigned __stdcall StartNativeThread(void* opaque) {
     ReleaseThread(self);
     return 0;
 }
+#else
+static void* StartNativeThread(void* opaque) {
+    std::unique_ptr<NativeThreadArgs> args(static_cast<NativeThreadArgs*>(opaque));
+    auto* self = args->guest->self;
+    try {
+        std::uintptr_t low = 0;
+        std::uintptr_t high = 0;
+        const auto reserve = HostStackReserve();
+        if (!HostStackLimits(&low, &high) || high - low < reserve + self->stackSize)
+            throw std::runtime_error("Cannot query guest thread stack");
+        self->stackAddress = reinterpret_cast<void*>(high - reserve - self->stackSize);
+        self->threadId = std::this_thread::get_id();
+        args->initialized.set_value();
+    } catch (...) {
+        args->initialized.set_exception(std::current_exception());
+        return nullptr;
+    }
+    if (!args->start.get())
+        return nullptr;
+    auto* guest = args->guest.release();
+    args.reset();
+    struct ThreadGuard {
+        PthreadPrivate* self;
+        ~ThreadGuard() {
+            CurrentGuestState().current = nullptr;
+            ReleaseThread(self);
+        }
+    } guard{self};
+    std::jmp_buf exitJump;
+    if (setjmp(exitJump) == 0) {
+        threadExitJump = &exitJump;
+        RunThread(std::unique_ptr<ThreadArgs>(guest));
+    }
+    threadExitJump = nullptr;
+    return nullptr;
+}
 #endif
 
 extern "C" {
@@ -414,29 +475,32 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
     if (detached)
         ReleaseThread(published);
 #else
-    auto* self = p.get();
-    p->_thr = std::thread([self, args = std::move(args), ready = start.get_future()]() mutable {
-        if (!ready.get()) return;
-        self->threadId = std::this_thread::get_id();
-        struct ThreadGuard {
-            PthreadPrivate* self;
-            ~ThreadGuard() {
-                CurrentGuestState().current = nullptr;
-                ReleaseThread(self);
-            }
-        } guard{self};
-        std::jmp_buf exitJump;
-        if (setjmp(exitJump) == 0) {
-            threadExitJump = &exitJump;
-            RunThread(std::move(args));
-        }
-        threadExitJump = nullptr;
-    });
+    const auto reserve = HostStackReserve();
+    const auto page = static_cast<std::size_t>(sysconf(_SC_PAGESIZE));
+    if (p->stackSize < 16384 || p->stackSize > std::numeric_limits<std::size_t>::max() - reserve - page)
+        throw std::runtime_error("scePthreadCreate: invalid stack size");
+    const std::size_t nativeStack = (p->stackSize + reserve + page - 1) / page * page;
+    auto native = std::make_unique<NativeThreadArgs>(NativeThreadArgs{std::move(args), start.get_future(), {}});
+    auto initialized = native->initialized.get_future();
+    pthread_attr_t nativeAttr;
+    if (const int result = pthread_attr_init(&nativeAttr); result != 0)
+        throw std::system_error(result, std::generic_category(), "Creating guest thread");
+    int created = pthread_attr_setstacksize(&nativeAttr, nativeStack);
+    if (created == 0)
+        created = pthread_create(&p->hostThread, &nativeAttr, StartNativeThread, native.get());
+    pthread_attr_destroy(&nativeAttr);
+    if (created != 0)
+        throw std::system_error(created, std::generic_category(), "Creating guest thread");
+    native.release();
     try {
-        if (detached) p->_thr.detach();
+        initialized.get();
+        if (detached) {
+            if (const int result = pthread_detach(p->hostThread); result != 0)
+                throw std::system_error(result, std::generic_category(), "Detaching guest thread");
+        }
     } catch (...) {
         start.set_value(false);
-        p->_thr.join();
+        pthread_join(p->hostThread, nullptr);
         throw;
     }
     auto* published = p.release();
@@ -459,7 +523,8 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) {
     if (retval) *retval = thread->_retval;
     ReleaseThread(thread);
 #else
-    if (thread->_thr.joinable()) thread->_thr.join();
+    if (const int result = pthread_join(thread->hostThread, nullptr); result != 0)
+        throw std::system_error(result, std::generic_category(), "Joining guest thread");
     if (retval) *retval = thread->_retval;
     ReleaseThread(thread);
 #endif
@@ -473,7 +538,8 @@ int APS5_VABI scePthreadDetach(Pthread thread) {
 #ifdef _WIN32
     ReleaseThread(thread);
 #else
-    if (thread->_thr.joinable()) thread->_thr.detach();
+    if (const int result = pthread_detach(thread->hostThread); result != 0)
+        throw std::system_error(result, std::generic_category(), "Detaching guest thread");
     ReleaseThread(thread);
 #endif
     return SCE_OK;

core/libs/tests/GuestPthreadStack.cpp

@@ -0,0 +1,76 @@
+#include "SceTypes.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+
+extern "C" {
+int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name);
+int APS5_VABI scePthreadJoin(Pthread thread, void** retval);
+int APS5_VABI scePthreadAttrInit(PthreadAttr* attr);
+int APS5_VABI scePthreadAttrDestroy(PthreadAttr* attr);
+int APS5_VABI scePthreadAttrGet(Pthread thread, PthreadAttr* attr);
+int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stackSize);
+int APS5_VABI scePthreadAttrGetstack(const PthreadAttr* attr, void** stackAddress, std::size_t* stackSize);
+Pthread APS5_VABI scePthreadSelf();
+}
+
+static constexpr int SCE_OK = 0;
+static constexpr std::size_t FRAME_SIZE = 4096;
+static constexpr std::size_t FRAME_MARGIN = 2 * FRAME_SIZE;
+static constexpr std::size_t STACK_SIZES[] = {16384, 65536, 1u << 20, 16u << 20};
+
+thread_local unsigned char threadLocalBlock[512 * 1024];
+
+static void Require(bool value) { if (!value) std::abort(); }
+
+static std::uintptr_t Descend(std::uintptr_t floor);
+static std::uintptr_t (*volatile descend)(std::uintptr_t) = Descend;
+
+static std::uintptr_t Descend(std::uintptr_t floor) {
+    volatile unsigned char frame[FRAME_SIZE];
+    frame[0] = 0x5a;
+    frame[FRAME_SIZE - 1] = 0xa5;
+    const auto here = reinterpret_cast<std::uintptr_t>(&frame[0]);
+    const auto deepest = here < floor + FRAME_MARGIN ? here : descend(floor);
+    Require(frame[0] == 0x5a && frame[FRAME_SIZE - 1] == 0xa5);
+    return deepest;
+}
+
+static void* APS5_VABI Worker(void* arg) {
+    const auto requested = *static_cast<const std::size_t*>(arg);
+    threadLocalBlock[0] = 1;
+    threadLocalBlock[sizeof(threadLocalBlock) - 1] = 2;
+
+    PthreadAttr attr = nullptr;
+    Require(scePthreadAttrInit(&attr) == SCE_OK);
+    Require(scePthreadAttrGet(scePthreadSelf(), &attr) == SCE_OK);
+    void* address = nullptr;
+    std::size_t size = 0;
+    Require(scePthreadAttrGetstack(&attr, &address, &size) == SCE_OK);
+    Require(scePthreadAttrDestroy(&attr) == SCE_OK);
+    Require(size == requested);
+
+    const auto begin = reinterpret_cast<std::uintptr_t>(address);
+    const auto local = reinterpret_cast<std::uintptr_t>(&size);
+    Require(local > begin && local < begin + size);
+    Require(local - begin + FRAME_SIZE >= requested);
+    const auto deepest = Descend(begin);
+    Require(deepest >= begin && deepest < begin + FRAME_MARGIN);
+
+    Require(threadLocalBlock[0] == 1 && threadLocalBlock[sizeof(threadLocalBlock) - 1] == 2);
+    return arg;
+}
+
+int main() {
+    for (auto stackSize : STACK_SIZES) {
+        PthreadAttr attr = nullptr;
+        Require(scePthreadAttrInit(&attr) == SCE_OK);
+        Require(scePthreadAttrSetstacksize(&attr, stackSize) == SCE_OK);
+        Pthread thread = nullptr;
+        Require(scePthreadCreate(&thread, &attr, Worker, &stackSize, nullptr) == SCE_OK);
+        Require(scePthreadAttrDestroy(&attr) == SCE_OK);
+        void* result = nullptr;
+        Require(scePthreadJoin(thread, &result) == SCE_OK);
+        Require(result == &stackSize);
+    }
+}

core/libs/tests/PthreadStart.cpp

@@ -41,8 +41,7 @@ static void* APS5_VABI CheckThread(void* arg) {
     if (context.thread->threadId != std::this_thread::get_id() || !context.thread->nativeHandle)
         throw std::runtime_error("Native thread was not initialized");
 #else
-    if (context.thread->_thr.get_id() != std::this_thread::get_id()) throw std::runtime_error("Thread object was not initialized");
-    if (!context.thread->_thr.joinable()) throw std::runtime_error("Thread object is not joinable");
+    if (!pthread_equal(context.thread->hostThread, pthread_self())) throw std::runtime_error("Thread object was not initialized");
 #endif
     PthreadAttr attr = nullptr;
     if (scePthreadAttrInit(&attr) != 0) throw std::runtime_error("Attribute initialization failed");