fix(libkernel): return ESRCH from join, detach and cancel of a released thread handle

core/libs/prx/libkernel/CMakeLists.txt

@@ -738,6 +738,46 @@ if(BUILD_TESTING)
         set_tests_properties(guest_pthread_join_detach PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
+    add_test_executable(guest_pthread_handle_after_exit_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadHandleAfterExit.cpp)
+    target_include_directories(guest_pthread_handle_after_exit_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_handle_after_exit_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_handle_after_exit_tests)
+    add_test(NAME guest_pthread_handle_after_exit COMMAND guest_pthread_handle_after_exit_tests)
+    set_tests_properties(guest_pthread_handle_after_exit PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_handle_after_exit PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
+    add_test_executable(guest_pthread_detach_attr_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadDetachAttr.cpp)
+    target_include_directories(guest_pthread_detach_attr_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_detach_attr_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_detach_attr_tests)
+    add_test(NAME guest_pthread_detach_attr COMMAND guest_pthread_detach_attr_tests)
+    set_tests_properties(guest_pthread_detach_attr PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_detach_attr PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
+    add_test_executable(guest_pthread_released_handle_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadReleasedHandle.cpp)
+    target_include_directories(guest_pthread_released_handle_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_released_handle_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_released_handle_tests)
+    add_test(NAME guest_pthread_released_handle COMMAND guest_pthread_released_handle_tests)
+    set_tests_properties(guest_pthread_released_handle PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_released_handle PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
+    add_test_executable(guest_pthread_final_release_race_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadFinalReleaseRace.cpp)
+    target_include_directories(guest_pthread_final_release_race_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_pthread_final_release_race_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_pthread_final_release_race_tests)
+    add_test(NAME guest_pthread_final_release_race COMMAND guest_pthread_final_release_race_tests)
+    set_tests_properties(guest_pthread_final_release_race PROPERTIES TIMEOUT 20)
+    if(WIN32)
+        set_tests_properties(guest_pthread_final_release_race PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
+    endif()
     add_test_executable(guest_pthread_attr_errors_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttrErrors.cpp)
     target_include_directories(guest_pthread_attr_errors_tests PRIVATE ${LIBS_INCLUDE_DIR})
     target_link_libraries(guest_pthread_attr_errors_tests PRIVATE libkernel libc)

core/libs/prx/libkernel/Pthread/include/Pthread.hpp

@@ -160,7 +160,7 @@ struct PthreadPrivate {
     std::string name;
     std::atomic<bool> _finished;
     void* _retval;
-    bool _detached;
+    std::atomic<bool> _detached;
     bool _adopted;
     Pthread* _handleStorage;
     std::atomic<bool> _joined;

core/libs/prx/libkernel/Pthread/src/Thread.cpp

@@ -554,6 +554,15 @@ std::unique_lock<std::mutex> LockLiveGuestThread(const PthreadPrivate* thread) {
     return {};
 }
 
+static std::unique_lock<std::mutex> LockRegisteredGuestThread(const PthreadPrivate* thread) {
+    auto& registry = CpuClocks();
+    std::unique_lock lock(registry.lock);
+    for (const auto& [id, registered] : registry.threads) {
+        if (registered == thread) return lock;
+    }
+    return {};
+}
+
 bool GuestThreadIsLive(const PthreadPrivate* thread) {
     return LockLiveGuestThread(thread).owns_lock();
 }
@@ -806,11 +815,20 @@ static void RunThread(std::unique_ptr<ThreadArgs> args) {
     CurrentGuestState().current = nullptr;
 }
 
+using FinalThreadReleaseTestHook = void (*)(Pthread);
+static std::atomic<PthreadPrivate*> finalThreadReleaseTestTarget{nullptr};
+static std::atomic<FinalThreadReleaseTestHook> finalThreadReleaseTestHook{nullptr};
+
 static void ReleaseThread(PthreadPrivate* thread) {
     if (thread->_adopted)
         return;
     if (thread->references.fetch_sub(1, std::memory_order_acq_rel) != 1)
         return;
+    auto* expected = thread;
+    if (finalThreadReleaseTestTarget.compare_exchange_strong(expected, nullptr, std::memory_order_acq_rel)) {
+        if (const auto hook = finalThreadReleaseTestHook.exchange(nullptr, std::memory_order_acq_rel))
+            hook(reinterpret_cast<Pthread>(thread));
+    }
 #ifdef _WIN32
     if (thread->wakeEvent)
         CloseHandle(static_cast<HANDLE>(thread->wakeEvent));
@@ -820,6 +838,38 @@ static void ReleaseThread(PthreadPrivate* thread) {
     delete thread;
 }
 
+class GuestThreadReference {
+public:
+    GuestThreadReference() = default;
+    bool Retain(PthreadPrivate* value) {
+        auto references = value->references.load(std::memory_order_relaxed);
+        while (references != 0) {
+            if (value->references.compare_exchange_weak(references, references + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) {
+                thread = value;
+                return true;
+            }
+        }
+        return false;
+    }
+    void Reset() {
+        if (!thread) return;
+        auto* retained = thread;
+        thread = nullptr;
+        ReleaseThread(retained);
+    }
+    ~GuestThreadReference() { Reset(); }
+    GuestThreadReference(const GuestThreadReference&) = delete;
+    GuestThreadReference& operator=(const GuestThreadReference&) = delete;
+
+private:
+    PthreadPrivate* thread = nullptr;
+};
+
+extern "C" void APS5_Test_SetFinalThreadReleaseHook(Pthread thread, FinalThreadReleaseTestHook hook) {
+    finalThreadReleaseTestHook.store(hook, std::memory_order_relaxed);
+    finalThreadReleaseTestTarget.store(static_cast<PthreadPrivate*>(thread), std::memory_order_release);
+}
+
 static std::mutex joinedLock;
 static std::map<Pthread*, PthreadPrivate*> joinedThreads;
 
@@ -1073,15 +1123,26 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread
 
 int APS5_VABI JoinThread_nid_no_patch(Pthread thread, void** retval, const KernelTimespec* abstime) {
     if (!thread) return SCE_KERNEL_ERROR_EINVAL;
+    GuestThreadReference reference;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
+    if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH;
     if (thread == CurrentGuestState().current) return SCE_KERNEL_ERROR_EDEADLK;
     if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL;
     if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH;
+    registered.unlock();
+    reference.Reset();
     ThreadCancel::Check();
+    registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
+    if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH;
     {
         std::unique_lock lock(thread->_join_mtx);
         if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL;
+        if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH;
         if (thread->_joining) return SCE_KERNEL_ERROR_EOPNOTSUPP;
         thread->_joining = true;
+        registered.unlock();
         while (!thread->_finished.load()) {
             std::optional<std::uint64_t> deadline;
             if (abstime != nullptr) {
@@ -1130,12 +1191,17 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval) {
 
 int APS5_VABI scePthreadDetach(Pthread thread) {
     if (!thread) return SCE_KERNEL_ERROR_EINVAL;
+    GuestThreadReference reference;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
+    if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH;
     {
         std::lock_guard lock(thread->_join_mtx);
         if (thread->_joined.load(std::memory_order_acquire)) return SCE_KERNEL_ERROR_ESRCH;
         if (thread->_detached || thread->_joining) return SCE_KERNEL_ERROR_EINVAL;
         thread->_detached = true;
     }
+    registered.unlock();
 #ifdef _WIN32
     ReleaseThread(thread);
 #else
@@ -1223,6 +1289,11 @@ void APS5_VABI scePthreadYield() {
 
 int APS5_VABI scePthreadCancel(Pthread thread) {
     if (!thread) return SCE_KERNEL_ERROR_ESRCH;
+    GuestThreadReference reference;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
+    if (!reference.Retain(thread)) return SCE_KERNEL_ERROR_ESRCH;
+    registered.unlock();
     thread->cancelPending.store(true);
     for (;;) {
         {
@@ -1272,12 +1343,16 @@ KernelCpumask APS5_VABI sceKernelGetAvailableCpumask(void) {
 
 int APS5_VABI scePthreadGetaffinity(Pthread thread, KernelCpumask* mask) {
     if (!thread || !mask) return SCE_KERNEL_ERROR_EINVAL;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
     *mask = thread->affinity.load(std::memory_order_relaxed);
     return SCE_OK;
 }
 
 int APS5_VABI scePthreadGetname(Pthread thread, char* name) {
     if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
     std::lock_guard lock(thread->nameLock);
     const std::size_t length = std::min<std::size_t>(thread->name.size(), THREAD_NAME_CAPACITY - 1);
     std::memcpy(name, thread->name.data(), length);
@@ -1287,6 +1362,8 @@ int APS5_VABI scePthreadGetname(Pthread thread, char* name) {
 
 int APS5_VABI scePthreadGetprio(Pthread thread, int* prio) {
     if (!thread || !prio) return SCE_KERNEL_ERROR_EINVAL;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
     *prio = thread->priority.load(std::memory_order_relaxed);
     return SCE_OK;
 }
@@ -1297,6 +1374,8 @@ int APS5_VABI scePthreadGetthreadid(void) {
 
 int APS5_VABI scePthreadRename(Pthread thread, const char* name) {
     if (!thread || !name) return SCE_KERNEL_ERROR_EINVAL;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
     std::lock_guard lock(thread->nameLock);
     thread->name = name;
 #ifdef _WIN32
@@ -1309,6 +1388,8 @@ int APS5_VABI scePthreadRename(Pthread thread, const char* name) {
 
 int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask) {
     if (!thread || mask == 0) return SCE_KERNEL_ERROR_EINVAL;
+    auto registered = LockRegisteredGuestThread(thread);
+    if (!registered) return SCE_KERNEL_ERROR_ESRCH;
     thread->affinity.store(mask, std::memory_order_relaxed);
     return SCE_OK;
 }

core/libs/tests/GuestPthreadDetachAttr.cpp

@@ -0,0 +1,57 @@
+#include "prx/libkernel/include/Api.hpp"
+#include "prx/libkernel/Pthread/include/Pthread.hpp"
+#include "prx/libkernel/KernelErrors.hpp"
+#include <atomic>
+#include <chrono>
+#include <cstdlib>
+#include <thread>
+
+static constexpr int SCE_OK = 0;
+static constexpr int ROUNDS = 50;
+static constexpr int DETACHED = 1;
+
+static void Require(bool condition) { if (!condition) std::abort(); }
+
+struct Worker {
+    std::atomic<bool> release{false};
+    std::atomic<bool> finished{false};
+};
+
+static void* APS5_VABI Wait(void* opaque) {
+    auto* worker = static_cast<Worker*>(opaque);
+    while (!worker->release.load()) std::this_thread::yield();
+    worker->finished.store(true);
+    return nullptr;
+}
+
+int main() {
+    for (int round = 0; round < ROUNDS; ++round) {
+        Worker worker;
+        Pthread thread = nullptr;
+        Require(scePthreadCreate(&thread, nullptr, Wait, &worker, "detach_attr") == SCE_OK);
+        std::atomic<bool> observing{true};
+        std::atomic<bool> started{false};
+        std::thread observer([&] {
+            PthreadAttr attr = nullptr;
+            Require(scePthreadAttrInit(&attr) == SCE_OK);
+            while (observing.load()) {
+                Require(scePthreadAttrGet(thread, &attr) == SCE_OK);
+                started.store(true);
+            }
+            Require(scePthreadAttrDestroy(&attr) == SCE_OK);
+        });
+        while (!started.load()) std::this_thread::yield();
+        Require(scePthreadDetach(thread) == SCE_OK);
+        PthreadAttr attr = nullptr;
+        Require(scePthreadAttrInit(&attr) == SCE_OK);
+        Require(scePthreadAttrGet(thread, &attr) == SCE_OK);
+        int state = -1;
+        Require(scePthreadAttrGetdetachstate(&attr, &state) == SCE_OK && state == DETACHED);
+        Require(scePthreadAttrDestroy(&attr) == SCE_OK);
+        observing.store(false);
+        observer.join();
+        worker.release.store(true);
+        while (!worker.finished.load()) std::this_thread::yield();
+    }
+    return 0;
+}

core/libs/tests/GuestPthreadFinalReleaseRace.cpp

@@ -0,0 +1,104 @@
+#include "prx/libkernel/include/Api.hpp"
+#include "prx/libkernel/Pthread/include/Pthread.hpp"
+#include "prx/libkernel/KernelErrors.hpp"
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdlib>
+#include <mutex>
+#include <thread>
+
+static constexpr int SCE_OK = 0;
+
+extern "C" void APS5_Test_SetFinalThreadReleaseHook(Pthread thread, void (*hook)(Pthread));
+
+static void Require(bool condition) { if (!condition) std::abort(); }
+
+struct Worker {
+    std::atomic<bool> started{false};
+    std::atomic<bool> release{false};
+};
+
+static void* APS5_VABI Wait(void* opaque) {
+    auto* worker = static_cast<Worker*>(opaque);
+    worker->started.store(true);
+    while (!worker->release.load()) std::this_thread::yield();
+    return nullptr;
+}
+
+struct FinalReleaseGate {
+    std::mutex mutex;
+    std::condition_variable changed;
+    Pthread thread = nullptr;
+    bool reached = false;
+    bool release = false;
+};
+
+static FinalReleaseGate gate;
+
+static void HoldFinalRelease(Pthread thread) {
+    std::unique_lock lock(gate.mutex);
+    Require(gate.thread == thread);
+    gate.reached = true;
+    gate.changed.notify_all();
+    gate.changed.wait(lock, [] { return gate.release; });
+}
+
+static void WaitForFinalRelease() {
+    std::unique_lock lock(gate.mutex);
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+    Require(gate.changed.wait_until(lock, deadline, [] { return gate.reached; }));
+}
+
+static void ResumeFinalRelease() {
+    {
+        std::lock_guard lock(gate.mutex);
+        gate.release = true;
+    }
+    gate.changed.notify_all();
+}
+
+static void WaitUntilReleased(Pthread thread) {
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
+    int priority = 0;
+    while (scePthreadGetprio(thread, &priority) != SCE_KERNEL_ERROR_ESRCH) {
+        Require(std::chrono::steady_clock::now() < deadline);
+        std::this_thread::yield();
+    }
+}
+
+static void RequireAdmissionRejectsFinalRelease(int operation) {
+    Worker worker;
+    Pthread thread = nullptr;
+    Require(scePthreadCreate(&thread, nullptr, Wait, &worker, "release_race") == SCE_OK);
+    while (!worker.started.load()) std::this_thread::yield();
+    Require(scePthreadDetach(thread) == SCE_OK);
+
+    {
+        std::lock_guard lock(gate.mutex);
+        gate.thread = thread;
+        gate.reached = false;
+        gate.release = false;
+    }
+    APS5_Test_SetFinalThreadReleaseHook(thread, HoldFinalRelease);
+    worker.release.store(true);
+    WaitForFinalRelease();
+
+    int result = SCE_OK;
+    if (operation == 0) result = scePthreadJoin(thread, nullptr);
+    else if (operation == 1) result = scePthreadDetach(thread);
+    else result = scePthreadCancel(thread);
+
+    ResumeFinalRelease();
+    WaitUntilReleased(thread);
+    Require(result == SCE_KERNEL_ERROR_ESRCH);
+}
+
+int main() {
+    for (int round = 0; round < 20; ++round) {
+        RequireAdmissionRejectsFinalRelease(0);
+        RequireAdmissionRejectsFinalRelease(1);
+        RequireAdmissionRejectsFinalRelease(2);
+    }
+    return 0;
+}