fix(libc): drop the kernel state of a descriptor that fclose or freopen closes

core/libs/prx/libc/include/FileStream.hpp

@@ -5,6 +5,7 @@
 #include <cerrno>
 #include <cstdio>
 #include <filesystem>
+#include <optional>
 #include <stdexcept>
 #include <string>
 #include <string_view>
@@ -34,6 +35,34 @@ extern "C" void ForgetDirectoryStream_nid_no_patch(int descriptor);
 extern "C" int _IO_switch_to_get_mode(std::FILE* stream);
 #endif
 extern "C" int DuplicateHostDescriptor_nid_no_patch(int descriptor);
+extern "C" void LibcRegisterClosedDescriptors_nid_no_patch(void* (*begin)(int descriptor), void (*end)(void* closing));
+extern "C" void* BeginClosingDescriptor_nid_no_patch(int descriptor);
+extern "C" void EndClosingDescriptor_nid_no_patch(void* closing);
+
+class ClosingDescriptor {
+public:
+    ClosingDescriptor([[maybe_unused]] std::FILE* stream, int descriptor)
+#ifdef _WIN32
+        : _stream(stream), _closing(BeginClosingDescriptor_nid_no_patch(descriptor)) {}
+#else
+        : _closing(BeginClosingDescriptor_nid_no_patch(descriptor)) {}
+#endif
+    ~ClosingDescriptor() { EndClosingDescriptor_nid_no_patch(_closing); }
+    ClosingDescriptor(const ClosingDescriptor&) = delete;
+    ClosingDescriptor& operator=(const ClosingDescriptor&) = delete;
+
+private:
+#ifdef _WIN32
+    struct LockedStream {
+        explicit LockedStream(std::FILE* value) : stream(value) { ::_lock_file(stream); }
+        ~LockedStream() { ::_unlock_file(stream); }
+        LockedStream(const LockedStream&) = delete;
+        LockedStream& operator=(const LockedStream&) = delete;
+        std::FILE* stream;
+    } _stream;
+#endif
+    void* _closing;
+};
 
 struct GuestFilePrefix {
     unsigned char* position = nullptr;
@@ -158,10 +187,12 @@ class FileStream {
         if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(replaced);
         auto* const standard = HostStandardHandle_nid_no_patch(replaced);
         GuestFileLockForget_nid_no_patch(replaced);
+        std::optional<ClosingDescriptor> closing(std::in_place, previous, replaced);
         if (open && !filename.empty()) {
             if (!IsHostStandard()) {
                 _handle = nullptr;
                 std::fclose(previous);
+                closing.reset();
                 _handle = open(filename, mode);
             } else {
                 std::wstring resetMode;
@@ -201,6 +232,7 @@ class FileStream {
         KeepStandardHandle_nid_no_patch(replaced, standard);
 #else
         if (::fileno(previous) == 2) KeepHostStandardError_nid_no_patch();
+        const ClosingDescriptor closing(previous, ::fileno(previous));
         const int kept = IsHostStandard() ? DuplicateHostDescriptor_nid_no_patch(::fileno(previous)) : -1;
         if (kept >= 0) ::flockfile(previous);
         _handle = std::freopen(filename.c_str(), mode, previous);
@@ -302,8 +334,12 @@ class FileStream {
 #ifdef _WIN32
         GuestFileLockForget_nid_no_patch(::_fileno(_handle));
         if (std::exchange(directory, false)) ForgetDirectoryStream_nid_no_patch(::_fileno(_handle));
+        const ClosingDescriptor closing(_handle, ::_fileno(_handle));
+#else
+        const ClosingDescriptor closing(_handle, ::fileno(_handle));
 #endif
-        const int result = std::fclose(std::exchange(_handle, nullptr));
+        const int result = std::fclose(_handle);
+        _handle = nullptr;
         _guest.flags = 0;
         _guest.descriptor = -1;
         return result;

core/libs/prx/libc/src/FileAndHeap.cpp

@@ -336,6 +336,22 @@ static int OpenFailure(const std::filesystem::path& path, bool writes, int hostE
     return writes && error == 13 && std::filesystem::is_directory(path, status) ? 21 : error;
 }
 
+static void* (*g_beginClosingDescriptor)(int) = nullptr;
+static void (*g_endClosingDescriptor)(void*) = nullptr;
+
+void LibcRegisterClosedDescriptors_nid_no_patch(void* (*begin)(int), void (*end)(void*)) {
+    g_beginClosingDescriptor = begin;
+    g_endClosingDescriptor = end;
+}
+
+void* BeginClosingDescriptor_nid_no_patch(int descriptor) {
+    return g_beginClosingDescriptor ? g_beginClosingDescriptor(descriptor) : nullptr;
+}
+
+void EndClosingDescriptor_nid_no_patch(void* closing) {
+    if (g_endClosingDescriptor) g_endClosingDescriptor(closing);
+}
+
 #ifdef _WIN32
 static int (*g_openDirectoryDescriptor)(const wchar_t*) = nullptr;
 static void (*g_forgetDirectoryDescriptor)(int) = nullptr;

core/libs/prx/libkernel/CMakeLists.txt

@@ -1975,8 +1975,14 @@ if(BUILD_TESTING)
         set_tests_properties(guest_kernel_fcntl_sync PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
     endif()
-    if(WIN32)
-        set_tests_properties(guest_kernel_fcntl guest_append_writes PROPERTIES ENVIRONMENT_MODIFICATION
+    add_test_executable(guest_stream_close_tests ${LIBS_INCLUDE_DIR}/tests/GuestStreamClose.cpp)
+    target_include_directories(guest_stream_close_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_stream_close_tests PRIVATE libkernel libc)
+    configure_windows_unwind(guest_stream_close_tests)
+    add_test(NAME guest_stream_close COMMAND guest_stream_close_tests)
+    set_tests_properties(guest_stream_close PROPERTIES TIMEOUT 15)
+    if(WIN32)
+        set_tests_properties(guest_kernel_fcntl guest_append_writes guest_stream_close PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel>")
         add_test_executable(guest_file_lock_tests ${LIBS_INCLUDE_DIR}/tests/GuestFileLock.cpp)
         target_include_directories(guest_file_lock_tests PRIVATE ${LIBS_INCLUDE_DIR})

core/libs/prx/libkernel/File/include/File.hpp

@@ -4,6 +4,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <filesystem>
+#include <memory>
 #include <mutex>
 #include <optional>
 
@@ -33,7 +34,19 @@ namespace File {
 std::optional<std::filesystem::path> WindowsDescriptorPath_nid_no_patch(int descriptor);
 #endif
 int HostStandardError();
-[[nodiscard]] std::unique_lock<std::mutex> LockPositionedTransfer(int descriptor);
+void ForgetClosedDescriptor(int descriptor);
+class PositionedTransferLock {
+public:
+    explicit PositionedTransferLock(std::shared_ptr<std::mutex> mutex) : _mutex(std::move(mutex)), _lock(*_mutex) {}
+    PositionedTransferLock(PositionedTransferLock&&) = default;
+    PositionedTransferLock& operator=(PositionedTransferLock&&) = delete;
+    void unlock() { _lock.unlock(); }
+
+private:
+    std::shared_ptr<std::mutex> _mutex;
+    std::unique_lock<std::mutex> _lock;
+};
+[[nodiscard]] PositionedTransferLock LockPositionedTransfer(int descriptor);
 void SharePositionedTransfer(int from, int to);
 #ifdef _WIN32
 std::int64_t WriteAtEnd(int descriptor, const void* buf, std::size_t nbytes);

core/libs/prx/libkernel/File/src/Open.cpp

@@ -7,6 +7,7 @@
 #include "prx/libc/include/RenameDot.hpp"
 #include "prx/libc/include/GuestArena.hpp"
 #include "prx/libc/include/AppArchive.hpp"
+#include "prx/libc/include/FileStream.hpp"
 #include "prx/libkernel/DirectMemory/DirectMemory.hpp"
 #include "prx/libkernel/File/include/File.hpp"
 #include "prx/libkernel/File/include/SharedMemory.hpp"
@@ -24,6 +25,8 @@
 #include <cerrno>
 #include <cstdarg>
 #include <limits>
+#include <memory>
+#include <mutex>
 #include <stdexcept>
 #include <string>
 
@@ -275,6 +278,45 @@ static int OpenNative(const std::filesystem::path& native, int flags, std::uint1
     return fd;
 }
 
+void File::ForgetClosedDescriptor(int descriptor) {
+    File::ForgetRandomDevice(descriptor);
+    File::ForgetDescriptor(descriptor);
+    ForgetJitSharedMemory(descriptor);
+#ifdef _WIN32
+    File::ForgetFileLock(descriptor);
+#endif
+    File::ForgetDirectoryDescriptor(descriptor);
+}
+
+namespace {
+
+void* BeginClosingStreamDescriptor(int descriptor) {
+    EqueueDescriptorClosed(descriptor);
+#ifdef _WIN32
+    auto positioned = std::make_unique<File::PositionedTransferLock>(File::LockPositionedTransfer(descriptor));
+    File::ForgetClosedDescriptor(descriptor);
+    return positioned.release();
+#else
+    File::ForgetClosedDescriptor(descriptor);
+    return nullptr;
+#endif
+}
+
+void EndClosingStreamDescriptor(void* closing) {
+    delete static_cast<File::PositionedTransferLock*>(closing);
+}
+
+struct ClosedDescriptorRegistration {
+    ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(&BeginClosingStreamDescriptor, &EndClosingStreamDescriptor); }
+    ~ClosedDescriptorRegistration() { LibcRegisterClosedDescriptors_nid_no_patch(nullptr, nullptr); }
+    ClosedDescriptorRegistration(const ClosedDescriptorRegistration&) = delete;
+    ClosedDescriptorRegistration& operator=(const ClosedDescriptorRegistration&) = delete;
+};
+
+const ClosedDescriptorRegistration g_closedDescriptorRegistration;
+
+}
+
 extern "C" {
 
 int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) {
@@ -321,13 +363,7 @@ int APS5_VABI sceKernelClose(int d) {
 #ifdef _WIN32
     const auto positioned = File::LockPositionedTransfer(d);
 #endif
-    File::ForgetRandomDevice(d);
-    File::ForgetDescriptor(d);
-    ForgetJitSharedMemory(d);
-#ifdef _WIN32
-    File::ForgetFileLock(d);
-#endif
-    File::ForgetDirectoryDescriptor(d);
+    File::ForgetClosedDescriptor(d);
     if (NativeClose(d) != 0) {
         if (errno == EBADF) return SCE_KERNEL_ERROR_EBADF;
         throw std::runtime_error(std::string(__func__) + ": close failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno));

core/libs/prx/libkernel/File/src/Stdio.cpp

@@ -245,15 +245,15 @@ struct PositionedLock {
 };
 std::mutex g_positionedTableLock;
 std::unordered_map<int, std::shared_ptr<PositionedLock>> g_positionedLocks;
-std::unique_lock<std::mutex> File::LockPositionedTransfer(int descriptor) {
+File::PositionedTransferLock File::LockPositionedTransfer(int descriptor) {
     std::shared_ptr<PositionedLock> entry;
     {
         std::lock_guard table(g_positionedTableLock);
         auto& slot = g_positionedLocks[descriptor];
         if (!slot) slot = std::make_shared<PositionedLock>();
         entry = slot;
     }
-    return std::unique_lock<std::mutex>(entry->mutex);
+    return PositionedTransferLock(std::shared_ptr<std::mutex>(entry, &entry->mutex));
 }
 void File::SharePositionedTransfer(int from, int to) {
     std::lock_guard table(g_positionedTableLock);
@@ -773,12 +773,8 @@ int APS5_VABI close_nid_postfix(int d) {
 #ifdef _WIN32
     const auto positioned = File::LockPositionedTransfer(d);
 #endif
-    File::ForgetRandomDevice(d);
-    File::ForgetDescriptor(d);
-    ForgetJitSharedMemory(d);
-    File::ForgetDirectoryDescriptor(d);
+    File::ForgetClosedDescriptor(d);
 #ifdef _WIN32
-    File::ForgetFileLock(d);
     const int shared = File::CloseSharedMemoryDescriptor_nid_no_patch(d);
     if (shared != -2) return shared;
     return WithoutParameterHandler([d] { return ::_close(d); });

core/libs/tests/GuestKernelAppArchive.cpp

@@ -317,6 +317,21 @@ void Directories() {
     Check(!DescriptorIsOpen_nid_no_patch(owned));
 }
 
+void DirectoryClosedThroughStream(const std::filesystem::path& root) {
+    std::ofstream(root / "app0" / "StreamClose.txt", std::ios::binary) << "abc";
+    const int directory = sceKernelOpen("/app0/sce_sys/Concurrent", SCE_KERNEL_O_RDONLY | SCE_KERNEL_O_DIRECTORY, 0);
+    Check(directory >= 0);
+    auto* stream = fdopen_nid_postfix(directory, "r");
+    Check(stream != nullptr);
+    Check(fclose_nid_postfix(stream) == 0);
+    const int file = sceKernelOpen("/app0/StreamClose.txt", SCE_KERNEL_O_RDONLY, 0);
+    Check(file == directory);
+    char data[4]{};
+    Check(sceKernelRead(file, data, sizeof(data)) == 3);
+    Check(sceKernelClose(file) == 0);
+    Check(std::filesystem::remove(root / "app0" / "StreamClose.txt"));
+}
+
 void Reads() {
     const auto expected = AppArchiveFixture::Pattern(BigSize);
     const int fd = sceKernelOpen("/app0/data/big.BIN", SCE_KERNEL_O_RDONLY, 0);
@@ -496,6 +511,7 @@ int main(int argc, char** argv) {
         Archive(root);
         FileNumbers();
         ConcurrentArchiveDirectoryEntries();
+        DirectoryClosedThroughStream(root);
         Reads();
         Status();
         Refusals();