fix(libc): keep buffered input when a seek target is rejected

core/libs/prx/libc/src/FileAndHeap.cpp

@@ -15,6 +15,7 @@
 #include <utility>
 #include <cerrno>
 #include <cstring>
+#include <sys/stat.h>
 #ifdef _WIN32
 #ifndef NOMINMAX
 #define NOMINMAX
@@ -753,12 +754,59 @@ size_t APS5_VABI fwrite_nid_postfix(const void* buffer, size_t size, size_t coun
     return result;
 }
 
+static bool WritingStream(std::FILE* native) {
+#ifdef _WIN32
+    constexpr long UcrtWriting = 0x0002;
+    std::atomic_ref<long> state(reinterpret_cast<UcrtStreamState*>(native)->flags);
+    return (state.load() & UcrtWriting) != 0;
+#elif defined(__GLIBC__)
+    return ::__fwriting(native) != 0;
+#else
+    return true;
+#endif
+}
+
+static std::optional<std::int64_t> RegularFileSize(std::FILE* native) {
+#ifdef _WIN32
+    struct _stat64 status;
+    if (_fstat64(_fileno(native), &status) != 0 || (status.st_mode & _S_IFMT) != _S_IFREG) return std::nullopt;
+#else
+    struct stat status;
+    if (::fstat(::fileno(native), &status) != 0 || !S_ISREG(status.st_mode)) return std::nullopt;
+#endif
+    if (status.st_size < 0) return std::nullopt;
+    return status.st_size;
+}
+
+static bool ResolveSeekTarget(FileStream* stream, std::int64_t& offset, int& origin) {
+    std::int64_t base = 0;
+    if (origin == SEEK_CUR) {
+        base = ftello_nid_postfix(stream);
+        if (base == -1) return false;
+        if (base < 0) { errno = 29; return false; }
+    } else if (origin == SEEK_END) {
+        auto* native = GetNativeStream(stream);
+        if (WritingStream(native)) return true;
+        const auto size = RegularFileSize(native);
+        if (!size) return true;
+        base = *size;
+    }
+    if (offset > 0 && base > std::numeric_limits<std::int64_t>::max() - offset) { errno = 84; return false; }
+    offset += base;
+    if (offset < 0) { errno = 22; return false; }
+    origin = SEEK_SET;
+    return true;
+}
+
 int APS5_VABI fseeko_nid_postfix(FileStream* stream, std::int64_t offset, int origin) {
     if (stream && !stream->IsOpen()) { errno = 9; return -1; }
     if (origin != SEEK_SET && origin != SEEK_CUR && origin != SEEK_END) { errno = 22; return -1; }
 #ifdef _WIN32
     auto* native = GetNativeStream(stream);
     if (IsPipeStream(native)) { errno = 29; return -1; }
+#endif
+    if (!ResolveSeekTarget(stream, offset, origin)) return -1;
+#ifdef _WIN32
     const int result = _fseeki64(native, offset, origin);
 #else
     static_assert(sizeof(off_t) == 8);

core/libs/tests/FilePosition.cpp

@@ -1,11 +1,16 @@
 #include "prx/libc/include/Api.hpp"
 #include "prx/libc/include/FileStream.hpp"
+#include "prx/libc/include/General.hpp"
 #include "prx/libc/include/general/VabiMacros.hpp"
 #include "tests/TemporaryFile.hpp"
 #include <cerrno>
+#include <chrono>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
+#include <filesystem>
+#include <limits>
+#include <string>
 #ifdef _WIN32
 #include <fcntl.h>
 #include <io.h>
@@ -36,7 +41,71 @@ static void CheckFseekBounds() {
     file.Close();
 }
 
+static void CheckRejectedBufferedSeeks() {
+    const auto root = std::filesystem::temp_directory_path() / ("anyps5-failed-seek-" +
+        std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
+    Require(std::filesystem::create_directory(root));
+    const auto host = root.string();
+    AddPathAlias_nid_no_patch("/failed-seek", host.c_str());
+    auto* writer = fopen_nid_postfix("/failed-seek/input", "wb");
+    Require(writer != nullptr);
+    Require(fwrite_nid_postfix("12345678", 1, 8, writer) == 8);
+    Require(fclose_nid_postfix(writer) == 0);
+    auto* reader = fopen_nid_postfix("/failed-seek/input", "rb");
+    Require(reader != nullptr);
+    char buffer[4];
+    Require(fread_nid_postfix(buffer, 1, sizeof(buffer), reader) == 4);
+    Require(std::string(buffer, sizeof(buffer)) == "1234");
+    const int endOfFile = feof_nid_postfix(reader);
+    const int error = ferror_nid_postfix(reader);
+    const std::int64_t negative = -1;
+    for (int attempt = 0; attempt < 5; ++attempt) {
+        errno = 0;
+        const int result = attempt == 0 ? fsetpos_nid_postfix(reader, &negative) :
+            attempt == 1 ? fseeko_nid_postfix(reader, -1, SEEK_SET) :
+            attempt == 2 ? fseek_nid_postfix(reader, -5, SEEK_CUR) :
+            attempt == 3 ? fseeko_nid_postfix(reader, -9, SEEK_END) :
+            fseeko_nid_postfix(reader, std::numeric_limits<std::int64_t>::max(), SEEK_CUR);
+        Require(result == -1);
+        Require(errno == (attempt == 4 ? 84 : 22));
+        Require(ftello_nid_postfix(reader) == 4);
+        Require(feof_nid_postfix(reader) == endOfFile);
+        Require(ferror_nid_postfix(reader) == error);
+    }
+    Require(fgetc_nid_postfix(reader) == '5');
+    Require(ungetc_nid_postfix('5', reader) == '5');
+    errno = 0;
+    Require(fseeko_nid_postfix(reader, -5, SEEK_CUR) == -1 && errno == 22);
+    Require(ftello_nid_postfix(reader) == 4);
+    Require(feof_nid_postfix(reader) == endOfFile);
+    Require(ferror_nid_postfix(reader) == error);
+    Require(fgetc_nid_postfix(reader) == '5');
+    Require(ungetc_nid_postfix('X', reader) == 'X');
+    errno = 0;
+    Require(fseeko_nid_postfix(reader, -1, SEEK_SET) == -1 && errno == 22);
+    Require(ftello_nid_postfix(reader) == 4);
+    Require(feof_nid_postfix(reader) == endOfFile);
+    Require(ferror_nid_postfix(reader) == error);
+    Require(fgetc_nid_postfix(reader) == 'X');
+    Require(fgetc_nid_postfix(reader) == '6');
+    Require(fclose_nid_postfix(reader) == 0);
+    RemovePathAlias_nid_no_patch("/failed-seek");
+    Require(std::filesystem::remove(root / "input"));
+    Require(std::filesystem::remove(root));
+}
+
+static void CheckBufferedOutputSeek() {
+    FileStream file(TemporaryFile::Open());
+    Require(fwrite_nid_postfix("12345678", 1, 8, &file) == 8);
+    Require(fseeko_nid_postfix(&file, -3, SEEK_END) == 0);
+    Require(ftello_nid_postfix(&file) == 5);
+    Require(fgetc_nid_postfix(&file) == '6');
+    file.Close();
+}
+
 int main() {
+    CheckRejectedBufferedSeeks();
+    CheckBufferedOutputSeek();
     CheckFseekBounds();
     FileStream file(TemporaryFile::Open());
     Require(std::fputs("position", file.GetHandle()) >= 0);

docs/dev/debt/libc.md

@@ -204,7 +204,7 @@
 - [`fopen` and `freopen` of /dev/random and /dev/urandom](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) in a read mode return a stream over the host null device whose `fread` fills the buffer with host OS entropy, as the libkernel [random device](../../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) descriptors do. Only the exact paths are recognised, the stream has no descriptor, `setvbuf`, `feof`, `ferror`, `clearerr`, `flockfile`, `ftrylockfile`, `funlockfile` and `fclose` work on it, `fflush` returns 0 as on a FreeBSD read-only stream, and every other stream function throws, as does opening the paths for writing. ProsperoTV (PPSA99003) reads its phone pairing code this way
 - [setvbuf and setbuf](../../../core/libs/prx/libc/src/StandardStreams.cpp) (libc) on Windows never hand the guest's buffer to the host C library: UCRT allocates the stream's buffer itself, so later refills, which go through `ReadFile`, cannot fail on write-watched pages of shared guest memory. FreeBSD's `__srefill` reads into the guest's buffer, which on Windows keeps its old contents; a title that reads stream data out of its own `setvbuf` buffer sees a difference. On Linux and macOS the guest's buffer is still handed to the host library; macOS's write watch protects watched pages, so a refill into one is expected to fail there (not tested)
 - [RemoveTree](../../../core/libs/prx/libc/include/RemoveTree.hpp) (libc) removes a host tree itself on Windows instead of calling `std::filesystem::remove_all`: MinGW libstdc++ up to GCC 16 does not treat a reparse point as a link, so `remove_all` recurses through it and never terminates, and any tree that holds a symlink hangs. It builds the entry list from `GetFileAttributesW`, removes a reparse point as the link it is rather than its target, clears the read-only attribute and retries as `remove_all` does, and reports the first native error. The libstdc++ fix is in GCC 17 (commit 88856ad855ae); other hosts call `remove_all`.
-- [fseeko and fsetpos](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) on Windows move a stream whose seek fails: the UCRT drops the read buffer before its `lseek` to a negative target fails with `EINVAL`, so the stream continues at the descriptor's read-ahead offset. After 4 of 8 buffered bytes, a rejected `fsetpos` to -1 leaves `fgetpos` at 8 and the next read at end of file. FreeBSD 11.4's [`_fseeko`](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libc/stdio/fseek.c) keeps the buffer and position when `_sseek` fails. Linux is unverified
+- [fseeko and fsetpos](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) leave a failed `SEEK_END` seek on a stream in write mode, on a nonregular file or on macOS to the host C library, which on Windows drops the read buffer when the seek fails, so the stream continues at the descriptor's read-ahead offset. Other targets are checked first as FreeBSD 11.4's [`_fseeko`](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libc/stdio/fseek.c) does: a negative `SEEK_SET`, `SEEK_CUR` or regular-file `SEEK_END` target is `EINVAL` and one past `OFF_MAX` is `EOVERFLOW`, leaving the buffer, pushback and EOF state alone. Buffered output is not flushed to find the end. A host seek that fails on a valid target is not checked
 - The [`_Unwind_Context`](../../../core/libs/prx/libc/include/exceptions/Unwind.hpp) that `_Unwind_Backtrace` passes to its callback starts with a table laid out like LLVM libunwind's `AbstractUnwindCursor` virtual table, because homebrew built with the ps5-payload-dev SDK links libunwind statically and reads the frames with its own `_Unwind_GetIP` (`__unw_get_reg`), which calls through that table. Only register reads and writes (`validReg`, `getReg`, `setReg`, with `UNW_REG_IP` and `UNW_REG_SP`), `validFloatReg` (always false) and `isSignalFrame` are answered; the other slots throw. The layout follows libunwind's `UnwindCursor.hpp`, not the console's own `libSceLibcInternal`, which is not checked
 - The Windows libc [exception unwinder](../../../core/libs/prx/libc/src/exception/Unwind.cpp) cannot run arbitrary Windows exception or termination handlers. It accepts GCC C and C++ handlers identified by a project's `.sehgcc` section and interprets their LSDA with the guest personalities. A handler in an unmarked module, such as a C++ cleanup through `__gxx_personality_seh0` in `libstdc++-6.dll` or a system DLL function like `qsort` in `ucrtbase.dll` (Windows 10 22H2), still stops unwinding and the exception is reported as unhandled. SjLj exceptions are unsupported; Scoop GCC 15.2.0 selects them when SEH is disabled and lacks the System V ABI SEH compiler fix carried by MSYS2.
 - A guest frame built with `-fno-asynchronous-unwind-tables`, and libsystem_pthread `thread_start`, still fail to unwind on macOS. Compact unwind records them as encoding 0, and the [unwinder](../../../core/libs/prx/libc/src/exception/Unwind.cpp) rejects that encoding instead of inventing a CFA. A DWARF-mode entry whose `__eh_frame` FDE is missing or rejected fails the same way, with no fallback frame. libc itself is built with `-femit-dwarf-unwind=always` and `-Wl,-no_compact_unwind` (#8311), so its own frames stay in DWARF. The unwinder walks `__TEXT,__unwind_info` when `__eh_frame` misses the frame. A personality index in that section is a signed 32-bit delta from the Mach-O image base.