feat(libc): wire the codecvt<wchar_t,char> facet vtable

core/libs/prx/libc/src/LocaleSupport.cpp

@@ -304,6 +304,66 @@ std::size_t APS5_VABI CodecvtLength(GuestLocale::Facet*, std::uint64_t*, const c
 
 const GuestLocale::CodecvtVtable g_codecvtVtable{{DestroyCollate, DeleteCollate, RetainFacet, ReleaseFacet}, CodecvtAlwaysNoconv, CodecvtConstantOne, CodecvtConstantOne, CodecvtNoConversion, CodecvtNoConversion, CodecvtUnshift, CodecvtLength};
 
+constexpr std::size_t GuestMbstateSize = 16;
+
+bool APS5_VABI WideCodecvtAlwaysNoconv(const GuestLocale::Facet*) {
+    return false;
+}
+
+int APS5_VABI WideCodecvtMaxLength(const GuestLocale::Facet*) {
+    return 6;
+}
+
+int APS5_VABI WideCodecvtEncoding(const GuestLocale::Facet*) {
+    return 0;
+}
+
+int APS5_VABI WideCodecvtIn(GuestLocale::Facet*, std::uint64_t* state, const char* first1, const char* last1, const char** next1, char* first3, char* last3, char** next3) {
+    *next1 = first1;
+    *next3 = first3;
+    while (first1 != last1) {
+        if (first3 == last3) return 1;
+        const auto result = mbrtowc_nid_postfix(reinterpret_cast<std::uint16_t*>(first3), first1, static_cast<std::size_t>(last1 - first1), state);
+        if (result == static_cast<std::size_t>(-1)) return 2;
+        if (result == static_cast<std::size_t>(-2)) {
+            *next1 = last1;
+            return 1;
+        }
+        first1 += result == 0 ? 1 : result;
+        first3 += sizeof(std::uint16_t);
+        *next1 = first1;
+        *next3 = first3;
+    }
+    return 0;
+}
+
+int APS5_VABI WideCodecvtOut(GuestLocale::Facet*, std::uint64_t* state, const char* first1, const char* last1, const char** next1, char* first3, char* last3, char** next3) {
+    *next1 = first1;
+    *next3 = first3;
+    while (first1 != last1) {
+        char converted[8];
+        const auto length = wcrtomb_nid_postfix(converted, *reinterpret_cast<const std::uint16_t*>(first1), state);
+        if (length == static_cast<std::size_t>(-1)) return 2;
+        if (length > static_cast<std::size_t>(last3 - first3)) return 1;
+        std::memcpy(first3, converted, length);
+        first1 += sizeof(std::uint16_t);
+        first3 += length;
+        *next1 = first1;
+        *next3 = first3;
+    }
+    return 0;
+}
+
+int APS5_VABI WideCodecvtUnshift(GuestLocale::Facet*, std::uint64_t* state, char* first3, char*, char** next3) {
+    *next3 = first3;
+    std::memset(state, 0, GuestMbstateSize);
+    return 0;
+}
+
+std::size_t APS5_VABI WideCodecvtLength(GuestLocale::Facet*, std::uint64_t*, const char*, const char*, std::size_t) {
+    throw std::runtime_error("codecvt<wchar_t,char>::length: console behavior unverified");
+}
+
 char APS5_VABI NumpunctDecimalPoint(const GuestLocale::NumpunctFacet* self) {
     return self->decimalPoint;
 }
@@ -671,7 +731,21 @@ std::uintptr_t _ZTVSt7collateIcE_nid_postfix[9] {
 };
 std::uint64_t _ZNSt7codecvtIcc9_MbstatetE2idE_nid_postfix = 0;
 std::uintptr_t _ZTVSt7codecvtIcc9_MbstatetE_nid_postfix[16] {};
-std::uintptr_t _ZTVSt7codecvtIwc9_MbstatetE_nid_postfix[13] {};
+std::uintptr_t _ZTVSt7codecvtIwc9_MbstatetE_nid_postfix[13] {
+    0,
+    0,
+    reinterpret_cast<std::uintptr_t>(&DestroyCollate),
+    reinterpret_cast<std::uintptr_t>(&DeleteCollate),
+    reinterpret_cast<std::uintptr_t>(&RetainFacet),
+    reinterpret_cast<std::uintptr_t>(&ReleaseFacet),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtAlwaysNoconv),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtMaxLength),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtEncoding),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtIn),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtOut),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtUnshift),
+    reinterpret_cast<std::uintptr_t>(&WideCodecvtLength),
+};
 std::uint64_t _ZNSt7num_putIcSt19ostreambuf_iteratorIcSt11char_traitsIcEEE2idE_nid_postfix = 0;
 std::uintptr_t _ZTVSt7num_putIcSt19ostreambuf_iteratorIcSt11char_traitsIcEEE_nid_postfix[kNumPutCharVtableSlots] {};
 std::uint64_t _ZNSt7num_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE2idE_nid_postfix = 0;
@@ -857,18 +931,16 @@ void APS5_VABI _ZNSt8__sce_v28_Locinfo13_Locinfo_dtorEPS0__nid_postfix(GuestLoca
 
 int APS5_VABI _Mbtowcx_nid_postfix(std::uint16_t* dst, const char* src, std::size_t count, mbstate_t* st) {
     if (dst == nullptr || src == nullptr || st == nullptr || count == 0) throw std::invalid_argument("_Mbtowcx: invalid conversion arguments");
-    wchar_t converted{};
-    const auto result = std::mbrtowc(&converted, src, count, st);
+    const auto result = mbrtowc_nid_postfix(dst, src, count, st);
     if (result == static_cast<std::size_t>(-1)) throw std::runtime_error("_Mbtowcx: invalid multibyte character");
     if (result == static_cast<std::size_t>(-2)) throw std::runtime_error("_Mbtowcx: incomplete multibyte character");
-    if (result > static_cast<std::size_t>(std::numeric_limits<int>::max()) || static_cast<std::uint32_t>(converted) > 0xffff) throw std::runtime_error("_Mbtowcx: conversion exceeds guest character limits");
-    *dst = static_cast<std::uint16_t>(converted);
+    if (result > static_cast<std::size_t>(std::numeric_limits<int>::max())) throw std::runtime_error("_Mbtowcx: conversion size exceeds guest limits");
     return static_cast<int>(result);
 }
 
 int APS5_VABI _Wctombx_nid_postfix(char* dst, std::uint16_t src, mbstate_t* st) {
     if (dst == nullptr || st == nullptr) throw std::invalid_argument("_Wctombx: invalid conversion arguments");
-    const auto result = std::wcrtomb(dst, static_cast<wchar_t>(src), st);
+    const auto result = wcrtomb_nid_postfix(dst, src, st);
     if (result == static_cast<std::size_t>(-1)) throw std::runtime_error("_Wctombx: invalid wide character");
     if (result > static_cast<std::size_t>(std::numeric_limits<int>::max())) throw std::runtime_error("_Wctombx: conversion size exceeds guest limits");
     return static_cast<int>(result);
@@ -887,13 +959,13 @@ const short* APS5_VABI _Getptoupper_nid_postfix() {
 }
 
 mbstate_t* APS5_VABI _Getpmbstate_nid_postfix() {
-    thread_local mbstate_t state {};
-    return &state;
+    alignas(mbstate_t) thread_local unsigned char state[GuestMbstateSize] {};
+    return reinterpret_cast<mbstate_t*>(state);
 }
 
 mbstate_t* APS5_VABI _Getpwcstate_nid_postfix() {
-    thread_local mbstate_t state {};
-    return &state;
+    alignas(mbstate_t) thread_local unsigned char state[GuestMbstateSize] {};
+    return reinterpret_cast<mbstate_t*>(state);
 }
 
 GuestLocale::TimeInfo* APS5_VABI _Getptimes_nid_postfix() {

core/libs/tests/GuestLocale.cpp

@@ -71,12 +71,60 @@ static void CheckInputFacetObjects() {
 }
 
 static void CheckCodecvtWideVtableObject() {
-    for (std::uintptr_t& slot : _ZTVSt7codecvtIwc9_MbstatetE_nid_postfix) {
-        Require(slot == 0);
-        slot = 0x1111;
-        Require(slot == 0x1111);
-        slot = 0;
-    }
+    Require(_ZTVSt7codecvtIwc9_MbstatetE_nid_postfix[0] == 0 && _ZTVSt7codecvtIwc9_MbstatetE_nid_postfix[1] == 0);
+    for (std::size_t index = 2; index < std::size(_ZTVSt7codecvtIwc9_MbstatetE_nid_postfix); ++index) Require(_ZTVSt7codecvtIwc9_MbstatetE_nid_postfix[index] != 0);
+    const auto* vtable = reinterpret_cast<const GuestLocale::CodecvtVtable*>(&_ZTVSt7codecvtIwc9_MbstatetE_nid_postfix[2]);
+    GuestLocale::Facet facet{};
+    facet.references = 0;
+    vtable->facet.retain(&facet);
+    Require(facet.references == 1);
+    Require(vtable->facet.release(&facet) == &facet && facet.references == 0);
+    Require(!vtable->alwaysNoconv(&facet) && vtable->maxLength(&facet) == 6 && vtable->encoding(&facet) == 0);
+
+    std::uint64_t state[2]{};
+    const char source[] = "A\xc3\xa9\xe2\x82\xac";
+    std::uint16_t wide[4]{};
+    const char* nextSource = nullptr;
+    char* nextWide = nullptr;
+    Require(vtable->in(&facet, state, source, source + 6, &nextSource, reinterpret_cast<char*>(wide), reinterpret_cast<char*>(wide + 4), &nextWide) == 0);
+    Require(nextSource == source + 6 && nextWide == reinterpret_cast<char*>(wide + 3));
+    Require(wide[0] == 'A' && wide[1] == 0xe9 && wide[2] == 0x20ac);
+    RequireException([&] { vtable->length(&facet, state, source, source + 6, 2); });
+
+    std::uint64_t outputState[2]{};
+    char bytes[8]{};
+    const std::uint16_t wideSource[] = {'A', 0xe9, 0x20ac};
+    const char* nextWideSource = nullptr;
+    char* nextByte = nullptr;
+    Require(vtable->out(&facet, outputState, reinterpret_cast<const char*>(wideSource), reinterpret_cast<const char*>(wideSource + 3), &nextWideSource, bytes, bytes + 8, &nextByte) == 0);
+    Require(nextWideSource == reinterpret_cast<const char*>(wideSource + 3) && nextByte == bytes + 6);
+    Require(std::memcmp(bytes, "A\xc3\xa9\xe2\x82\xac", 6) == 0);
+
+    char tiny[2]{};
+    Require(vtable->out(&facet, outputState, reinterpret_cast<const char*>(wideSource + 2), reinterpret_cast<const char*>(wideSource + 3), &nextWideSource, tiny, tiny + 2, &nextByte) == 1);
+    Require(nextWideSource == reinterpret_cast<const char*>(wideSource + 2) && nextByte == tiny);
+
+    const char invalid[] = "\xf0\x9f\x98\x80";
+    Require(vtable->in(&facet, state, invalid, invalid + 4, &nextSource, reinterpret_cast<char*>(wide), reinterpret_cast<char*>(wide + 4), &nextWide) == 2);
+    Require(nextSource == invalid && nextWide == reinterpret_cast<char*>(wide));
+
+    std::uint64_t partialState[2]{};
+    const char partial[] = "\xe2";
+    const char continuation[] = "\x82\xac";
+    std::uint16_t partialWide[1]{};
+    Require(vtable->in(&facet, partialState, partial, partial + 1, &nextSource, reinterpret_cast<char*>(partialWide), reinterpret_cast<char*>(partialWide + 1), &nextWide) == 1);
+    Require(nextSource == partial + 1 && nextWide == reinterpret_cast<char*>(partialWide));
+    Require(vtable->in(&facet, partialState, continuation, continuation + 2, &nextSource, reinterpret_cast<char*>(partialWide), reinterpret_cast<char*>(partialWide + 1), &nextWide) == 0);
+    Require(nextSource == continuation + 2 && nextWide == reinterpret_cast<char*>(partialWide + 1) && partialWide[0] == 0x20ac);
+
+    std::uint64_t fullState[2]{};
+    std::uint16_t one[1]{};
+    Require(vtable->in(&facet, fullState, source, source + 6, &nextSource, reinterpret_cast<char*>(one), reinterpret_cast<char*>(one + 1), &nextWide) == 1);
+    Require(nextSource == source + 1 && nextWide == reinterpret_cast<char*>(one + 1) && one[0] == 'A');
+
+    std::memset(state, 0xff, sizeof(state));
+    Require(vtable->unshift(&facet, state, bytes, bytes + 8, &nextByte) == 0 && nextByte == bytes);
+    Require(state[0] == 0 && state[1] == 0);
 }
 
 static void CheckCollateVtableObject() {
@@ -170,24 +218,44 @@ static void CheckCharacterConversions() {
     for (std::uint16_t value = 0; value < 128; ++value) {
         std::array<char, MB_LEN_MAX + 1> bytes{};
         bytes.fill('!');
-        std::mbstate_t encodeState{};
-        Require(_Wctombx_nid_postfix(bytes.data(), value, &encodeState) == 1);
+        alignas(mbstate_t) unsigned char encodeState[16]{};
+        Require(_Wctombx_nid_postfix(bytes.data(), value, reinterpret_cast<mbstate_t*>(encodeState)) == 1);
         Require(bytes[0] == static_cast<char>(value) && bytes[1] == '!');
         std::array<std::uint16_t, 2> wide{0xffff, 0x1234};
-        std::mbstate_t decodeState{};
-        Require(_Mbtowcx_nid_postfix(wide.data(), bytes.data(), 2, &decodeState) == (value == 0 ? 0 : 1));
+        alignas(mbstate_t) unsigned char decodeState[16]{};
+        Require(_Mbtowcx_nid_postfix(wide.data(), bytes.data(), 2, reinterpret_cast<mbstate_t*>(decodeState)) == (value == 0 ? 0 : 1));
         Require(wide[0] == value && wide[1] == 0x1234);
         const bool alnum = (value >= '0' && value <= '9') || (value >= 'A' && value <= 'Z') || (value >= 'a' && value <= 'z');
         Require(((classification[static_cast<unsigned char>(bytes[0])] & 0x232) != 0) == alnum);
     }
-    std::mbstate_t state{};
+    alignas(mbstate_t) unsigned char state[16]{};
+    char bytes[8]{};
+    std::uint16_t wide = 0;
+    Require(_Wctombx_nid_postfix(bytes, 0xe9, reinterpret_cast<mbstate_t*>(state)) == 2 && std::memcmp(bytes, "\xc3\xa9", 2) == 0);
+    Require(_Wctombx_nid_postfix(bytes, 0x20ac, reinterpret_cast<mbstate_t*>(state)) == 3 && std::memcmp(bytes, "\xe2\x82\xac", 3) == 0);
+    Require(_Wctombx_nid_postfix(bytes, 0xd800, reinterpret_cast<mbstate_t*>(state)) == 3 && std::memcmp(bytes, "\xed\xa0\x80", 3) == 0);
+    Require(_Wctombx_nid_postfix(bytes, 0xffff, reinterpret_cast<mbstate_t*>(state)) == 3 && std::memcmp(bytes, "\xef\xbf\xbf", 3) == 0);
+    std::memset(state, 0, sizeof(state));
+    Require(_Mbtowcx_nid_postfix(&wide, "\xc3\xa9", 2, reinterpret_cast<mbstate_t*>(state)) == 2 && wide == 0xe9);
+    std::memset(state, 0, sizeof(state));
+    Require(_Mbtowcx_nid_postfix(&wide, "\xe2\x82\xac", 3, reinterpret_cast<mbstate_t*>(state)) == 3 && wide == 0x20ac);
+    std::memset(state, 0, sizeof(state));
+    RequireException([&] { _Mbtowcx_nid_postfix(&wide, "\xe2", 1, reinterpret_cast<mbstate_t*>(state)); });
+    Require(_Mbtowcx_nid_postfix(&wide, "\x82\xac", 2, reinterpret_cast<mbstate_t*>(state)) == 2 && wide == 0x20ac);
+    auto* threadState = reinterpret_cast<unsigned char*>(_Getpmbstate_nid_postfix());
+    std::memset(threadState, 0, 16);
+    RequireException([&] { _Mbtowcx_nid_postfix(&wide, "\xe2", 1, reinterpret_cast<mbstate_t*>(threadState)); });
+    Require(_Mbtowcx_nid_postfix(&wide, "\x82\xac", 2, reinterpret_cast<mbstate_t*>(threadState)) == 2 && wide == 0x20ac);
+    std::memset(state, 0, sizeof(state));
+    RequireException([&] { _Mbtowcx_nid_postfix(&wide, "\xf0\x9f\x98\x80", 4, reinterpret_cast<mbstate_t*>(state)); });
+    std::memset(state, 0, sizeof(state));
+    RequireException([&] { _Mbtowcx_nid_postfix(&wide, "\xc3\x28", 2, reinterpret_cast<mbstate_t*>(state)); });
     char byte{};
-    std::uint16_t wide{};
-    RequireException([&] { _Mbtowcx_nid_postfix(&wide, "]", 0, &state); });
-    RequireException([&] { _Mbtowcx_nid_postfix(nullptr, "]", 1, &state); });
-    RequireException([&] { _Mbtowcx_nid_postfix(&wide, nullptr, 1, &state); });
+    RequireException([&] { _Mbtowcx_nid_postfix(&wide, "]", 0, reinterpret_cast<mbstate_t*>(state)); });
+    RequireException([&] { _Mbtowcx_nid_postfix(nullptr, "]", 1, reinterpret_cast<mbstate_t*>(state)); });
+    RequireException([&] { _Mbtowcx_nid_postfix(&wide, nullptr, 1, reinterpret_cast<mbstate_t*>(state)); });
     RequireException([&] { _Mbtowcx_nid_postfix(&wide, "]", 1, nullptr); });
-    RequireException([&] { _Wctombx_nid_postfix(nullptr, ']', &state); });
+    RequireException([&] { _Wctombx_nid_postfix(nullptr, ']', reinterpret_cast<mbstate_t*>(state)); });
     RequireException([&] { _Wctombx_nid_postfix(&byte, ']', nullptr); });
 }
 

docs/dev/debt/libc.md

@@ -254,7 +254,7 @@
 - [exp2f](../../../core/libs/prx/libc/src/Math.cpp) (libc) is FreeBSD msun's [`s_exp2f.c`](https://github.com/freebsd/freebsd-src/blob/release/11.4.0/lib/msun/src/s_exp2f.c), with `huge` and `twom100` read at run time and `x + redux` rounded to float, as msun's `volatile` reads and `STRICT_ASSIGN` do. As in the `libc.prx` that titles ship, `errno` is set to `ERANGE` only in msun's overflow (`x >= 128`) and underflow (`x <= -150`) branches, so a subnormal result leaves it unchanged. Floating-point exceptions are not compared. It gives the same bits and `errno` as the `exp2f` of the `libc.prx` bundled with PPSA07923, PPSA05916 and PPSA13934 for all 2^32 arguments, run on a Windows host.
 - [app0.zar](../../../core/libs/prx/libc/src/AppArchive.cpp) (libc) - a [ZArchive](https://github.com/Exzap/ZArchive) beside `app0/` is a read-only layer under it: a path missing from `app0/` is looked up in the archive, case-insensitively. It is not a console format; titles ship their files in a PFS image instead. Resolving an archive-only path through any call other than the archive-aware libkernel ones throws, so `fopen`, `stat`, `access`, `opendir` and the rest do not read the archive yet, and `opendir` of a directory that exists in both places, `app0/` itself included, and `fdopen` of an archived descriptor throw instead of missing its entries or reading the null device. A damaged compressed block throws instead of returning `EIO`
 - [fopen_s](../../../core/libs/prx/libc/src/FileAndHeap.cpp) (libc) - a null `FILE **` calls the constraint handler with "fopen_s: bad FILE ** argument", and a null filename or mode with "fopen_s: bad argument", each with `EINVAL` and a null pointer, as the `libc.prx` PPSA07923 and PPSA05916 ship does. A failed open returns the errno of the failure, as C11 K.3.5.2.1 leaves the nonzero value open and Microsoft's CRT returns errno; the titles' paths that open a file are not compared
-- [codecvt<wchar_t,char,_Mbstatet> vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `H-TDszhsYuY`) is thirteen zeroed `uintptr_t` slots matching `st_size` 104 on title `libc.prx` from PPSA26344 and on `libSceLibcInternal.sprx` (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and seven conversion virtuals). Facet calls through the object fault until those methods are wired.
+- [codecvt<wchar_t,char,_Mbstatet> vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `H-TDszhsYuY`) is thirteen `uintptr_t` slots matching `st_size` 104 on title `libc.prx` from PPSA26344 and `libSceLibcInternal.sprx` (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, seven conversion virtuals). Slots 2 through 11 are wired: `always_noconv` false, `max_length` 6, `encoding` 0, `in`/`out` convert one-to-three-byte UTF-8 through guest `mbrtowc`/`wcrtomb`, and `unshift` clears the state. The `length` slot throws: no console disassembly established the module's return value or state effect, where standard `do_length` consumes input bytes. The layout and the `in`/`out` delegation to `_Mbtowcx`/`_Wctombx` come from PPSA03446's `libc.prx`, which rejects four-byte UTF-8 leads; those helpers now use the guest UTF-8 code instead of the host C locale. Offset-to-top and typeinfo remain zero, unlike the console's relocated entries.
 - [num_put<wchar_t> vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `59oywaaZbJk`) is fourteen zeroed `uintptr_t` slots matching `st_size` 112 on the title `libc.prx` from PPSA16833 (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and eight `do_put` virtuals). Facet calls through the object fault until those methods are wired.
 - [time_get<char> vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `o4kt51-uO48`) is thirteen zeroed `uintptr_t` slots matching `st_size` 104 on console `libSceLibcInternal.sprx` (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and seven `do_*` virtuals). Facet calls through the object fault until those methods are wired.
 - [time_get<wchar_t> vtable](../../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc, NID `sEca1nUOueA`) is thirteen zeroed `uintptr_t` slots matching `st_size` 104 on console `libSceLibcInternal.sprx` (offset-to-top, typeinfo, two destructors, `_Incref`/`_Decref`, and seven `do_*` virtuals). Facet calls through the object fault until those methods are wired.