fix(libc): format guest wide printf arguments on Linux

core/libs/prx/libc/include/WindowsFormatting.hpp

@@ -210,7 +210,8 @@ inline int FormatWindows(char* buffer, size_t size, const char* format, const vo
         } else if ((conversion == 'c' && length == "l") || (conversion == 'C' && length.empty())) {
             std::string utf8;
             AppendUtf8(utf8, static_cast<char16_t>(args.Next<unsigned int>()));
-            output.Value(spec + 's', utf8.c_str());
+            if (utf8[0] == 0) output.Value(spec + 'c', 0);
+            else output.Value(spec + 's', utf8.c_str());
         } else if (conversion == 's' && (length.empty() || length == "h")) {
             const char* value = args.Next<const char*>();
             if (!value) value = "(null)";

core/libs/prx/libc/src/Formatting.cpp

@@ -15,14 +15,29 @@
 #include "prx/libc/include/FileStream.hpp"
 #include "prx/libc/include/ApplicationHeap.hpp"
 
-#ifdef _WIN32
 #include "prx/libc/include/WindowsFormatting.hpp"
+
+#ifdef _WIN32
 #include "prx/libc/include/WindowsScanning.hpp"
 #include "prx/libc/include/WindowsWideFormatting.hpp"
 #endif
 
 namespace {
 
+#ifndef _WIN32
+bool HasGuestWideFormat(const char* format) {
+    if (format == nullptr) return false;
+    while ((format = std::strchr(format, '%')) != nullptr) {
+        ++format;
+        if (*format == '%') { ++format; continue; }
+        while (*format && std::strchr("-+ #0'0123456789.*$", *format)) ++format;
+        if (*format == 'S' || *format == 'C' || (*format == 'l' && (format[1] == 's' || format[1] == 'c'))) return true;
+        if (*format) ++format;
+    }
+    return false;
+}
+#endif
+
 std::string HostLength(const std::string& length, char conversion) {
 #ifdef _WIN32
     if (length == "l" && std::strchr("diouxXn", conversion) != nullptr) return "ll";
@@ -167,14 +182,16 @@ int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaLi
         const int count = LibcDetail::FormatWindows(nullptr, 0, format, args, &text);
         const char* source = text.c_str();
 #else
+        std::string guestText;
+        const bool guestWide = HasGuestWideFormat(format);
         char* text = nullptr;
         std::va_list copy;
         va_copy(copy, *reinterpret_cast<std::va_list*>(args));
-        const int count = ::vasprintf(&text, format, copy);
+        const int count = guestWide ? LibcDetail::FormatWindows(nullptr, 0, format, args, &guestText) : ::vasprintf(&text, format, copy);
         va_end(copy);
         std::unique_ptr<char, decltype(&std::free)> owner(text, std::free);
         if (count < 0) return -1;
-        const char* source = text;
+        const char* source = guestWide ? guestText.c_str() : text;
 #endif
         auto* output = static_cast<char*>(ApplicationHeapAllocate_nid_no_patch(static_cast<size_t>(count) + 1));
         std::memcpy(output, source, static_cast<size_t>(count) + 1);
@@ -194,6 +211,13 @@ int APS5_VABI vfprintf_nid_postfix(FileStream* stream, const char* format, VaLis
     const int result = std::fwrite(buffer.data(), 1, static_cast<size_t>(count), native) ==
         static_cast<size_t>(count) ? count : -1;
 #else
+    if (HasGuestWideFormat(format)) {
+        std::string buffer;
+        const int count = LibcDetail::FormatWindows(nullptr, 0, format, args, &buffer);
+        const int result = std::fwrite(buffer.data(), 1, static_cast<size_t>(count), native) == static_cast<size_t>(count) ? count : -1;
+        stream->SyncStatus();
+        return result;
+    }
     std::va_list copy;
     va_copy(copy, *reinterpret_cast<std::va_list*>(args));
     const int result = std::vfprintf(native, format, copy);
@@ -276,31 +300,31 @@ int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) {
 int APS5_VABI printf_nid_postfix(const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vprintf(format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::PrintWindows(format, args) : std::vprintf(format, args);
     va_end(args);
     return result;
 }
 
 int APS5_VABI libc_printf_nid_postfix(const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vprintf(format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::PrintWindows(format, args) : std::vprintf(format, args);
     va_end(args);
     return result;
 }
 
 int APS5_VABI snprintf_nid_postfix(char* buffer, size_t size, const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vsnprintf(buffer, size, format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, size, format, args) : std::vsnprintf(buffer, size, format, args);
     va_end(args);
     return result;
 }
 
 int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vsprintf(buffer, format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, SIZE_MAX, format, args) : std::vsprintf(buffer, format, args);
     va_end(args);
     return result;
 }
@@ -363,6 +387,7 @@ int APS5_VABI vprintf_nid_postfix(const char* str, VaList* c) {
 #ifdef _WIN32
     return LibcDetail::PrintWindows(str, c);
 #else
+    if (HasGuestWideFormat(str)) return LibcDetail::PrintWindows(str, c);
     std::va_list copy;
     va_copy(copy, *reinterpret_cast<std::va_list*>(c));
     const int result = std::vprintf(str, copy);
@@ -375,6 +400,7 @@ int APS5_VABI vsprintf_nid_postfix(char* str, const char* format, VaList* args)
 #ifdef _WIN32
     return LibcDetail::FormatWindows(str, SIZE_MAX, format, args);
 #else
+    if (HasGuestWideFormat(format)) return LibcDetail::FormatWindows(str, SIZE_MAX, format, args);
     std::va_list copy;
     va_copy(copy, *reinterpret_cast<std::va_list*>(args));
     const int result = std::vsprintf(str, format, copy);
@@ -387,6 +413,7 @@ int APS5_VABI vsnprintf_nid_postfix(char* str, size_t size, const char* format,
 #ifdef _WIN32
     return LibcDetail::FormatWindows(str, size, format, c);
 #else
+    if (HasGuestWideFormat(format)) return LibcDetail::FormatWindows(str, size, format, c);
     std::va_list copy;
     va_copy(copy, *reinterpret_cast<std::va_list*>(c));
     const int result = std::vsnprintf(str, size, format, copy);
@@ -416,6 +443,7 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) {
 #else
 
 int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* format, VaList* args) {
+    if (HasGuestWideFormat(format)) return LibcDetail::FormatWindows(buffer, size, format, args);
     std::va_list copy;
     va_copy(copy, *reinterpret_cast<std::va_list*>(args));
     const int result = std::vsnprintf(buffer, size, format, copy);
@@ -426,23 +454,23 @@ int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* form
 int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vsnprintf(buffer, size, format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, size, format, args) : std::vsnprintf(buffer, size, format, args);
     va_end(args);
     return result;
 }
 
 int APS5_VABI snprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vsnprintf(buffer, size, format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, size, format, args) : std::vsnprintf(buffer, size, format, args);
     va_end(args);
     return result;
 }
 
 int APS5_VABI printf_s_nid_postfix(const char* format, ...) {
     std::va_list args;
     va_start(args, format);
-    const int result = std::vprintf(format, args);
+    const int result = HasGuestWideFormat(format) ? LibcDetail::PrintWindows(format, args) : std::vprintf(format, args);
     va_end(args);
     return result;
 }

core/libs/tests/Formatting.cpp

@@ -1,4 +1,5 @@
 #include "SceTypes.hpp"
+#include <algorithm>
 #include <cstring>
 #include <stdexcept>
 #include <cstdio>
@@ -39,7 +40,6 @@ static int APS5_VABI PrintList(const char* format, ...) {
     return result;
 }
 
-#ifdef _WIN32
 static bool CheckWidePrecision() {
     const char16_t input[] = u"A\u00e9\u20ac\U0001f600Z";
     const char* expected[] = {"", "A", "A", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9\xe2\x82\xac",
@@ -65,9 +65,23 @@ static bool CheckWidePrecision() {
     Require(snprintf_nid_postfix(output, sizeof(output), "%.2ls", bounded) == 2 && std::strcmp(output, "AB") == 0);
     Require(snprintf_nid_postfix(nullptr, 0, "%.1ls", u"\u00e9") == 0);
     Require(snprintf_nid_postfix(output, sizeof(output), "%.1s", "\xc3\xa9") == 1 && static_cast<unsigned char>(output[0]) == 0xc3 && output[1] == 0);
+    Require(FormatList(output, sizeof(output), "%S:%lc:%C:%d", u"\u00e9", 0x20ac, 0x41, 7) == 10 && std::strcmp(output, "\xc3\xa9:\xe2\x82\xac:A:7") == 0);
+    Require(snprintf_nid_postfix(output, sizeof(output), "%.*S", 3, u"\u00e9\u20ac") == 2 && std::strcmp(output, "\xc3\xa9") == 0);
+    Require(snprintf_nid_postfix(output, sizeof(output), "[%ls]", static_cast<const char16_t*>(nullptr)) == 8 && std::strcmp(output, "[(null)]") == 0);
+    Require(snprintf_nid_postfix(output, sizeof(output), "a%lcb%C", 0, 0) == 4 && std::memcmp(output, "a\0b\0", 5) == 0);
+#ifndef _WIN32
+    Require(FormatList(output, sizeof(output), "%%ls:%2$d:%1$d", 3, 7) == 7 && std::strcmp(output, "%ls:7:3") == 0);
+#endif
+    for (std::size_t capacity = 0; capacity <= 6; ++capacity) {
+        std::memset(output, '!', sizeof(output));
+        int count = -1;
+        Require(FormatList(capacity ? output : nullptr, capacity, "%ls%n", u"A\u00e9Z", &count) == 4 && count == 4);
+        const auto copied = capacity == 0 ? 0 : std::min<std::size_t>(capacity - 1, 4);
+        Require(std::memcmp(output, "A\xc3\xa9Z", copied) == 0 && output[capacity] == '!');
+        if (capacity) Require(output[copied] == 0);
+    }
     return correct;
 }
-#endif
 
 __attribute__((noinline)) static void APS5_VABI RunChecks() {
     char buffer[1024];
@@ -126,8 +140,4 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() {
     std::puts("Formatting checks passed: 10000 iterations");
 }
 
-#ifdef _WIN32
 int main() { RunChecks(); return CheckWidePrecision() ? 0 : 1; }
-#else
-int main() { RunChecks(); }
-#endif

docs/dev/TechnicalDebt.md

@@ -327,7 +327,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [Min/max S#](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver, shader recompiler) - throws with anisotropy, a linear mip filter, depth or color-texture compare, or bilinear filtering of an image the recompiler point-samples (sint, converted, depth-bits).
 - [Raw `buffer_load_dwordx2`](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`.
 - [wcsrtombs_s](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached
-- [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) passes `%ls` and `%S` to glibc on Linux, which reads them as strings of 32-bit `wchar_t`; the guest's `wchar_t` is 16 bits (UTF-16), so these conversions fail with -1. The Windows path converts them to UTF-8, and `guest_formatting` checks wide conversions on Windows only
+- [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) converts 16-bit guest wide arguments to UTF-8 on both platforms, independently of the host locale. It does not use the guest's single-byte `wcrtomb`; the console's treatment of invalid UTF-16 and locale-specific encoding remains unverified. Linux retains host formatting for formats without wide arguments; positional arguments, the `'` flag and `%m` throw when a format requires the guest formatter
 - [collate<char>::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation.
 - [poll](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) waits only on guest sockets, through `WSAPoll` on Windows and `poll` on Linux; the `pollfd` layout and flag values are FreeBSD's. A descriptor below the socket range (files, pipes, stdio) throws, as do event bits other than `POLLIN`, `POLLPRI`, `POLLOUT`, `POLLRDNORM`, `POLLRDBAND` and `POLLWRBAND` (the `POLLERR`, `POLLHUP` and `POLLNVAL` bits are ignored in `events`), `POLLPRI` on Windows, which `WSAPoll` rejects, and an infinite timeout with no socket to wait on. A closed socket descriptor reports `POLLNVAL`; the console's limit on the descriptor count is unknown and not checked
 - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped