feat(libc): accept positional arguments with wide scanf fields on Linux

core/libs/prx/libc/include/ScanWideField.hpp

@@ -6,7 +6,7 @@
 namespace LibcDetail {
 
 bool InWideScanset(const std::string& specifier, unsigned char byte);
-void RequireNoPositionalWideScan(const char* format);
+void RejectPositionalScan(const char* what);
 
 }
 

core/libs/prx/libc/src/Formatting.cpp

@@ -10,6 +10,7 @@
 #include <cerrno>
 #include <cstdlib>
 #include <memory>
+#include <vector>
 #include <climits>
 #include <stdexcept>
 
@@ -84,12 +85,6 @@ bool HasGuestWideScan(const char* format) {
     return false;
 }
 
-bool ScansWideInProcess(const char* format) {
-    if (!HasGuestWideScan(format)) return false;
-    LibcDetail::RequireNoPositionalWideScan(format);
-    return true;
-}
-
 const char* CutAtQuadScanDirective(const char* format, std::string& head) {
     const char* const quad = LibcDetail::FindQuadScanDirective(format);
     if (quad == nullptr) return format;
@@ -248,9 +243,45 @@ class StreamSource {
     }
 };
 
+std::size_t NumberedScanArguments(const char* format) {
+    std::size_t highest = 0;
+    std::size_t sequential = 0;
+    bool numbered = false;
+    while ((format = std::strchr(format, '%')) != nullptr) {
+        ++format;
+        if (*format == '%') { ++format; continue; }
+        const std::size_t position = LibcDetail::ArgumentPosition(format);
+        if (position != 0) numbered = true;
+        const bool suppress = *format == '*';
+        while (*format && std::strchr("*0123456789hljztL", *format)) ++format;
+        const char conversion = *format;
+        if (conversion == '\0') break;
+        if (conversion == '[') {
+            ++format;
+            if (*format == '^') ++format;
+            if (*format == ']') ++format;
+            while (*format && *format != ']') ++format;
+            if (*format == '\0') break;
+        }
+        ++format;
+        if (suppress) continue;
+        if (position != 0) highest = std::max(highest, position);
+        else ++sequential;
+    }
+    if (numbered && sequential != 0) LibcDetail::RejectPositionalScan("scanf positional arguments mixed with unnumbered assignments");
+    return numbered ? highest : 0;
+}
+
 template <typename TSource, typename NextPointer, typename NextCapacity>
 int ScanSource(TSource source, const char* format, bool secure, NextPointer nextPointer, NextCapacity nextCapacity) {
     if (format == nullptr) return EOF;
+    std::vector<void*> numbered(NumberedScanArguments(format));
+    if (secure && !numbered.empty()) LibcDetail::RejectPositionalScan("scanf_s positional arguments");
+    for (auto& pointer : numbered) pointer = nextPointer();
+    const auto pointerAt = [&](std::size_t position) {
+        if (numbered.empty()) return nextPointer();
+        return numbered.at(position - 1);
+    };
     int assigned = 0;
     bool converted = false;
     const char* cursor = format;
@@ -270,6 +301,7 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
             continue;
         }
         ++cursor;
+        const std::size_t position = LibcDetail::ArgumentPosition(cursor);
         const bool suppress = *cursor == '*';
         if (suppress) ++cursor;
         unsigned long width = 0;
@@ -299,7 +331,7 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
         ++cursor;
         if (conversion == 'n') {
             if (!suppress) {
-                void* target = nextPointer();
+                void* target = pointerAt(position);
                 if (target == nullptr) return EOF;
                 const auto count = source.Consumed();
                 if (length == "hh") *static_cast<signed char*>(target) = static_cast<signed char>(count);
@@ -319,7 +351,7 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
                 while (std::isspace(source.Peek())) source.Skip();
             }
             if (source.Peek() == EOF) return finish(converted ? assigned : EOF);
-            auto* target = suppress ? nullptr : static_cast<char16_t*>(nextPointer());
+            auto* target = suppress ? nullptr : static_cast<char16_t*>(pointerAt(position));
             unsigned long limit = ULONG_MAX;
             if (secure && !suppress) {
                 const auto capacity = static_cast<unsigned long>(nextCapacity());
@@ -389,7 +421,7 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
         void* target = nullptr;
         if (suppress) directive += '*';
         else {
-            target = nextPointer();
+            target = pointerAt(position);
             if (target == nullptr) return EOF;
         }
         if constexpr (!TSource::Contiguous) if (conversion == 'c' && length != "l") {
@@ -472,7 +504,6 @@ int ScanGuestStream(FileStream* stream, const char* format, NextPointer nextPoin
     auto* native = GetNativeStream(stream);
     stream->Orientation(-1);
     if (!stream->SwitchToReading()) return EOF;
-    if (HasGuestWideScan(format)) LibcDetail::RequireNoPositionalWideScan(format);
     const int result = ScanSource(StreamSource(native), format, true, nextPointer, nextCapacity);
     stream->SyncStatus();
     return result;
@@ -569,14 +600,14 @@ int APS5_VABI fscanf_nid_postfix(FileStream* stream, const char* format, ...) {
 #ifdef _WIN32
     __builtin_sysv_va_list args;
     __builtin_sysv_va_start(args, format);
-    const int result = ScansWideInProcess(format) ? ScanSource(StreamSource(native), format, false, [&] { return __builtin_va_arg(args, void*); }, [] { return 0u; })
-                                                  : LibcDetail::ScanFileWindows_nid_no_patch(native, format, args);
+    const int result = HasGuestWideScan(format) ? ScanSource(StreamSource(native), format, false, [&] { return __builtin_va_arg(args, void*); }, [] { return 0u; })
+                                                : LibcDetail::ScanFileWindows_nid_no_patch(native, format, args);
     __builtin_sysv_va_end(args);
 #else
     std::va_list args;
     va_start(args, format);
-    const int result = ScansWideInProcess(format) ? ScanSource(StreamSource(native), format, false, [&] { return va_arg(args, void*); }, [] { return 0u; })
-                                                  : HostVfscanf(native, format, args);
+    const int result = HasGuestWideScan(format) ? ScanSource(StreamSource(native), format, false, [&] { return va_arg(args, void*); }, [] { return 0u; })
+                                                : HostVfscanf(native, format, args);
     va_end(args);
 #endif
     stream->SyncStatus();
@@ -665,7 +696,7 @@ int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) {
 int APS5_VABI vsscanf_nid_postfix(const char* input, const char* format, VaList* args) {
     std::string head;
     format = CutAtQuadScanDirective(format, head);
-    if (ScansWideInProcess(format)) {
+    if (HasGuestWideScan(format)) {
         LibcDetail::FormatArguments arguments(args);
         return ScanGuest(input, format, false, [&] { return arguments.Next<void*>(); }, [] { return 0u; });
     }
@@ -694,8 +725,8 @@ int APS5_VABI sscanf_nid_postfix(const char* input, const char* format, ...) {
     va_start(args, format);
     std::string head;
     format = CutAtQuadScanDirective(format, head);
-    const int result = ScansWideInProcess(format) ? ScanGuest(input, format, false, [&] { return va_arg(args, void*); }, [] { return 0u; })
-                                                  : HostVsscanf(input, format, args);
+    const int result = HasGuestWideScan(format) ? ScanGuest(input, format, false, [&] { return va_arg(args, void*); }, [] { return 0u; })
+                                                : HostVsscanf(input, format, args);
     va_end(args);
     return result;
 }
@@ -911,7 +942,7 @@ int APS5_VABI vfscanf_nid_postfix(FileStream* stream, const char* format, VaList
     stream->Orientation(-1);
     if (!stream->SwitchToReading()) return EOF;
     int result = 0;
-    if (ScansWideInProcess(format)) {
+    if (HasGuestWideScan(format)) {
         LibcDetail::FormatArguments arguments(args);
         result = ScanSource(StreamSource(native), format, false, [&] { return arguments.Next<void*>(); }, [] { return 0u; });
     } else {

core/libs/prx/libc/src/ScanWideField.cpp

@@ -1,8 +1,6 @@
 #include "prx/libc/include/ScanWideField.hpp"
 #include "prx/libc/include/General.hpp"
 
-#include <cstring>
-
 namespace LibcDetail {
 
 bool InWideScanset(const std::string& specifier, unsigned char byte) {
@@ -24,8 +22,8 @@ bool InWideScanset(const std::string& specifier, unsigned char byte) {
     return member != negated;
 }
 
-void RequireNoPositionalWideScan(const char* format) {
-    if (std::strchr(format, '$') != nullptr) NotImplemented_nid_no_patch("sscanf positional arguments with a 16-bit wide conversion");
+void RejectPositionalScan(const char* what) {
+    NotImplemented_nid_no_patch(what);
 }
 
 }

core/libs/tests/Formatting.cpp

@@ -549,10 +549,22 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() {
         Require(sscanf_nid_postfix("x \x80", "%ls %ls", first, fits) == 1 && std::u16string(first) == u"x");
         Require(sscanf_nid_postfix("\xc3\xa9", "%*ls") == 0 && sscanf_nid_postfix("\x80", "%*ls") == EOF);
         Require(sscanf_nid_postfix("\xc3\xa9", "%*lc") == EOF && sscanf_nid_postfix("a\xc3", "%*ls") == EOF);
+        char16_t second[8] = {};
+        Require(sscanf_nid_postfix("ab cd", "%2$ls %1$ls", first, second) == 2);
+        Require(std::u16string(first) == u"cd" && std::u16string(second) == u"ab");
+        Require(sscanf_nid_postfix("7 xy z", "%3$d %1$ls %2$lc", first, &letter, &number) == 3);
+        Require(number == 7 && std::u16string(first) == u"xy" && letter == u'z');
+        Require(sscanf_nid_postfix("ab cd", "%*ls %2$ls%1$n", &consumed, first) == 1 && consumed == 5 && std::u16string(first) == u"cd");
+        Require(ScanList("h there", "%2$lc %1$ls", first, &letter) == 2 && std::u16string(first) == u"there" && letter == u'h');
+        Require(sscanf_nid_postfix("ab", "%1$ls %2$ls", first, second) == 1 && std::u16string(first) == u"ab");
         char16_t untouched[4] = {u'!', u'!', u'!', u'!'};
-        bool positional = false;
-        try { sscanf_nid_postfix("a b", "%2$ls", untouched); } catch (const std::runtime_error&) { positional = untouched[0] == u'!'; }
-        Require(positional);
+        const auto rejects = [&](auto scan) {
+            try { scan(); } catch (const std::runtime_error&) { return untouched[0] == u'!'; }
+            return false;
+        };
+        Require(rejects([&] { ScanListS("ab", "%1$ls", untouched, 4u); }));
+        Require(rejects([&] { sscanf_nid_postfix("1 ab", "%d %1$ls", untouched); }));
+        Require(rejects([&] { sscanf_nid_postfix("ab 1", "%1$ls %n", untouched); }));
     }
 #ifndef _WIN32
     char narrow[8] = {};

core/libs/tests/GuestWideStreamScan.cpp

@@ -143,6 +143,10 @@ static void CheckNarrowFormatWideFields() {
     Require(ScanNarrowInput(" %lc", &letter) == 1 && letter == u'c', "vscanf %lc reads the guest standard input");
     __stdinp_nid_postfix = standardInput;
     Close(stream);
+
+    stream = Open("7 xy z");
+    Require(fscanf_nid_postfix(stream, "%3$d %1$ls %2$lc", word, &letter, &value) == 3 && value == 7 && std::u16string(word) == u"xy" && letter == u'z', "fscanf positional wide fields");
+    Close(stream);
 }
 
 static void CheckConsumption() {

docs/dev/debt/libc.md

@@ -243,7 +243,8 @@
 - [std::__throw_bad_variant_access](../../../core/libs/prx/libc/src/exception/StandardExceptions.cpp) (libc) throws `std::bad_variant_access` with the `what()` string `bad_variant_access`, the one [LLVM libc++](https://github.com/llvm/llvm-project/blob/main/libcxx/src/variant.cpp) returns, since the export carries libc++'s `__throw_bad_variant_access` name; the console's string is unverified. Onimusha: Way of the Sword (PPSA27836 01.000.001) imports it from `libc`, and its bundled `libc.prx` does not export it.
 - [std::bad_optional_access and std::_Throw_bad_optional_access](../../../core/libs/prx/libc/src/exception/StandardExceptions.cpp) (libc) provide the exception type, constructor, destructor, and `what()` exports alongside `_Throw_bad_optional_access`, returning the `what()` string `bad_optional_access` matching Dinkumware and libc++ conventions; the console libc.prx string is unverified. PPSA26578 (issue #7624) and PPSA34015 import `_ZSt26_Throw_bad_optional_accessv` (`KwzEip7m6TM`) from `libc`.
 - [std::_Throw(const std::bad_array_new_length&)](../../../core/libs/prx/libc/src/exception/StandardExceptions.cpp) (libc, NID `1cKlQv7Kvmg`) is a 52-byte function at `0xcb2d0` in the `libc.prx` bundled with PPSA02530: it ignores its argument, allocates the 8-byte exception object with the static `_ZTVSt20bad_array_new_length`, and throws it. PPSA28416 imports it from `libc`.
-- [The scanf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc), the `_s` forms included: where a stream is left after a failed 16-bit wide field (`%ls`, `%lc`, `%l[`, `%S`) was not measured; the byte the field failed on stays in the stream. Positional arguments next to a wide conversion throw. The rest follows the `libc.prx` of PPSA07923, PPSA05916 and PPSA13934 on every platform: each byte goes to `mbrtowc` with the state kept for the field, so a byte from 0x80 up ends the conversion. A held UTF-8 lead byte fails the field with `EILSEQ` when the field takes another byte, and is an input failure when the field ends; any other refused byte is an input failure as the field's first byte and a matching failure after it. Units read before a failure stay stored without a terminator, widths count bytes, scanset members and `X-Y` ranges are bytes (a reversed range matches nothing; a `-` first, last or after a range is a member), and `%C` ends the scan
+- [The scanf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc), the `_s` forms included: where a stream is left after a failed 16-bit wide field (`%ls`, `%lc`, `%l[`, `%S`) was not measured; the byte the field failed on stays in the stream. The rest follows the `libc.prx` of PPSA07923, PPSA05916 and PPSA13934 on every platform: each byte goes to `mbrtowc` with the state kept for the field, so a byte from 0x80 up ends the conversion. A held UTF-8 lead byte fails the field with `EILSEQ` when the field takes another byte, and is an input failure when the field ends; any other refused byte is an input failure as the field's first byte and a matching failure after it. Units read before a failure stay stored without a terminator, widths count bytes, scanset members and `X-Y` ranges are bytes (a reversed range matches nothing; a `-` first, last or after a range is a member), and `%C` ends the scan
+- [The scanf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc) reads positional `%n$` arguments next to a 16-bit wide conversion: `n$` takes the nth pointer. FreeBSD's `vfscanf` has no positional arguments and stops at the `$`; the console's behaviour is unverified. A format that mixes numbered and unnumbered assignments, which POSIX leaves undefined, throws, as do positional arguments in the `_s` forms.
 - [The scanf family](../../../core/libs/prx/libc/include/WindowsScanning.hpp) (libc) on Windows returns 0, as C17 and FreeBSD do, when input fails after only suppressed conversions have completed (`sscanf("12", "%*d%d", &value)`). Linux hands the format to glibc, which returns `EOF` there. On Windows the host stops a `%x`, `%X`, `%i` or `%p` after the `x` of a `0x` that no hex digit follows (or that the width cuts off), and a float conversion (`%e`, `%f`, `%g`, `%a`) after the `e` of an exponent with no digits (`1e`), where C17 and FreeBSD stop before the `x` or the `e`
 - [Narrow scansets, string lengths and unknown conversions](../../../core/libs/prx/libc/include/WindowsScanning.hpp) (libc): on Linux the plain narrow `scanf` family still hands these to glibc. In the measured Linux build, plain and secure `%lls` and `%Ls` store host 32-bit wide units where the bundled libraries store narrow bytes; plain `%s` at end of input can clear errno. The shared secure scanner stops unknown conversions before testing input EOF. A format byte from 0x80 up outside a scanset matches its input byte here, where the `libc.prx` of PPSA07923 and PPSA05916 fails the match; neither was checked further. On Windows, scansets are rewritten as explicit byte lists that follow those titles: `X-Y` is a range when a `-` follows a member and is not last, a reversed range matches nothing, a `-` first, last or after a range is a member, a `]` that opens the set can start a range, and an unterminated set ends the scan. `%s`, `%c` and `%[` read narrow with any length but `l`, and an unknown conversion such as `%y` ends the scan as a matching failure, also in the `_s` forms
 - [The scanf family](../../../core/libs/prx/libc/src/Formatting.cpp) (libc) - a `q` length is not measured on a console for `vsscanf`, `fscanf`, `vfscanf`, `scanf`, `vscanf` or the `_s` forms. On Linux and macOS the first four read it as `ll`, as the host libc and FreeBSD 11.4's source do; on Windows they end the scan at it. The `_s` forms take `q` as the conversion and pass `%q` to the host libc. `sscanf` ends the scan at the `q` directive with nothing stored on every host, as it does on a PS5 (firmware 12.70) for `%qd`, `%qS` and `%qC`, and `vsscanf` follows it, as it already did on Windows.