feat(libc): read 16-bit wide fields in fscanf on Linux

core/libs/prx/libc/include/ScanWideField.hpp

@@ -6,6 +6,7 @@
 
 namespace LibcDetail {
 
+std::size_t WideScanCharacterLength(unsigned char lead);
 std::size_t ReadWideScanCharacter(const char* text, char32_t* point);
 bool InWideScanset(const std::string& specifier, char32_t point);
 void RequireNoPositionalWideScan(const char* format);

core/libs/prx/libc/src/Formatting.cpp

@@ -84,10 +84,6 @@ bool ScansWideInProcess(const char* format) {
     return true;
 }
 
-void RejectGuestWideScan(const char* format) {
-    if (HasGuestWideScan(format)) NotImplemented_nid_no_patch("fscanf 16-bit wide conversion on Linux");
-}
-
 bool IsWideScan(const std::string& length, char conversion) {
     return conversion == 'S' || conversion == 'C' || (length == "l" && (conversion == 's' || conversion == 'c' || conversion == '['));
 }
@@ -211,6 +207,12 @@ class StringSource {
     std::ptrdiff_t Consumed() const { return position - start; }
     const char* Remaining() const { return position; }
 
+#ifndef _WIN32
+    std::size_t PeekWide(char32_t* point) const {
+        return *position == '\0' ? 0 : LibcDetail::ReadWideScanCharacter(position, point);
+    }
+#endif
+
     int Scan(const char* directive, bool suppress, void* target, int* consumed) const {
         return suppress ? HostSscanf(position, directive, consumed) : HostSscanf(position, directive, target, consumed);
     }
@@ -239,6 +241,26 @@ class StreamSource {
     void Advance(std::ptrdiff_t count) { consumed += count; }
     std::ptrdiff_t Consumed() const { return consumed; }
 
+#ifndef _WIN32
+    std::size_t PeekWide(char32_t* point) const {
+        char bytes[5] = {};
+        int next = std::fgetc(file);
+        if (next == EOF) return 0;
+        const std::size_t length = std::max<std::size_t>(LibcDetail::WideScanCharacterLength(static_cast<unsigned char>(next)), 1);
+        std::size_t size = 0;
+        while (true) {
+            bytes[size++] = static_cast<char>(next);
+            if (size == length || (next = std::fgetc(file)) == EOF) break;
+            if ((next & 0xC0) != 0x80) {
+                std::ungetc(next, file);
+                break;
+            }
+        }
+        for (std::size_t index = size; index != 0; --index) std::ungetc(static_cast<unsigned char>(bytes[index - 1]), file);
+        return LibcDetail::ReadWideScanCharacter(bytes, point);
+    }
+#endif
+
     int Scan(const char* directive, bool suppress, void* target, int* count) const {
         return suppress ? HostFscanf(file, directive, count) : HostFscanf(file, directive, target, count);
     }
@@ -301,14 +323,12 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
             continue;
         }
 #ifndef _WIN32
-        if constexpr (TSource::Contiguous) if (IsWideScan(length, conversion)) {
-            const char* const input = source.Remaining();
+        if (IsWideScan(length, conversion)) {
             const char kind = conversion == 'S' ? 's' : conversion == 'C' ? 'c' : conversion;
-            const char* field = input;
             if (kind == 's') {
-                while (std::isspace(static_cast<unsigned char>(*field))) ++field;
+                while (std::isspace(source.Peek())) source.Skip();
             }
-            if (*field == '\0') return finish(converted ? assigned : EOF);
+            if (source.Peek() == EOF) return finish(converted ? assigned : EOF);
             auto* target = suppress ? nullptr : static_cast<char16_t*>(nextPointer());
             unsigned long limit = ULONG_MAX;
             if (secure && !suppress) {
@@ -327,18 +347,17 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
                 if (kind == '[') return LibcDetail::InWideScanset(specifier, point);
                 return true;
             };
-            while (text.size() < taken && *field != '\0') {
+            while (text.size() < taken) {
                 char32_t point = 0;
-                const std::size_t size = LibcDetail::ReadWideScanCharacter(field, &point);
-                if (!accepts(point)) break;
+                const std::size_t size = source.PeekWide(&point);
+                if (size == 0 || !accepts(point)) break;
                 text.push_back(static_cast<char16_t>(point));
-                field += size;
+                for (std::size_t index = 0; index < size; ++index) source.Skip();
             }
             if (text.empty()) return finish(assigned);
-            if (target != nullptr && kind != 'c' && wanted > limit && text.size() == limit && *field != '\0') {
+            if (target != nullptr && kind != 'c' && wanted > limit && text.size() == limit) {
                 char32_t point = 0;
-                LibcDetail::ReadWideScanCharacter(field, &point);
-                if (accepts(point)) {
+                if (source.PeekWide(&point) != 0 && accepts(point)) {
                     target[0] = u'\0';
                     return finish(assigned);
                 }
@@ -348,7 +367,6 @@ int ScanSource(TSource source, const char* format, bool secure, NextPointer next
                 if (kind != 'c') target[text.size()] = u'\0';
                 ++assigned;
             }
-            source.Advance(field - input);
             converted = true;
             continue;
         }
@@ -446,7 +464,7 @@ int ScanGuestStream(FileStream* stream, const char* format, NextPointer nextPoin
     auto* native = GetNativeStream(stream);
     if (!stream->SwitchToReading()) return EOF;
 #ifndef _WIN32
-    RejectGuestWideScan(format);
+    if (HasGuestWideScan(format)) LibcDetail::RequireNoPositionalWideScan(format);
 #endif
     const int result = ScanSource(StreamSource(native), format, true, nextPointer, nextCapacity);
     stream->SyncStatus();
@@ -545,10 +563,10 @@ int APS5_VABI fscanf_nid_postfix(FileStream* stream, const char* format, ...) {
     const int result = LibcDetail::ScanFileWindows_nid_no_patch(native, format, args);
     __builtin_sysv_va_end(args);
 #else
-    RejectGuestWideScan(format);
     std::va_list args;
     va_start(args, format);
-    const int result = HostVfscanf(native, format, args);
+    const int result = ScansWideInProcess(format) ? ScanSource(StreamSource(native), format, false, [&] { return va_arg(args, void*); }, [] { return 0u; })
+                                                  : HostVfscanf(native, format, args);
     va_end(args);
 #endif
     stream->SyncStatus();
@@ -874,11 +892,16 @@ int APS5_VABI vfscanf_nid_postfix(FileStream* stream, const char* format, VaList
 #ifdef _WIN32
     const int result = LibcDetail::ScanFileWindows_nid_no_patch(native, format, args);
 #else
-    RejectGuestWideScan(format);
-    std::va_list copy;
-    va_copy(copy, *reinterpret_cast<std::va_list*>(args));
-    const int result = HostVfscanf(native, format, copy);
-    va_end(copy);
+    int result = 0;
+    if (ScansWideInProcess(format)) {
+        LibcDetail::FormatArguments arguments(args);
+        result = ScanSource(StreamSource(native), format, false, [&] { return arguments.Next<void*>(); }, [] { return 0u; });
+    } else {
+        std::va_list copy;
+        va_copy(copy, *reinterpret_cast<std::va_list*>(args));
+        result = HostVfscanf(native, format, copy);
+        va_end(copy);
+    }
 #endif
     stream->SyncStatus();
     stream->KeepDirectoryReadError(result == -1);

core/libs/prx/libc/src/ScanWideField.cpp

@@ -11,12 +11,9 @@ std::size_t DecodeUtf8(const char* text, char32_t* point) {
         *point = lead;
         return 1;
     }
-    std::size_t size = 0;
-    char32_t value = 0;
-    if ((lead & 0xE0) == 0xC0) { size = 2; value = lead & 0x1F; }
-    else if ((lead & 0xF0) == 0xE0) { size = 3; value = lead & 0x0F; }
-    else if ((lead & 0xF8) == 0xF0) { size = 4; value = lead & 0x07; }
-    else return 0;
+    const std::size_t size = LibcDetail::WideScanCharacterLength(lead);
+    if (size == 0) return 0;
+    char32_t value = lead & (0x7F >> size);
     for (std::size_t index = 1; index < size; ++index) {
         const auto next = static_cast<unsigned char>(text[index]);
         if ((next & 0xC0) != 0x80) return 0;
@@ -32,6 +29,14 @@ std::size_t DecodeUtf8(const char* text, char32_t* point) {
 
 namespace LibcDetail {
 
+std::size_t WideScanCharacterLength(unsigned char lead) {
+    if (lead < 0x80) return 1;
+    if ((lead & 0xE0) == 0xC0) return 2;
+    if ((lead & 0xF0) == 0xE0) return 3;
+    if ((lead & 0xF8) == 0xF0) return 4;
+    return 0;
+}
+
 std::size_t ReadWideScanCharacter(const char* text, char32_t* point) {
     const std::size_t size = DecodeUtf8(text, point);
     if (size == 0) NotImplemented_nid_no_patch("scanf 16-bit wide conversion of input that is not UTF-8");

core/libs/tests/GuestWideStreamScan.cpp

@@ -6,6 +6,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
+#include <cstring>
 #include <string>
 
 extern "C" {
@@ -55,6 +56,29 @@ static int APS5_VABI ScanInputSized(const char16_t* format, ...) {
     return result;
 }
 
+#ifndef _WIN32
+static int APS5_VABI ScanNarrow(FileStream* stream, const char* format, ...) {
+    APS5_VA_BEGIN(format);
+    const int result = vfscanf_nid_postfix(stream, format, reinterpret_cast<VaList*>(args));
+    APS5_VA_END();
+    return result;
+}
+
+static int APS5_VABI ScanNarrowSized(FileStream* stream, const char* format, ...) {
+    APS5_VA_BEGIN(format);
+    const int result = vfscanf_s_nid_postfix(stream, format, reinterpret_cast<VaList*>(args));
+    APS5_VA_END();
+    return result;
+}
+
+static int APS5_VABI ScanNarrowInput(const char* format, ...) {
+    APS5_VA_BEGIN(format);
+    const int result = vscanf_nid_postfix(format, reinterpret_cast<VaList*>(args));
+    APS5_VA_END();
+    return result;
+}
+#endif
+
 static FileStream* Open(const char* text) {
     auto* stream = new FileStream(TemporaryFile::Open());
     Require(std::fputs(text, stream->GetHandle()) >= 0, "seed the stream");
@@ -67,6 +91,61 @@ static void Close(FileStream* stream) {
     delete stream;
 }
 
+#ifndef _WIN32
+static void CheckNarrowFormatWideFields() {
+    std::int32_t value = 0;
+    std::int64_t count = -1;
+    char16_t word[8]{};
+    char16_t letter = u'!';
+    char narrow[8]{};
+
+    FileStream* stream = Open("  \xc5\xbelu\xc5\xa5 k\xc5\xaf\xc5\x88 7");
+    Require(fscanf_nid_postfix(stream, "%ls %lc", word, &letter) == 2 && std::u16string(word) == u"\u017elu\u0165" && letter == u'k', "fscanf %ls and %lc decode UTF-8");
+    Require(ScanNarrow(stream, "%S%ln %d", word, &count, &value) == 2 && std::u16string(word) == u"\u016f\u0148" && count == 4 && value == 7, "vfscanf %S with %ln counting bytes");
+    Require(fgetc_nid_postfix(stream) == EOF, "the whole input is consumed");
+    Close(stream);
+
+    stream = Open("12 ab cd");
+    Require(fscanf_nid_postfix(stream, "%d %ls %7s", &value, word, narrow) == 3 && value == 12 && std::u16string(word) == u"ab" && std::strcmp(narrow, "cd") == 0, "wide and narrow fields share the stream");
+    Close(stream);
+
+    stream = Open("ab\xc3\xa9t rest");
+    Require(fscanf_nid_postfix(stream, "%l[a-c]", word) == 1 && std::u16string(word) == u"ab", "a scanset stops at a non-ASCII character");
+    Require(fscanf_nid_postfix(stream, "%l[^ ] %7s", word, narrow) == 2 && std::u16string(word) == u"\u00e9t" && std::strcmp(narrow, "rest") == 0, "the non-ASCII character stays in the stream");
+    Close(stream);
+
+    char16_t three[5] = {u'!', u'!', u'!', u'!', u'!'};
+    stream = Open("xyzw");
+    Require(fscanf_nid_postfix(stream, "%3lc", three) == 1 && three[0] == u'x' && three[1] == u'y' && three[2] == u'z' && three[3] == u'!', "%3lc stores three units without a terminator");
+    Require(fgetc_nid_postfix(stream) == 'w', "the character after %3lc stays in the stream");
+    Close(stream);
+
+    stream = Open("xyz");
+    Require(fscanf_nid_postfix(stream, "%l[a-c]", word) == 0, "a scanset matching nothing");
+    Require(fgetc_nid_postfix(stream) == 'x', "the rejected character stays in the stream");
+    Close(stream);
+
+    stream = Open("   ");
+    Require(fscanf_nid_postfix(stream, "%ls", word) == EOF, "only white space before %ls");
+    Close(stream);
+
+    stream = Open("hello world");
+    char16_t fits[6]{};
+    char16_t small[4] = {u'!', u'!', u'!', u'!'};
+    Require(fscanf_s_nid_postfix(stream, "%ls", fits, 6u) == 1 && std::u16string(fits) == u"hello", "fscanf_s %ls that fits");
+    Require(ScanNarrowSized(stream, "%ls", small, 4u) == 0 && small[0] == 0, "vfscanf_s rejects a too small buffer");
+    Close(stream);
+
+    FileStream* const standardInput = __stdinp_nid_postfix;
+    stream = Open("ab c");
+    __stdinp_nid_postfix = stream;
+    Require(scanf_nid_postfix("%ls", word) == 1 && std::u16string(word) == u"ab", "scanf %ls reads the guest standard input");
+    Require(ScanNarrowInput(" %C", &letter) == 1 && letter == u'c', "vscanf %C reads the guest standard input");
+    __stdinp_nid_postfix = standardInput;
+    Close(stream);
+}
+#endif
+
 static void CheckConsumption() {
     std::int32_t value = -1;
     std::int64_t count = -1;
@@ -166,5 +245,8 @@ int main() {
     CheckConsumption();
     CheckWideText();
     CheckStandardInput();
+#ifndef _WIN32
+    CheckNarrowFormatWideFields();
+#endif
     return 0;
 }