feat(libSceIcu): implement u_strCompareIter and u_getUnicodeVersion

core/libs/prx/libSceIcu/CMakeLists.txt

@@ -3,6 +3,7 @@ target_include_directories(libSceIcu PRIVATE ${LIBS_INCLUDE_DIR})
 configure_windows_unwind(libSceIcu)
 target_sources(libSceIcu PRIVATE ConverterCallbacks.cpp)
 target_sources(libSceIcu PRIVATE CString.cpp)
+target_sources(libSceIcu PRIVATE CompareIter.cpp)
 target_sources(libSceIcu PRIVATE Space.cpp)
 target_sources(libSceIcu PRIVATE Utf8.cpp)
 
@@ -83,3 +84,16 @@ if(BUILD_TESTING)
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceIcu>")
     endif()
 endif()
+
+if(BUILD_TESTING)
+    add_test_executable(guest_icu_compare_iter_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/CompareIter.cpp)
+    target_include_directories(guest_icu_compare_iter_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(guest_icu_compare_iter_tests PRIVATE libSceIcu)
+    configure_windows_unwind(guest_icu_compare_iter_tests)
+    add_test(NAME guest_icu_compare_iter COMMAND guest_icu_compare_iter_tests)
+    set_tests_properties(guest_icu_compare_iter PROPERTIES TIMEOUT 30)
+    if(WIN32)
+        set_tests_properties(guest_icu_compare_iter PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceIcu>")
+    endif()
+endif()

core/libs/prx/libSceIcu/CompareIter.cpp

@@ -0,0 +1,62 @@
+#include "prx/libSceIcu/libSceIcu.hpp"
+#include <cstdint>
+#include <stdexcept>
+
+namespace {
+
+constexpr std::int32_t Sentinel = -1;
+
+bool IsLead(std::int32_t unit) {
+    return (static_cast<std::uint32_t>(unit) & 0xfffffc00u) == 0xd800u;
+}
+
+bool IsTrail(std::int32_t unit) {
+    return (static_cast<std::uint32_t>(unit) & 0xfffffc00u) == 0xdc00u;
+}
+
+template<typename TCallback>
+TCallback Require(TCallback callback) {
+    if (!callback) throw std::invalid_argument("u_strCompareIter: null iterator callback");
+    return callback;
+}
+
+std::int32_t Next(UCharIterator* iterator) {
+    return Require(iterator->next)(iterator);
+}
+
+bool InPair(UCharIterator* iterator, std::int32_t unit) {
+    if (unit <= 0xdbff) return IsTrail(Require(iterator->current)(iterator));
+    if (!IsTrail(unit)) return false;
+    const auto previous = Require(iterator->previous);
+    previous(iterator);
+    return IsLead(Require(iterator->previous)(iterator));
+}
+
+std::int32_t Order(UCharIterator* iterator, std::int32_t unit) {
+    return InPair(iterator, unit) ? unit : unit - 0x2800;
+}
+
+}
+
+extern "C" {
+
+std::int32_t APS5_VABI u_strCompareIter_nid_postfix(UCharIterator* first, UCharIterator* second, std::uint8_t codePointOrder) {
+    if (!first || !second || first == second) return 0;
+    Require(first->move)(first, 0, UITER_START);
+    Require(second->move)(second, 0, UITER_START);
+    std::int32_t left;
+    std::int32_t right;
+    for (;;) {
+        left = Next(first);
+        right = Next(second);
+        if (left != right) break;
+        if (left == Sentinel) return 0;
+    }
+    if (left >= 0xd800 && right >= 0xd800 && codePointOrder) {
+        left = Order(first, left);
+        right = Order(second, right);
+    }
+    return static_cast<std::int32_t>(static_cast<std::uint32_t>(left) - static_cast<std::uint32_t>(right));
+}
+
+}

core/libs/prx/libSceIcu/Export.cpp

@@ -204,11 +204,6 @@ int APS5_VABI sceIcuUnknown_5__ZFSRQ0_k_nid_no_patch() {
 }
 APS5_EXPORT("5+-ZFSRQ0+k", sceIcuUnknown_5__ZFSRQ0_k_nid_no_patch);
 
-int APS5_VABI sceIcuUnknown_zGuhKwXM9I0_nid_no_patch() {
-    NotImplemented_nid_no_patch(__func__);
-}
-APS5_EXPORT("zGuhKwXM9I0", sceIcuUnknown_zGuhKwXM9I0_nid_no_patch);
-
 int APS5_VABI u_hasBinaryProperty_nid_postfix() {
     NotImplemented_nid_no_patch(__func__);
 }
@@ -348,11 +343,6 @@ int APS5_VABI sceIcuUnknown_pgvyZZ_d1ck_nid_no_patch() {
 }
 APS5_EXPORT("pgvyZZ+d1ck", sceIcuUnknown_pgvyZZ_d1ck_nid_no_patch);
 
-int APS5_VABI sceIcuUnknown_TNZrG2iXikw_nid_no_patch() {
-    NotImplemented_nid_no_patch(__func__);
-}
-APS5_EXPORT("TNZrG2iXikw", sceIcuUnknown_TNZrG2iXikw_nid_no_patch);
-
 int APS5_VABI u_strFoldCase_nid_postfix() {
     NotImplemented_nid_no_patch(__func__);
 }

core/libs/prx/libSceIcu/Iterator.hpp

@@ -0,0 +1,51 @@
+#pragma once
+#include "prx/libc/include/general/VabiMacros.hpp"
+#include <cstddef>
+#include <cstdint>
+
+enum UCharIteratorOrigin : std::int32_t {
+    UITER_START,
+    UITER_CURRENT,
+    UITER_LIMIT,
+    UITER_ZERO,
+    UITER_LENGTH
+};
+
+struct UCharIterator {
+    using GetIndexFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator, UCharIteratorOrigin origin);
+    using MoveFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator, std::int32_t delta, UCharIteratorOrigin origin);
+    using HasNextFn = std::int8_t (APS5_VABI *)(UCharIterator* iterator);
+    using HasPreviousFn = std::int8_t (APS5_VABI *)(UCharIterator* iterator);
+    using CurrentFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator);
+    using NextFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator);
+    using PreviousFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator);
+    using ReservedFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator, std::int32_t something);
+    using GetStateFn = std::uint32_t (APS5_VABI *)(const UCharIterator* iterator);
+    using SetStateFn = void (APS5_VABI *)(UCharIterator* iterator, std::uint32_t state, std::int32_t* errorCode);
+
+    const void* context;
+    std::int32_t length;
+    std::int32_t start;
+    std::int32_t index;
+    std::int32_t limit;
+    std::int32_t reservedField;
+    GetIndexFn getIndex;
+    MoveFn move;
+    HasNextFn hasNext;
+    HasPreviousFn hasPrevious;
+    CurrentFn current;
+    NextFn next;
+    PreviousFn previous;
+    ReservedFn reservedFn;
+    GetStateFn getState;
+    SetStateFn setState;
+};
+
+static_assert(sizeof(UCharIteratorOrigin) == 4);
+static_assert(offsetof(UCharIterator, getIndex) == 32);
+static_assert(offsetof(UCharIterator, setState) == 104);
+static_assert(sizeof(UCharIterator) == 112);
+
+extern "C" {
+std::int32_t APS5_VABI u_strCompareIter_nid_postfix(UCharIterator* first, UCharIterator* second, std::uint8_t codePointOrder);
+}

core/libs/prx/libSceIcu/Version.cpp

@@ -7,6 +7,7 @@ namespace {
 constexpr std::uint16_t VersionLength = 4;
 constexpr std::int32_t VersionStringLength = 20;
 constexpr char VersionDelimiter = '.';
+constexpr std::uint8_t UnicodeVersion[VersionLength] = {17, 0, 0, 0};
 
 char* WriteField(char* text, std::uint8_t field) {
     if (field >= 100) {
@@ -66,4 +67,9 @@ void APS5_VABI u_versionToString_nid_postfix(const std::uint8_t* version, char*
     *text = 0;
 }
 
+void APS5_VABI u_getUnicodeVersion_nid_postfix(std::uint8_t* version) {
+    if (!version) return;
+    for (std::uint16_t part = 0; part < VersionLength; ++part) version[part] = UnicodeVersion[part];
+}
+
 }

core/libs/prx/libSceIcu/Version.hpp

@@ -6,4 +6,5 @@ extern "C" {
 void APS5_VABI u_versionFromString_nid_postfix(std::uint8_t*, const char*);
 void APS5_VABI u_versionFromUString_nid_postfix(std::uint8_t*, const char16_t*);
 void APS5_VABI u_versionToString_nid_postfix(const std::uint8_t*, char*);
+void APS5_VABI u_getUnicodeVersion_nid_postfix(std::uint8_t*);
 }

core/libs/prx/libSceIcu/libSceIcu.hpp

@@ -1,5 +1,6 @@
 #pragma once
 #include "Invariant.hpp"
+#include "Iterator.hpp"
 #include "Utf8.hpp"
 #include "Version.hpp"
 #include "ConverterCallbacks.hpp"

core/libs/prx/libSceIcu/tests/CompareIter.cpp

@@ -0,0 +1,195 @@
+#include "prx/libSceIcu/libSceIcu.hpp"
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <limits>
+#include <source_location>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+void Require(bool condition, std::source_location location = std::source_location::current()) {
+    if (condition) return;
+    std::fprintf(stderr, "ICU iterator comparison check failed at line %u\n", location.line());
+    std::abort();
+}
+
+struct Text {
+    std::vector<std::int32_t> units;
+    std::string calls;
+};
+
+Text& Of(UCharIterator* iterator) {
+    return *static_cast<Text*>(const_cast<void*>(iterator->context));
+}
+
+std::int32_t APS5_VABI Move_nid_no_patch(UCharIterator* iterator, std::int32_t delta, UCharIteratorOrigin origin) {
+    auto& text = Of(iterator);
+    Require(origin == UITER_START);
+    text.calls += 'm';
+    iterator->index = iterator->start + delta;
+    return iterator->index;
+}
+
+std::int32_t APS5_VABI Next_nid_no_patch(UCharIterator* iterator) {
+    auto& text = Of(iterator);
+    text.calls += 'n';
+    return iterator->index < iterator->limit ? text.units[iterator->index++] : -1;
+}
+
+std::int32_t APS5_VABI Current_nid_no_patch(UCharIterator* iterator) {
+    auto& text = Of(iterator);
+    text.calls += 'c';
+    return iterator->index < iterator->limit ? text.units[iterator->index] : -1;
+}
+
+std::int32_t APS5_VABI Previous_nid_no_patch(UCharIterator* iterator) {
+    auto& text = Of(iterator);
+    text.calls += 'p';
+    return iterator->index > iterator->start ? text.units[--iterator->index] : -1;
+}
+
+struct Iterator {
+    Text text;
+    UCharIterator iterator{};
+
+    explicit Iterator(std::vector<std::int32_t> units, std::int32_t index = 0) : text{std::move(units), {}} {
+        iterator.context = &text;
+        iterator.length = static_cast<std::int32_t>(text.units.size());
+        iterator.start = 0;
+        iterator.index = index;
+        iterator.limit = iterator.length;
+        iterator.move = Move_nid_no_patch;
+        iterator.next = Next_nid_no_patch;
+        iterator.current = Current_nid_no_patch;
+        iterator.previous = Previous_nid_no_patch;
+    }
+};
+
+struct Result {
+    std::int32_t value;
+    std::string first;
+    std::string second;
+};
+
+Result Compare(std::vector<std::int32_t> left, std::vector<std::int32_t> right, std::uint8_t codePointOrder) {
+    Iterator first(std::move(left), 1);
+    Iterator second(std::move(right), 1);
+    const auto value = u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, codePointOrder);
+    return {value, first.text.calls, second.text.calls};
+}
+
+void Expect(std::vector<std::int32_t> left, std::vector<std::int32_t> right, std::uint8_t codePointOrder,
+    std::int32_t value, const char* first, const char* second, std::source_location location = std::source_location::current()) {
+    const auto result = Compare(std::move(left), std::move(right), codePointOrder);
+    Require(result.value == value, location);
+    Require(result.first == first, location);
+    Require(result.second == second, location);
+}
+
+template<typename TAction>
+void RequireInvalidArgument(TAction action, std::source_location location = std::source_location::current()) {
+    try { action(); } catch (const std::invalid_argument&) { return; }
+    Require(false, location);
+}
+
+void Arguments() {
+    Iterator text({u'a'});
+    Require(u_strCompareIter_nid_postfix(nullptr, &text.iterator, 1) == 0);
+    Require(u_strCompareIter_nid_postfix(&text.iterator, nullptr, 1) == 0);
+    Require(u_strCompareIter_nid_postfix(nullptr, nullptr, 0) == 0);
+    Require(u_strCompareIter_nid_postfix(&text.iterator, &text.iterator, 1) == 0);
+    Require(text.text.calls.empty());
+}
+
+void Units() {
+    Expect({}, {}, 0, 0, "mn", "mn");
+    Expect({u'a', u'b'}, {u'a', u'b'}, 1, 0, "mnnn", "mnnn");
+    Expect({u'a', u'b', u'c'}, {u'a', u'b', u'd'}, 0, -1, "mnnn", "mnnn");
+    Expect({u'a', u'b'}, {u'a', u'b', u'c'}, 1, -1 - u'c', "mnnn", "mnnn");
+    Expect({u'b'}, {u'a', u'z'}, 1, 1, "mn", "mn");
+    Expect({0xff61}, {0xd800, 0xdc00}, 0, 0xff61 - 0xd800, "mn", "mn");
+    Expect({0xdc00}, {0xd800, 0xdc00}, 0, 0x400, "mn", "mn");
+}
+
+void CodePoints() {
+    Expect({0xff61}, {0xd800, 0xdc00}, 1, 0xff61 - 0x2800 - 0xd800, "mn", "mnc");
+    Expect({0xd800, 0xdc00}, {0xff61}, 0x80, 0xd800 - (0xff61 - 0x2800), "mnc", "mn");
+    Expect({0xdc00}, {0xd800, 0xdc00}, 1, 0xdc00 - 0x2800 - 0xd800, "mnpp", "mnc");
+    Expect({0xd800, 0xdc00}, {0xd800, 0xdc01}, 1, -1, "mnnpp", "mnnpp");
+    Expect({u'x', 0xdc00}, {u'x', 0xe000}, 1, 0xdc00 - 0xe000, "mnnpp", "mnn");
+    Expect({0xd800}, {0xe000}, 1, -0x800, "mnc", "mn");
+    Expect({0xd800, u'a'}, {0xdbff, 0xdfff}, 1, 0xd800 - 0x2800 - 0xdbff, "mnc", "mnc");
+    Expect({0xffff}, {0x10000}, 1, 0xffff - 0x2800 - (0x10000 - 0x2800), "mn", "mn");
+    Expect({u'a', 0xd7ff}, {u'a', 0xd800}, 1, -1, "mnn", "mnn");
+    Expect({0xd800}, {}, 1, 0xd801, "mn", "mn");
+}
+
+void Wrapping() {
+    Expect({std::numeric_limits<std::int32_t>::min()}, {1}, 0, std::numeric_limits<std::int32_t>::max(), "mn", "mn");
+    Expect({std::numeric_limits<std::int32_t>::max()}, {-1}, 1, std::numeric_limits<std::int32_t>::min(), "mn", "mn");
+}
+
+void Callbacks() {
+    for (int missing = 0; missing < 4; ++missing) {
+        Iterator first({0xdc00});
+        Iterator second({0xd800, 0xdc00});
+        auto& target = missing < 2 ? first.iterator : second.iterator;
+        if (missing % 2 == 0) target.move = nullptr; else target.next = nullptr;
+        RequireInvalidArgument([&] { u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1); });
+    }
+    {
+        Iterator first({0xdc00});
+        Iterator second({0xd800, 0xdc00});
+        first.iterator.previous = nullptr;
+        RequireInvalidArgument([&] { u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1); });
+    }
+    {
+        Iterator first({0xdc00});
+        Iterator second({0xd800, 0xdc00});
+        second.iterator.current = nullptr;
+        RequireInvalidArgument([&] { u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1); });
+    }
+    {
+        Iterator first({0xdc00});
+        Iterator second({0xd800, 0xdc00});
+        first.iterator.current = nullptr;
+        second.iterator.previous = nullptr;
+        Require(u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1) == 0xdc00 - 0x2800 - 0xd800);
+        Require(first.text.calls == "mnpp");
+        Require(second.text.calls == "mnc");
+    }
+    {
+        Iterator first({u'a'});
+        Iterator second({u'b'});
+        first.iterator.current = nullptr;
+        first.iterator.previous = nullptr;
+        second.iterator.current = nullptr;
+        second.iterator.previous = nullptr;
+        Require(u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1) == -1);
+    }
+}
+
+void UnicodeVersion() {
+    std::array<std::uint8_t, 6> version{};
+    version.fill(0xa5);
+    u_getUnicodeVersion_nid_postfix(version.data());
+    Require(version == std::array<std::uint8_t, 6>{17, 0, 0, 0, 0xa5, 0xa5});
+    u_getUnicodeVersion_nid_postfix(nullptr);
+}
+
+}
+
+int main() {
+    Arguments();
+    Units();
+    CodePoints();
+    Wrapping();
+    Callbacks();
+    UnicodeVersion();
+    return 0;
+}

docs/dev/debt/libSceIcu.md

@@ -18,6 +18,8 @@
 
 - [Invariant character and version helpers](../../../core/libs/prx/libSceIcu/Version.cpp) are unverified on a console. `u_charsToUChars`, `u_UCharsToChars`, `u_versionFromString`, `u_versionFromUString` and `u_versionToString` follow ICU 78.1 [uinvchar.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/uinvchar.cpp) and [putil.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/putil.cpp), unchanged since 65.1. Firmware 13.42 reports ICU 78.1 and exports none of them ([#11013](https://github.com/boykopovar/AnyPS5/pull/11013#issuecomment-6400470698)); the [3.20 table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceIcu.c) has all five. Version parts use the guest `strtoul`, truncated to eight bits; `u_versionToString` keeps ICU's dropped zero tens digit (105 becomes `15`). A null buffer with a positive conversion length throws; ICU dereferences it. Closes when: compared with the console.
 
+- [Iterator comparison and Unicode version](../../../core/libs/prx/libSceIcu/CompareIter.cpp) are unverified on a console. `u_strCompareIter` and `u_getUnicodeVersion` follow ICU 78.1 [ustring.cpp](https://github.com/unicode-org/icu/blob/049e0d6a4206/icu4c/source/common/ustring.cpp#L828-L882) and [uchar.cpp](https://github.com/unicode-org/icu/blob/049e0d6a4206/icu4c/source/common/uchar.cpp#L484-L489); the version is 17.0, the Unicode data of 78.1, whatever firmware is emulated. A null iterator callback that ICU would call throws `std::invalid_argument`; ICU dereferences it. The code unit difference wraps instead of overflowing. The `UBool` is read as one byte, nonzero meaning true. Closes when: compared with the console.
+
 - [Default-converter string copy helpers](../../../core/libs/prx/libSceIcu/Invariant.cpp) `u_austrcpy` and `u_austrncpy` are unverified on a console and throw `NotImplemented_nid_no_patch` on code units above `0x7f`. Their names and NIDs come from the PS5 3.20 inventory; signatures and ASCII (`0x01`–`0x7f`) behavior follow ICU 78.1's [ustr_cnv.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/ustr_cnv.cpp), unchanged since 65.1, without assuming the console's default converter for non-ASCII units. `u_austrncpy` writes a single NUL when the source ends before the count without zero-padding the remainder. Null buffers with `u_austrcpy` or a positive count throw an invalid-argument exception, whereas ICU dereferences in `u_austrcpy` and returns the destination unchanged in `u_austrncpy`. Closes when: compared with the console, including the default converter for non-ASCII units.
 
 ### Known export names; ABI prototypes unverified