diff --git a/core/libs/prx/libSceIcu/CMakeLists.txt b/core/libs/prx/libSceIcu/CMakeLists.txt index 4458d7708f25..ebffb2153c28 100644 --- a/core/libs/prx/libSceIcu/CMakeLists.txt +++ b/core/libs/prx/libSceIcu/CMakeLists.txt @@ -3,6 +3,7 @@ target_include_directories(libSceIcu PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceIcu) target_sources(libSceIcu PRIVATE ConverterCallbacks.cpp) target_sources(libSceIcu PRIVATE CString.cpp) +target_sources(libSceIcu PRIVATE CompareIter.cpp) target_sources(libSceIcu PRIVATE Space.cpp) target_sources(libSceIcu PRIVATE Utf8.cpp) @@ -83,3 +84,16 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + add_test_executable(guest_icu_compare_iter_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/CompareIter.cpp) + target_include_directories(guest_icu_compare_iter_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_icu_compare_iter_tests PRIVATE libSceIcu) + configure_windows_unwind(guest_icu_compare_iter_tests) + add_test(NAME guest_icu_compare_iter COMMAND guest_icu_compare_iter_tests) + set_tests_properties(guest_icu_compare_iter PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_icu_compare_iter PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceIcu/CompareIter.cpp b/core/libs/prx/libSceIcu/CompareIter.cpp new file mode 100644 index 000000000000..f214fd55e851 --- /dev/null +++ b/core/libs/prx/libSceIcu/CompareIter.cpp @@ -0,0 +1,62 @@ +#include "prx/libSceIcu/libSceIcu.hpp" +#include +#include + +namespace { + +constexpr std::int32_t Sentinel = -1; + +bool IsLead(std::int32_t unit) { + return (static_cast(unit) & 0xfffffc00u) == 0xd800u; +} + +bool IsTrail(std::int32_t unit) { + return (static_cast(unit) & 0xfffffc00u) == 0xdc00u; +} + +template +TCallback Require(TCallback callback) { + if (!callback) throw std::invalid_argument("u_strCompareIter: null iterator callback"); + return callback; +} + +std::int32_t Next(UCharIterator* iterator) { + return Require(iterator->next)(iterator); +} + +bool InPair(UCharIterator* iterator, std::int32_t unit) { + if (unit <= 0xdbff) return IsTrail(Require(iterator->current)(iterator)); + if (!IsTrail(unit)) return false; + const auto previous = Require(iterator->previous); + previous(iterator); + return IsLead(Require(iterator->previous)(iterator)); +} + +std::int32_t Order(UCharIterator* iterator, std::int32_t unit) { + return InPair(iterator, unit) ? unit : unit - 0x2800; +} + +} + +extern "C" { + +std::int32_t APS5_VABI u_strCompareIter_nid_postfix(UCharIterator* first, UCharIterator* second, std::uint8_t codePointOrder) { + if (!first || !second || first == second) return 0; + Require(first->move)(first, 0, UITER_START); + Require(second->move)(second, 0, UITER_START); + std::int32_t left; + std::int32_t right; + for (;;) { + left = Next(first); + right = Next(second); + if (left != right) break; + if (left == Sentinel) return 0; + } + if (left >= 0xd800 && right >= 0xd800 && codePointOrder) { + left = Order(first, left); + right = Order(second, right); + } + return static_cast(static_cast(left) - static_cast(right)); +} + +} diff --git a/core/libs/prx/libSceIcu/Export.cpp b/core/libs/prx/libSceIcu/Export.cpp index 2e8f0550c5e0..119bb9ed64fc 100644 --- a/core/libs/prx/libSceIcu/Export.cpp +++ b/core/libs/prx/libSceIcu/Export.cpp @@ -204,11 +204,6 @@ int APS5_VABI sceIcuUnknown_5__ZFSRQ0_k_nid_no_patch() { } APS5_EXPORT("5+-ZFSRQ0+k", sceIcuUnknown_5__ZFSRQ0_k_nid_no_patch); -int APS5_VABI sceIcuUnknown_zGuhKwXM9I0_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); -} -APS5_EXPORT("zGuhKwXM9I0", sceIcuUnknown_zGuhKwXM9I0_nid_no_patch); - int APS5_VABI u_hasBinaryProperty_nid_postfix() { NotImplemented_nid_no_patch(__func__); } @@ -348,11 +343,6 @@ int APS5_VABI sceIcuUnknown_pgvyZZ_d1ck_nid_no_patch() { } APS5_EXPORT("pgvyZZ+d1ck", sceIcuUnknown_pgvyZZ_d1ck_nid_no_patch); -int APS5_VABI sceIcuUnknown_TNZrG2iXikw_nid_no_patch() { - NotImplemented_nid_no_patch(__func__); -} -APS5_EXPORT("TNZrG2iXikw", sceIcuUnknown_TNZrG2iXikw_nid_no_patch); - int APS5_VABI u_strFoldCase_nid_postfix() { NotImplemented_nid_no_patch(__func__); } diff --git a/core/libs/prx/libSceIcu/Iterator.hpp b/core/libs/prx/libSceIcu/Iterator.hpp new file mode 100644 index 000000000000..ba2136bb31ea --- /dev/null +++ b/core/libs/prx/libSceIcu/Iterator.hpp @@ -0,0 +1,51 @@ +#pragma once +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include + +enum UCharIteratorOrigin : std::int32_t { + UITER_START, + UITER_CURRENT, + UITER_LIMIT, + UITER_ZERO, + UITER_LENGTH +}; + +struct UCharIterator { + using GetIndexFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator, UCharIteratorOrigin origin); + using MoveFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator, std::int32_t delta, UCharIteratorOrigin origin); + using HasNextFn = std::int8_t (APS5_VABI *)(UCharIterator* iterator); + using HasPreviousFn = std::int8_t (APS5_VABI *)(UCharIterator* iterator); + using CurrentFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator); + using NextFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator); + using PreviousFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator); + using ReservedFn = std::int32_t (APS5_VABI *)(UCharIterator* iterator, std::int32_t something); + using GetStateFn = std::uint32_t (APS5_VABI *)(const UCharIterator* iterator); + using SetStateFn = void (APS5_VABI *)(UCharIterator* iterator, std::uint32_t state, std::int32_t* errorCode); + + const void* context; + std::int32_t length; + std::int32_t start; + std::int32_t index; + std::int32_t limit; + std::int32_t reservedField; + GetIndexFn getIndex; + MoveFn move; + HasNextFn hasNext; + HasPreviousFn hasPrevious; + CurrentFn current; + NextFn next; + PreviousFn previous; + ReservedFn reservedFn; + GetStateFn getState; + SetStateFn setState; +}; + +static_assert(sizeof(UCharIteratorOrigin) == 4); +static_assert(offsetof(UCharIterator, getIndex) == 32); +static_assert(offsetof(UCharIterator, setState) == 104); +static_assert(sizeof(UCharIterator) == 112); + +extern "C" { +std::int32_t APS5_VABI u_strCompareIter_nid_postfix(UCharIterator* first, UCharIterator* second, std::uint8_t codePointOrder); +} diff --git a/core/libs/prx/libSceIcu/Version.cpp b/core/libs/prx/libSceIcu/Version.cpp index 5916903a8245..3c55a14e172c 100644 --- a/core/libs/prx/libSceIcu/Version.cpp +++ b/core/libs/prx/libSceIcu/Version.cpp @@ -7,6 +7,7 @@ namespace { constexpr std::uint16_t VersionLength = 4; constexpr std::int32_t VersionStringLength = 20; constexpr char VersionDelimiter = '.'; +constexpr std::uint8_t UnicodeVersion[VersionLength] = {17, 0, 0, 0}; char* WriteField(char* text, std::uint8_t field) { if (field >= 100) { @@ -66,4 +67,9 @@ void APS5_VABI u_versionToString_nid_postfix(const std::uint8_t* version, char* *text = 0; } +void APS5_VABI u_getUnicodeVersion_nid_postfix(std::uint8_t* version) { + if (!version) return; + for (std::uint16_t part = 0; part < VersionLength; ++part) version[part] = UnicodeVersion[part]; +} + } diff --git a/core/libs/prx/libSceIcu/Version.hpp b/core/libs/prx/libSceIcu/Version.hpp index 4963a77ed25f..7bfee06d7b09 100644 --- a/core/libs/prx/libSceIcu/Version.hpp +++ b/core/libs/prx/libSceIcu/Version.hpp @@ -6,4 +6,5 @@ extern "C" { void APS5_VABI u_versionFromString_nid_postfix(std::uint8_t*, const char*); void APS5_VABI u_versionFromUString_nid_postfix(std::uint8_t*, const char16_t*); void APS5_VABI u_versionToString_nid_postfix(const std::uint8_t*, char*); +void APS5_VABI u_getUnicodeVersion_nid_postfix(std::uint8_t*); } diff --git a/core/libs/prx/libSceIcu/libSceIcu.hpp b/core/libs/prx/libSceIcu/libSceIcu.hpp index 3a2edbaf7963..cc10c2ec4942 100644 --- a/core/libs/prx/libSceIcu/libSceIcu.hpp +++ b/core/libs/prx/libSceIcu/libSceIcu.hpp @@ -1,5 +1,6 @@ #pragma once #include "Invariant.hpp" +#include "Iterator.hpp" #include "Utf8.hpp" #include "Version.hpp" #include "ConverterCallbacks.hpp" diff --git a/core/libs/prx/libSceIcu/tests/CompareIter.cpp b/core/libs/prx/libSceIcu/tests/CompareIter.cpp new file mode 100644 index 000000000000..cf354c26a210 --- /dev/null +++ b/core/libs/prx/libSceIcu/tests/CompareIter.cpp @@ -0,0 +1,195 @@ +#include "prx/libSceIcu/libSceIcu.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, std::source_location location = std::source_location::current()) { + if (condition) return; + std::fprintf(stderr, "ICU iterator comparison check failed at line %u\n", location.line()); + std::abort(); +} + +struct Text { + std::vector units; + std::string calls; +}; + +Text& Of(UCharIterator* iterator) { + return *static_cast(const_cast(iterator->context)); +} + +std::int32_t APS5_VABI Move_nid_no_patch(UCharIterator* iterator, std::int32_t delta, UCharIteratorOrigin origin) { + auto& text = Of(iterator); + Require(origin == UITER_START); + text.calls += 'm'; + iterator->index = iterator->start + delta; + return iterator->index; +} + +std::int32_t APS5_VABI Next_nid_no_patch(UCharIterator* iterator) { + auto& text = Of(iterator); + text.calls += 'n'; + return iterator->index < iterator->limit ? text.units[iterator->index++] : -1; +} + +std::int32_t APS5_VABI Current_nid_no_patch(UCharIterator* iterator) { + auto& text = Of(iterator); + text.calls += 'c'; + return iterator->index < iterator->limit ? text.units[iterator->index] : -1; +} + +std::int32_t APS5_VABI Previous_nid_no_patch(UCharIterator* iterator) { + auto& text = Of(iterator); + text.calls += 'p'; + return iterator->index > iterator->start ? text.units[--iterator->index] : -1; +} + +struct Iterator { + Text text; + UCharIterator iterator{}; + + explicit Iterator(std::vector units, std::int32_t index = 0) : text{std::move(units), {}} { + iterator.context = &text; + iterator.length = static_cast(text.units.size()); + iterator.start = 0; + iterator.index = index; + iterator.limit = iterator.length; + iterator.move = Move_nid_no_patch; + iterator.next = Next_nid_no_patch; + iterator.current = Current_nid_no_patch; + iterator.previous = Previous_nid_no_patch; + } +}; + +struct Result { + std::int32_t value; + std::string first; + std::string second; +}; + +Result Compare(std::vector left, std::vector right, std::uint8_t codePointOrder) { + Iterator first(std::move(left), 1); + Iterator second(std::move(right), 1); + const auto value = u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, codePointOrder); + return {value, first.text.calls, second.text.calls}; +} + +void Expect(std::vector left, std::vector right, std::uint8_t codePointOrder, + std::int32_t value, const char* first, const char* second, std::source_location location = std::source_location::current()) { + const auto result = Compare(std::move(left), std::move(right), codePointOrder); + Require(result.value == value, location); + Require(result.first == first, location); + Require(result.second == second, location); +} + +template +void RequireInvalidArgument(TAction action, std::source_location location = std::source_location::current()) { + try { action(); } catch (const std::invalid_argument&) { return; } + Require(false, location); +} + +void Arguments() { + Iterator text({u'a'}); + Require(u_strCompareIter_nid_postfix(nullptr, &text.iterator, 1) == 0); + Require(u_strCompareIter_nid_postfix(&text.iterator, nullptr, 1) == 0); + Require(u_strCompareIter_nid_postfix(nullptr, nullptr, 0) == 0); + Require(u_strCompareIter_nid_postfix(&text.iterator, &text.iterator, 1) == 0); + Require(text.text.calls.empty()); +} + +void Units() { + Expect({}, {}, 0, 0, "mn", "mn"); + Expect({u'a', u'b'}, {u'a', u'b'}, 1, 0, "mnnn", "mnnn"); + Expect({u'a', u'b', u'c'}, {u'a', u'b', u'd'}, 0, -1, "mnnn", "mnnn"); + Expect({u'a', u'b'}, {u'a', u'b', u'c'}, 1, -1 - u'c', "mnnn", "mnnn"); + Expect({u'b'}, {u'a', u'z'}, 1, 1, "mn", "mn"); + Expect({0xff61}, {0xd800, 0xdc00}, 0, 0xff61 - 0xd800, "mn", "mn"); + Expect({0xdc00}, {0xd800, 0xdc00}, 0, 0x400, "mn", "mn"); +} + +void CodePoints() { + Expect({0xff61}, {0xd800, 0xdc00}, 1, 0xff61 - 0x2800 - 0xd800, "mn", "mnc"); + Expect({0xd800, 0xdc00}, {0xff61}, 0x80, 0xd800 - (0xff61 - 0x2800), "mnc", "mn"); + Expect({0xdc00}, {0xd800, 0xdc00}, 1, 0xdc00 - 0x2800 - 0xd800, "mnpp", "mnc"); + Expect({0xd800, 0xdc00}, {0xd800, 0xdc01}, 1, -1, "mnnpp", "mnnpp"); + Expect({u'x', 0xdc00}, {u'x', 0xe000}, 1, 0xdc00 - 0xe000, "mnnpp", "mnn"); + Expect({0xd800}, {0xe000}, 1, -0x800, "mnc", "mn"); + Expect({0xd800, u'a'}, {0xdbff, 0xdfff}, 1, 0xd800 - 0x2800 - 0xdbff, "mnc", "mnc"); + Expect({0xffff}, {0x10000}, 1, 0xffff - 0x2800 - (0x10000 - 0x2800), "mn", "mn"); + Expect({u'a', 0xd7ff}, {u'a', 0xd800}, 1, -1, "mnn", "mnn"); + Expect({0xd800}, {}, 1, 0xd801, "mn", "mn"); +} + +void Wrapping() { + Expect({std::numeric_limits::min()}, {1}, 0, std::numeric_limits::max(), "mn", "mn"); + Expect({std::numeric_limits::max()}, {-1}, 1, std::numeric_limits::min(), "mn", "mn"); +} + +void Callbacks() { + for (int missing = 0; missing < 4; ++missing) { + Iterator first({0xdc00}); + Iterator second({0xd800, 0xdc00}); + auto& target = missing < 2 ? first.iterator : second.iterator; + if (missing % 2 == 0) target.move = nullptr; else target.next = nullptr; + RequireInvalidArgument([&] { u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1); }); + } + { + Iterator first({0xdc00}); + Iterator second({0xd800, 0xdc00}); + first.iterator.previous = nullptr; + RequireInvalidArgument([&] { u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1); }); + } + { + Iterator first({0xdc00}); + Iterator second({0xd800, 0xdc00}); + second.iterator.current = nullptr; + RequireInvalidArgument([&] { u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1); }); + } + { + Iterator first({0xdc00}); + Iterator second({0xd800, 0xdc00}); + first.iterator.current = nullptr; + second.iterator.previous = nullptr; + Require(u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1) == 0xdc00 - 0x2800 - 0xd800); + Require(first.text.calls == "mnpp"); + Require(second.text.calls == "mnc"); + } + { + Iterator first({u'a'}); + Iterator second({u'b'}); + first.iterator.current = nullptr; + first.iterator.previous = nullptr; + second.iterator.current = nullptr; + second.iterator.previous = nullptr; + Require(u_strCompareIter_nid_postfix(&first.iterator, &second.iterator, 1) == -1); + } +} + +void UnicodeVersion() { + std::array version{}; + version.fill(0xa5); + u_getUnicodeVersion_nid_postfix(version.data()); + Require(version == std::array{17, 0, 0, 0, 0xa5, 0xa5}); + u_getUnicodeVersion_nid_postfix(nullptr); +} + +} + +int main() { + Arguments(); + Units(); + CodePoints(); + Wrapping(); + Callbacks(); + UnicodeVersion(); + return 0; +} diff --git a/docs/dev/debt/libSceIcu.md b/docs/dev/debt/libSceIcu.md index 5cf39df1f216..55dbf1a6acfa 100644 --- a/docs/dev/debt/libSceIcu.md +++ b/docs/dev/debt/libSceIcu.md @@ -18,6 +18,8 @@ - [Invariant character and version helpers](../../../core/libs/prx/libSceIcu/Version.cpp) are unverified on a console. `u_charsToUChars`, `u_UCharsToChars`, `u_versionFromString`, `u_versionFromUString` and `u_versionToString` follow ICU 78.1 [uinvchar.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/uinvchar.cpp) and [putil.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/putil.cpp), unchanged since 65.1. Firmware 13.42 reports ICU 78.1 and exports none of them ([#11013](https://github.com/boykopovar/AnyPS5/pull/11013#issuecomment-6400470698)); the [3.20 table](https://github.com/DNNDHH/PS5-3.20_Libs/blob/b2a77c09cd71c01f34c790cfbe98c751a13262ad/libSceIcu.c) has all five. Version parts use the guest `strtoul`, truncated to eight bits; `u_versionToString` keeps ICU's dropped zero tens digit (105 becomes `15`). A null buffer with a positive conversion length throws; ICU dereferences it. Closes when: compared with the console. +- [Iterator comparison and Unicode version](../../../core/libs/prx/libSceIcu/CompareIter.cpp) are unverified on a console. `u_strCompareIter` and `u_getUnicodeVersion` follow ICU 78.1 [ustring.cpp](https://github.com/unicode-org/icu/blob/049e0d6a4206/icu4c/source/common/ustring.cpp#L828-L882) and [uchar.cpp](https://github.com/unicode-org/icu/blob/049e0d6a4206/icu4c/source/common/uchar.cpp#L484-L489); the version is 17.0, the Unicode data of 78.1, whatever firmware is emulated. A null iterator callback that ICU would call throws `std::invalid_argument`; ICU dereferences it. The code unit difference wraps instead of overflowing. The `UBool` is read as one byte, nonzero meaning true. Closes when: compared with the console. + - [Default-converter string copy helpers](../../../core/libs/prx/libSceIcu/Invariant.cpp) `u_austrcpy` and `u_austrncpy` are unverified on a console and throw `NotImplemented_nid_no_patch` on code units above `0x7f`. Their names and NIDs come from the PS5 3.20 inventory; signatures and ASCII (`0x01`–`0x7f`) behavior follow ICU 78.1's [ustr_cnv.cpp](https://github.com/unicode-org/icu/blob/release-78.1/icu4c/source/common/ustr_cnv.cpp), unchanged since 65.1, without assuming the console's default converter for non-ASCII units. `u_austrncpy` writes a single NUL when the source ends before the count without zero-padding the remainder. Null buffers with `u_austrcpy` or a positive count throw an invalid-argument exception, whereas ICU dereferences in `u_austrcpy` and returns the destination unchanged in `u_austrncpy`. Closes when: compared with the console, including the default converter for non-ASCII units. ### Known export names; ABI prototypes unverified