fix(libc): sort with FreeBSD 11.4's merge.c in mergesort

core/libs/prx/libc/CMakeLists.txt

@@ -78,6 +78,7 @@ target_sources(libc PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/src/TimeExtras.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/AtomicBuiltins.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/FileAndHeap.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/src/Mergesort.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/HostErrno.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/FileLock.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/src/StdioExtras.cpp

core/libs/prx/libc/src/FileAndHeap.cpp

@@ -727,40 +727,6 @@ void* APS5_VABI bsearch_nid_postfix(const void* key, const void* base, size_t co
     return nullptr;
 }
 
-int APS5_VABI mergesort_nid_postfix(void* base, size_t count, size_t size, GuestCompare compare) {
-    if (size < 4) {
-        errno = 22;
-        return -1;
-    }
-    if (count == 0) return 0;
-    if (!compare) throw std::invalid_argument("mergesort: null comparator");
-    if (!base) throw std::invalid_argument("mergesort: null base");
-    if (count > (std::numeric_limits<size_t>::max() - 8) / size) throw std::overflow_error("mergesort: array size overflow");
-    auto* const scratch = static_cast<unsigned char*>(malloc_nid_postfix(count * size + 8));
-    if (!scratch) return -1;
-    auto* source = static_cast<unsigned char*>(base);
-    auto* target = scratch;
-    for (size_t width = 1; width < count; width *= 2) {
-        for (size_t low = 0; low < count; low += 2 * width) {
-            const size_t middle = std::min(low + width, count);
-            const size_t high = std::min(low + 2 * width, count);
-            size_t left = low, right = middle, out = low;
-            while (left < middle && right < high) {
-                const size_t taken = compare(source + left * size, source + right * size) <= 0 ? left++ : right++;
-                std::memcpy(target + out++ * size, source + taken * size, size);
-            }
-            std::memcpy(target + out * size, source + left * size, (middle - left) * size);
-            out += middle - left;
-            std::memcpy(target + out * size, source + right * size, (high - right) * size);
-        }
-        std::swap(source, target);
-    }
-    if (source != base) std::memcpy(base, source, count * size);
-    free_nid_postfix(scratch);
-
-    return 0;
-}
-
 int APS5_VABI heapsort_nid_postfix(void* base, size_t count, size_t size, GuestCompare compare) {
     if (count <= 1) return 0;
     if (size == 0) {

core/libs/prx/libc/src/Mergesort.cpp

@@ -0,0 +1,241 @@
+#include "prx/libc/include/Api.hpp"
+#include <algorithm>
+#include <cerrno>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+
+namespace {
+
+constexpr std::size_t LinkSize = sizeof(unsigned char*);
+constexpr std::size_t NaturalRunThreshold = 16;
+constexpr std::size_t GallopAfter = 6;
+
+unsigned char* LinkSlot(unsigned char* position) {
+    const auto address = reinterpret_cast<std::uintptr_t>(position);
+    return reinterpret_cast<unsigned char*>((address + LinkSize - 1) & ~static_cast<std::uintptr_t>(LinkSize - 1));
+}
+
+unsigned char* LoadLink(unsigned char* position) {
+    unsigned char* link;
+    std::memcpy(&link, LinkSlot(position), sizeof(link));
+    return link;
+}
+
+void StoreLink(unsigned char* position, unsigned char* link) {
+    std::memcpy(LinkSlot(position), &link, sizeof(link));
+}
+
+class MergeSorter {
+public:
+    MergeSorter(std::size_t size, GuestCompare compare) : size(size), compare(compare) {}
+
+    unsigned char* Sort(unsigned char* base, std::size_t count, unsigned char* scratch) {
+        unsigned char* list1 = base;
+        unsigned char* list2 = scratch;
+        Setup(list1, list2, count);
+        unsigned char* last = list2 + count * size;
+        while (LoadLink(list2) != last) {
+            MergePass(list1, list2, last);
+            std::swap(list1, list2);
+            last = list2 + count * size;
+        }
+        if (list2 == base) {
+            std::memmove(list2, list1, count * size);
+            list2 = list1;
+        }
+        return list2;
+    }
+
+private:
+    int Compare(const unsigned char* left, const unsigned char* right) const {
+        return compare(left, right);
+    }
+
+    void SwapElements(unsigned char* left, unsigned char* right) const {
+        std::swap_ranges(left, left + size, right);
+    }
+
+    void Reverse(unsigned char* bottom, unsigned char* top) const {
+        do {
+            SwapElements(bottom, top);
+            bottom += size;
+            top -= size;
+        } while (bottom < top);
+    }
+
+    void InsertionSort(unsigned char* first, std::size_t count) const {
+        for (unsigned char* inserted = first + size; --count >= 1; inserted += size) {
+            for (unsigned char* current = inserted; current > first; current -= size) {
+                unsigned char* previous = current - size;
+                if (Compare(previous, current) <= 0) break;
+                SwapElements(previous, current);
+            }
+        }
+    }
+
+    static void Link(unsigned char*& tail, unsigned char* next) {
+        StoreLink(tail, next);
+        tail = next;
+    }
+
+    void Setup(unsigned char* list1, unsigned char* list2, std::size_t count) const {
+        const std::size_t pair = size * 2;
+        if (count <= 5) {
+            InsertionSort(list1, count);
+            StoreLink(list2, list2 + count * size);
+            return;
+        }
+        const std::size_t tailCount = 4 + (count & 1);
+        InsertionSort(list1 + (count - tailCount) * size, tailCount);
+        unsigned char* const last = list1 + size * (count - tailCount);
+        StoreLink(list2 + (last - list1), list2 + count * size);
+        unsigned char* tail = list2;
+        unsigned char* first = list1;
+        bool descending = Compare(first, first + size) > 0;
+        for (; first < last; descending = !descending) {
+            std::size_t length = 2;
+            unsigned char* next = first + pair;
+            for (; next < last; next += pair) {
+                if ((Compare(next, next + size) > 0) != descending) break;
+                length += 2;
+            }
+            if (length < NaturalRunThreshold) {
+                do {
+                    Link(tail, list2 + (first + pair - list1));
+                    if (descending) SwapElements(first, first + size);
+                } while ((first += pair) < next);
+                continue;
+            }
+            unsigned char* const end = next;
+            for (next = first + pair; next < end; next += pair) {
+                if ((Compare(next - size, next) > 0) != descending) {
+                    Link(tail, list2 + (next - list1));
+                    if (descending) Reverse(first, next - size);
+                    first = next;
+                }
+            }
+            if (descending) Reverse(first, next - size);
+            first = next;
+            if (next < last || Compare(next - size, next) > 0) Link(tail, list2 + (next - list1));
+            else Link(tail, list2 + count * size);
+        }
+    }
+
+    void Gallop(const unsigned char* key, unsigned char*& low, unsigned char*& high, int sense, bool& galloping) const {
+        std::size_t step = size;
+        for (;; step <<= 1) {
+            if (step >= static_cast<std::size_t>(high - low)) {
+                unsigned char* const probe = high - size;
+                if (probe > low && Compare(key, probe) <= sense) high = probe;
+                else low = probe;
+                while (high > low + size) {
+                    unsigned char* const middle = low + (((high - low) / size) >> 1) * size;
+                    if (Compare(key, middle) <= sense) high = middle;
+                    else low = middle;
+                }
+                return;
+            }
+            unsigned char* const probe = low + step;
+            if (Compare(key, probe) <= sense) {
+                high = probe;
+                if (step == size) galloping = false;
+                break;
+            }
+            low = probe;
+        }
+        while (step > size) {
+            step >>= 1;
+            unsigned char* const probe = low + step;
+            if (Compare(key, probe) <= sense) high = probe;
+            else low = probe;
+        }
+    }
+
+    void MergePass(unsigned char* list1, unsigned char* list2, unsigned char* last) {
+        unsigned char* end2 = list1;
+        unsigned char* runLink = list1;
+        unsigned char* target = list2;
+        auto copyRun = [&target](unsigned char*& from, const unsigned char* to) {
+            const auto bytes = static_cast<std::size_t>(to - from);
+            std::memcpy(target, from, bytes);
+            target += bytes;
+            from += bytes;
+        };
+        for (unsigned char* link = list2; link != last; runLink = end2) {
+            link = LoadLink(link);
+            unsigned char* first1 = end2;
+            unsigned char* const end1 = list1 + (link - list2);
+            unsigned char* first2 = end1;
+            if (link != last) link = LoadLink(link);
+            end2 = list1 + (link - list2);
+            while (first1 < end1 && first2 < end2) {
+                const unsigned char* key;
+                unsigned char* low;
+                unsigned char* high;
+                int sense;
+                if (Compare(first1, first2) <= 0) {
+                    key = first2;
+                    low = first1;
+                    high = end1;
+                    sense = -1;
+                } else {
+                    key = first1;
+                    low = first2;
+                    high = end2;
+                    sense = 0;
+                }
+                bool gallop = galloping;
+                if (!galloping) {
+                    while ((low += size) < high && Compare(key, low) > sense) {
+                        if (++linearWins == GallopAfter) {
+                            galloping = true;
+                            gallop = true;
+                            break;
+                        }
+                    }
+                }
+                if (gallop) {
+                    Gallop(key, low, high, sense, galloping);
+                    low = high;
+                }
+                linearWins = 0;
+                if (key == first1) {
+                    copyRun(first2, low);
+                    copyRun(first1, first1 + size);
+                } else {
+                    copyRun(first1, low);
+                    copyRun(first2, first2 + size);
+                }
+            }
+            if (first2 < end2) copyRun(first2, end2);
+            else if (first1 < end1) copyRun(first1, end1);
+            StoreLink(runLink, end2);
+        }
+    }
+
+    std::size_t size;
+    GuestCompare compare;
+    std::size_t linearWins = 0;
+    bool galloping = false;
+};
+
+}
+
+int APS5_VABI mergesort_nid_postfix(void* base, size_t count, size_t size, GuestCompare compare) {
+    if (size < LinkSize / 2) {
+        errno = 22;
+        return -1;
+    }
+    if (count == 0) return 0;
+    if (!compare) throw std::invalid_argument("mergesort: null comparator");
+    if (!base) throw std::invalid_argument("mergesort: null base");
+    if (count > (std::numeric_limits<size_t>::max() - LinkSize) / size) throw std::overflow_error("mergesort: array size overflow");
+    auto* const scratch = static_cast<unsigned char*>(malloc_nid_postfix(count * size + LinkSize));
+    if (!scratch) return -1;
+    MergeSorter sorter(size, compare);
+    free_nid_postfix(sorter.Sort(static_cast<unsigned char*>(base), count, scratch));
+    return 0;
+}

core/libs/tests/GuestQsort.cpp

@@ -211,6 +211,79 @@ bool CheckMergesort() {
     return correct;
 }
 
+std::vector<int> mergesortCalls;
+
+int APS5_VABI RecordKeys(const void* left, const void* right) {
+    const auto* first = static_cast<const Element*>(left);
+    const auto* second = static_cast<const Element*>(right);
+    mergesortCalls.push_back(first->tag * 100 + second->tag);
+    return (first->key > second->key) - (first->key < second->key);
+}
+
+bool CheckMergesortComparisons() {
+    InitializeHeap();
+    struct Case {
+        std::vector<int> keys;
+        std::vector<int> calls;
+    };
+    const std::vector<Case> cases = {
+        {{3, 1, 2, 5, 4}, {1, 2, 102, 3, 304, 4}},
+        {{9, 4, 7, 1, 8, 2, 6, 3, 5, 0, 9, 4, 7, 1, 8, 2, 6, 3, 5, 0, 11},
+         {1617, 1618, 1718, 1619, 1819, 1719, 1620, 1, 203, 405, 607, 809, 1011, 1213, 1415, 102, 304, 506, 708, 910, 1112, 1314, 1419, 103, 102, 2, 507, 704,
+          406, 911, 1108, 810, 1315, 1512, 1214, 305, 501, 107, 106, 206, 204, 4, 913, 1311, 1115, 1112, 812, 1210, 1014, 309, 313, 513, 515, 715, 711, 111,
+          1106, 608, 612, 212, 1204, 414, 1400, 10, 919, 1903, 317, 1713, 1705, 1715, 1707, 1701, 118, 1811, 1808, 1806, 616, 1602, 220, 2012, 2004, 2014, 2000,
+          2010}},
+        {{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 0, 0, 30, 29, 28, 27},
+         {2223, 2224, 2324, 2225, 2325, 2425, 1, 203, 405, 607, 809, 1011, 1213, 1415, 1617, 1819, 2021, 102, 304, 506, 708, 910, 1112, 1314, 1516, 1718,
+          1920, 2125, 20, 21, 25, 125, 2502, 2503, 2504, 2505, 2506, 2507, 2508, 2510, 2514, 2519}},
+        {{40, 41, 42, 43, 44, 45, 46, 47, 0, 1, 2, 3, 4, 5, 6, 7, 48, 49, 50, 51, 52, 53, 54, 55, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17},
+         {3031, 3132, 3233, 1, 203, 405, 607, 809, 1011, 1213, 1415, 1617, 1819, 2021, 2223, 2425, 2627, 2829, 102, 304, 506, 708, 910, 1112, 1314, 1516, 1718,
+          1920, 2122, 2324, 2526, 2728, 2930, 8, 9, 10, 11, 12, 13, 14, 15, 17, 16, 116, 1602, 1604, 1607, 824, 2409, 2411, 2415, 2407, 2403, 2401, 2400, 25, 26,
+          28, 32, 33}},
+    };
+    bool correct = true;
+    for (const auto& sample : cases) {
+        std::vector<Element> elements;
+        for (std::size_t index = 0; index < sample.keys.size(); ++index) elements.push_back({sample.keys[index], static_cast<int>(index)});
+        mergesortCalls.clear();
+        correct &= mergesort_nid_postfix(elements.data(), elements.size(), sizeof(Element), RecordKeys) == 0;
+        if (mergesortCalls != sample.calls) {
+            std::fprintf(stderr, "mergesort of %zu elements made %zu comparisons in a different order than FreeBSD's merge.c (%zu expected)\n",
+                sample.keys.size(), mergesortCalls.size(), sample.calls.size());
+            correct = false;
+        }
+        for (std::size_t index = 1; index < elements.size(); ++index) {
+            correct &= elements[index - 1].key < elements[index].key ||
+                (elements[index - 1].key == elements[index].key && elements[index - 1].tag < elements[index].tag);
+        }
+    }
+    return correct;
+}
+
+bool CheckMergesortUnalignedElements() {
+    InitializeHeap();
+    constexpr std::size_t Size = 12;
+    constexpr std::size_t Count = 1000;
+    std::vector<unsigned char> storage(Count * Size + 8);
+    unsigned char* const base = storage.data() + 4;
+    for (std::size_t index = 0; index < Count; ++index) {
+        const Element element{static_cast<int>((index / 50) % 2 ? Count - index : (index * 7919) % 13), static_cast<int>(index)};
+        std::memcpy(base + index * Size, &element, sizeof(element));
+        std::memset(base + index * Size + sizeof(element), static_cast<int>(index & 0xff), Size - sizeof(element));
+    }
+    bool correct = mergesort_nid_postfix(base, Count, Size, CompareKeys) == 0;
+    Element previous{};
+    for (std::size_t index = 0; index < Count; ++index) {
+        Element element;
+        std::memcpy(&element, base + index * Size, sizeof(element));
+        correct &= base[index * Size + sizeof(element)] == static_cast<unsigned char>(element.tag & 0xff);
+        if (index != 0) correct &= previous.key < element.key || (previous.key == element.key && previous.tag < element.tag);
+        previous = element;
+    }
+    if (!correct) std::fprintf(stderr, "mergesort of unaligned 12-byte elements failed\n");
+    return correct;
+}
+
 bool CheckHeapsort() {
     bool correct = true;
     for (const std::size_t count : {0, 1, 2, 3, 7, 16, 33, 1000}) {
@@ -252,6 +325,8 @@ int main() {
         for (const std::size_t count : {0, 1, 2, 6, 7, 8, 40, 41, 1000})
             correct &= ExpectSorted(size, count);
     correct &= CheckMergesort();
+    correct &= CheckMergesortComparisons();
+    correct &= CheckMergesortUnalignedElements();
     correct &= CheckHeapsort();
     return correct ? 0 : 1;
 }