perf(agc): read committed guest pages directly on Windows

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -2775,6 +2775,7 @@ void visitReadablePages(std::byte* destination, std::uint64_t source, std::size_
 
 void copyCommittedBytes(std::byte* destination, std::uint64_t source, std::size_t bytes) {
 #ifdef _WIN32
+    if (GuestArena::GuestArenaCopyOwned_nid_no_patch(source, destination, bytes)) return;
     visitReadablePages(destination, source, bytes, [&](std::size_t offset, std::size_t count, bool read) {
         if (!read) std::memset(destination + offset, 0, count);
     });
@@ -2785,6 +2786,8 @@ void copyCommittedBytes(std::byte* destination, std::uint64_t source, std::size_
 
 bool equalsCommittedBytes(std::uint64_t address, const std::byte* expected, std::size_t bytes) {
 #ifdef _WIN32
+    bool same = false;
+    if (GuestArena::GuestArenaCompareOwned_nid_no_patch(address, expected, bytes, &same)) return same;
     constexpr std::size_t ChunkBytes = 64 * PageBytes;
     struct CompareChunkStorage {};
     auto& current = HostThreadLocal<std::vector<std::byte>, CompareChunkStorage>();

core/libs/prx/libSceAgcDriver/tests/execution/CommittedReadMapping.cpp

@@ -9,6 +9,7 @@
 #include <cstring>
 #include <stdexcept>
 #include <thread>
+#include <vector>
 #ifdef _WIN32
 #include <windows.h>
 #endif
@@ -42,6 +43,7 @@ void Run() {
         prime();
         DWORD previous = 0;
         Require(VirtualProtect(source + 2 * page, page, protection, &previous), "cannot protect snapshot hole");
+        if ((protection & PAGE_GUARD) != 0) { GuestAllocations::Mutation mutation; mutation.Protect(source + 2 * page, page, true, true, false, [] {}); }
         snapshot.fill(std::byte{0xff});
         AgcDriver::GuestMemory::ReadCommitted(address, snapshot);
         check(true);
@@ -55,6 +57,7 @@ void Run() {
         check(false);
         prime();
         Require(VirtualProtect(source + 2 * page, page, protection, &previous), "cannot protect comparison hole");
+        if ((protection & PAGE_GUARD) != 0) { GuestAllocations::Mutation mutation; mutation.Protect(source + 2 * page, page, true, true, false, [] {}); }
         std::fill(snapshot.begin() + 2 * page, snapshot.begin() + 3 * page, std::byte{0x77});
         Require(AgcDriver::GuestMemory::EqualsCommitted(address, snapshot), "committed comparison did not skip an inaccessible page");
         snapshot[page - 1] = std::byte{};
@@ -92,6 +95,126 @@ void Run() {
     GuestArena::GuestArenaReset_nid_postfix(source, bytes);
     GuestArena::GuestArenaRelease_nid_postfix(source, bytes);
 }
+
+void RunReleasedPages() {
+    constexpr std::size_t bytes = 65536;
+    constexpr std::size_t page = 4096;
+    constexpr std::size_t middle = 5 * page;
+    constexpr std::size_t last = bytes - page;
+    auto* source = static_cast<std::byte*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes));
+    Require(source != nullptr, "cannot reserve released page fixture");
+    GuestArena::GuestArenaCommit_nid_postfix(source, bytes, PAGE_READWRITE, bytes);
+    const auto address = reinterpret_cast<std::uintptr_t>(source);
+    { GuestAllocations::Mutation mutation; mutation.Add(source, bytes, true, true, false); }
+    const auto release = [&](std::size_t offset) { Require(VirtualFree(source + offset, page, MEM_DECOMMIT), "cannot release fixture page"); };
+    const auto restore = [&](std::size_t offset) { Require(VirtualAlloc(source + offset, page, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot restore fixture page"); };
+    std::vector<std::byte> pattern(bytes);
+    for (std::size_t index = 0; index < bytes; ++index) pattern[index] = static_cast<std::byte>(index * 31 >> 3);
+    const auto prepare = [&] {
+        restore(middle);
+        restore(last);
+        std::memcpy(source, pattern.data(), bytes);
+        { GuestAllocations::Mutation mutation; mutation.Protect(source, bytes, true, true, false, [] {}); }
+        Require(AgcDriver::GuestMemory::Accessible(source, bytes), "released page fixture is not readable");
+        release(middle);
+        release(last);
+    };
+    const auto hole = [&](std::size_t index) { return (index >= middle && index < middle + page) || index >= last; };
+
+    prepare();
+    std::vector<std::byte> copy(bytes, std::byte{0xff});
+    Require(!GuestArena::GuestArenaCopyOwned_nid_no_patch(address, copy.data(), bytes), "direct copy did not report a released page");
+    bool equal = true;
+    Require(!GuestArena::GuestArenaCompareOwned_nid_no_patch(address, pattern.data(), bytes, &equal), "direct comparison did not report a released page");
+    std::fill(copy.begin(), copy.end(), std::byte{0xff});
+    AgcDriver::GuestMemory::ReadCommitted(address, copy);
+    for (std::size_t index = 0; index < bytes; ++index) Require(copy[index] == (hole(index) ? std::byte{} : pattern[index]), "released pages were not zero in a copy, or readable pages were lost");
+    Require(!AgcDriver::GuestMemory::Accessible(source, bytes), "a released page kept its readable state");
+
+    prepare();
+    auto expected = pattern;
+    for (std::size_t index = 0; index < bytes; ++index) {
+        if (hole(index)) expected[index] = std::byte{0x77};
+    }
+    Require(AgcDriver::GuestMemory::EqualsCommittedUnsynced(address, expected), "comparison did not leave out released pages");
+    for (const std::size_t mismatch : {page - 1, middle - 1, middle + page, last - 1}) {
+        prepare();
+        expected[mismatch] ^= std::byte{1};
+        Require(!AgcDriver::GuestMemory::EqualsCommittedUnsynced(address, expected), "comparison missed a difference beside a released page");
+        expected[mismatch] ^= std::byte{1};
+    }
+
+    for (std::size_t before = 1; before < 130; before += 3) {
+        for (std::size_t after = 1; after < 40; after += 7) {
+            prepare();
+            std::vector<std::byte> straddle(before + after, std::byte{0xff});
+            AgcDriver::GuestMemory::ReadCommitted(address + middle - before, straddle);
+            for (std::size_t index = 0; index < before; ++index) Require(straddle[index] == pattern[middle - before + index], "straddling copy lost bytes before a released page");
+            for (std::size_t index = before; index < straddle.size(); ++index) Require(straddle[index] == std::byte{}, "straddling copy did not zero a released page");
+            prepare();
+            std::vector<std::byte> straddleExpected(pattern.data() + middle - before, pattern.data() + middle + after);
+            std::fill(straddleExpected.begin() + before, straddleExpected.end(), std::byte{0x77});
+            Require(AgcDriver::GuestMemory::EqualsCommittedUnsynced(address + middle - before, straddleExpected), "straddling comparison did not leave out a released page");
+        }
+    }
+
+    prepare();
+    for (std::size_t length = 1; length < 80; ++length) {
+        std::vector<std::byte> edge(length);
+        const auto first = address + middle - length;
+        Require(GuestArena::GuestArenaCopyOwned_nid_no_patch(first, edge.data(), length), "direct copy read past the end of its range");
+        Require(std::memcmp(edge.data(), pattern.data() + middle - length, length) == 0, "direct copy returned wrong bytes");
+        bool same = false;
+        Require(GuestArena::GuestArenaCompareOwned_nid_no_patch(first, edge.data(), length, &same) && same, "direct comparison read past the end of its range");
+        edge[length - 1] ^= std::byte{1};
+        Require(GuestArena::GuestArenaCompareOwned_nid_no_patch(first, edge.data(), length, &same) && !same, "direct comparison missed a last-byte difference");
+    }
+
+    auto* host = static_cast<std::byte*>(VirtualAlloc(nullptr, 3 * page, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+    Require(host != nullptr, "cannot allocate host pages");
+    std::memset(host, 0x5a, 3 * page);
+    DWORD previous = 0;
+    Require(VirtualProtect(host + page, page, PAGE_READWRITE | PAGE_GUARD, &previous), "cannot guard host page");
+    std::vector<std::byte> outside(3 * page, std::byte{0xff});
+    Require(!GuestArena::GuestArenaCopyOwned_nid_no_patch(reinterpret_cast<std::uintptr_t>(host), outside.data(), outside.size()), "host pages were read directly");
+    AgcDriver::GuestMemory::ReadCommitted(reinterpret_cast<std::uintptr_t>(host), outside);
+    for (std::size_t index = 0; index < outside.size(); ++index) Require(outside[index] == (index >= page && index < 2 * page ? std::byte{} : std::byte{0x5a}), "host guard page read changed");
+    MEMORY_BASIC_INFORMATION memory{};
+    Require(VirtualQuery(host + page, &memory, sizeof(memory)) == sizeof(memory) && memory.Protect == (PAGE_READWRITE | PAGE_GUARD), "host guard page protection changed");
+    Require(VirtualFree(host, 0, MEM_RELEASE), "cannot release host pages");
+
+    prepare();
+    restore(last);
+    std::memcpy(source + last, pattern.data() + last, page);
+    expected = pattern;
+    std::fill(expected.begin() + middle, expected.begin() + middle + page, std::byte{});
+    std::atomic<bool> done{false};
+    std::thread mutation([&] {
+        while (!done.load(std::memory_order_acquire)) {
+            VirtualFree(source + middle, page, MEM_DECOMMIT);
+            VirtualAlloc(source + middle, page, MEM_COMMIT, PAGE_READWRITE);
+        }
+    });
+    bool stable = true;
+    try {
+        for (unsigned iteration = 0; iteration < 2000; ++iteration) {
+            AgcDriver::GuestMemory::ReadCommittedUnsynced(address, copy);
+            stable = stable && std::memcmp(copy.data(), pattern.data(), middle) == 0 && std::memcmp(copy.data() + middle + page, pattern.data() + middle + page, bytes - middle - page) == 0;
+            stable = stable && AgcDriver::GuestMemory::EqualsCommittedUnsynced(address, expected);
+        }
+    } catch (...) {
+        done.store(true, std::memory_order_release);
+        mutation.join();
+        throw;
+    }
+    done.store(true, std::memory_order_release);
+    mutation.join();
+    Require(stable, "concurrent decommit changed the result of a read");
+    restore(middle);
+    { GuestAllocations::Mutation mutation; mutation.Remove(source); }
+    GuestArena::GuestArenaReset_nid_postfix(source, bytes);
+    GuestArena::GuestArenaRelease_nid_postfix(source, bytes);
+}
 #endif
 }
 
@@ -101,6 +224,7 @@ int main() {
         auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
         Run();
+        RunReleasedPages();
         device->WaitIdle();
         std::puts("committed snapshot mapping changes passed");
         return 0;

core/libs/prx/libc/CMakeLists.txt

@@ -1499,6 +1499,12 @@ if(MINGW)
         set_tests_properties(windows_backtrace_report PROPERTIES TIMEOUT 20
             ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>"
             PASS_REGULAR_EXPRESSION "aborting[^~]*SIGABRT. on thread[^~]*stack return address candidates:[^~]*rsp[+]0x")
+        foreach(guardedFault IN ITEMS destination expected outside)
+            add_test(NAME windows_guarded_read_wild_${guardedFault} COMMAND windows_backtrace_report_tests guarded-${guardedFault})
+            set_tests_properties(windows_guarded_read_wild_${guardedFault} PROPERTIES TIMEOUT 20
+                ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>"
+                PASS_REGULAR_EXPRESSION "FATAL: unhandled exception 0xc0000005")
+        endforeach()
         add_test(NAME windows_backtrace_failure_report COMMAND windows_backtrace_report_tests report)
         set_tests_properties(windows_backtrace_failure_report PROPERTIES TIMEOUT 20
             ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>"

core/libs/prx/libc/include/GuestArena.hpp

@@ -4,6 +4,10 @@
 #include <cstddef>
 #include <cstdint>
 
+#ifdef _WIN32
+struct _EXCEPTION_POINTERS;
+#endif
+
 // Guest virtual memory is placed inside one reserved arena below the PS5 application map limit
 // absolute address and breaks on host addresses outside that range.
 namespace GuestArena {
@@ -40,6 +44,9 @@ void GuestArenaSetAccessObserver_nid_postfix(void (*callback)(std::uintptr_t add
 #ifdef _WIN32
 bool GuestArenaHandleWrite_nid_postfix(std::uintptr_t address);
 bool GuestArenaHandleRead_nid_postfix(std::uintptr_t address);
+bool GuestArenaCopyOwned_nid_no_patch(std::uintptr_t source, void* destination, std::size_t bytes);
+bool GuestArenaCompareOwned_nid_no_patch(std::uintptr_t source, const void* expected, std::size_t bytes, bool* equal);
+bool GuestArenaRecoverRead_nid_no_patch(_EXCEPTION_POINTERS* exception);
 void GuestArenaPinWritable_nid_postfix(const void* pointer, std::size_t bytes);
 void GuestArenaUnpinWritable_nid_postfix(const void* pointer, std::size_t bytes);
 bool GuestArenaProtection_nid_postfix(std::uintptr_t address, std::uint32_t* protection);

core/libs/prx/libc/src/GuestArena.cpp

@@ -493,6 +493,171 @@ std::invalid_argument OutsideArena(const char* operation, const void* pointer, s
 }
 
 #ifdef _WIN32
+extern "C" {
+int GuardedCopy(std::uintptr_t source, void* destination, std::size_t bytes);
+int GuardedCompare(std::uintptr_t source, const void* expected, std::size_t bytes);
+int GuardedCopyAvx2(std::uintptr_t source, void* destination, std::size_t bytes);
+int GuardedCompareAvx2(std::uintptr_t source, const void* expected, std::size_t bytes);
+extern const char GuardedCopyAvx2Load[];
+extern const char GuardedCopyAvx2HighLoad[];
+extern const char GuardedCompareAvx2Load[];
+extern const char GuardedCompareAvx2HighLoad[];
+extern const char GuardedCopyWideLoad[];
+extern const char GuardedCopyByteLoad[];
+extern const char GuardedCompareWideLoad[];
+extern const char GuardedCompareByteLoad[];
+extern const char GuardedReadFailed[];
+extern const char GuardedReadFailedAvx2[];
+}
+
+asm(".text\n"
+    ".seh_proc GuardedCopy\n"
+    "GuardedCopy:\n"
+    ".seh_endprologue\n"
+    "    leaq (%rcx,%r8), %r9\n"
+    "    subq %rcx, %rdx\n"
+    "    jmp 2f\n"
+    "GuardedCopyWideLoad:\n"
+    "1:  movdqu (%rcx), %xmm0\n"
+    "    movdqu %xmm0, (%rcx,%rdx)\n"
+    "    addq $16, %rcx\n"
+    "GuardedCopyTail:\n"
+    "2:  leaq 16(%rcx), %rax\n"
+    "    cmpq %r9, %rax\n"
+    "    jbe 1b\n"
+    "    jmp 4f\n"
+    "GuardedCopyByteLoad:\n"
+    "3:  movzbl (%rcx), %eax\n"
+    "    movb %al, (%rcx,%rdx)\n"
+    "    incq %rcx\n"
+    "4:  cmpq %r9, %rcx\n"
+    "    jb 3b\n"
+    "    xorl %eax, %eax\n"
+    "    ret\n"
+    ".seh_endproc\n"
+    ".seh_proc GuardedCompare\n"
+    "GuardedCompare:\n"
+    ".seh_endprologue\n"
+    "    leaq (%rcx,%r8), %r9\n"
+    "    subq %rcx, %rdx\n"
+    "    jmp 2f\n"
+    "GuardedCompareWideLoad:\n"
+    "1:  movdqu (%rcx), %xmm0\n"
+    "    movdqu (%rcx,%rdx), %xmm1\n"
+    "    pcmpeqb %xmm1, %xmm0\n"
+    "    pmovmskb %xmm0, %eax\n"
+    "    cmpl $0xffff, %eax\n"
+    "    jne 5f\n"
+    "    addq $16, %rcx\n"
+    "GuardedCompareTail:\n"
+    "2:  leaq 16(%rcx), %rax\n"
+    "    cmpq %r9, %rax\n"
+    "    jbe 1b\n"
+    "    jmp 4f\n"
+    "GuardedCompareByteLoad:\n"
+    "3:  movzbl (%rcx), %eax\n"
+    "    cmpb (%rcx,%rdx), %al\n"
+    "    jne 5f\n"
+    "    incq %rcx\n"
+    "4:  cmpq %r9, %rcx\n"
+    "    jb 3b\n"
+    "    xorl %eax, %eax\n"
+    "    ret\n"
+    "5:  movl $1, %eax\n"
+    "    ret\n"
+    "GuardedReadFailed:\n"
+    "    movl $-1, %eax\n"
+    "    ret\n"
+    ".seh_endproc\n"
+    ".seh_proc GuardedCopyAvx2\n"
+    "GuardedCopyAvx2:\n"
+    ".seh_endprologue\n"
+    "    leaq (%rcx,%r8), %r9\n"
+    "    subq %rcx, %rdx\n"
+    "    jmp 2f\n"
+    "GuardedCopyAvx2Load:\n"
+    "1:  vmovdqu (%rcx), %ymm0\n"
+    "GuardedCopyAvx2HighLoad:\n"
+    "    vmovdqu 32(%rcx), %ymm1\n"
+    "    vmovdqu %ymm0, (%rcx,%rdx)\n"
+    "    vmovdqu %ymm1, 32(%rcx,%rdx)\n"
+    "    addq $64, %rcx\n"
+    "2:  leaq 64(%rcx), %rax\n"
+    "    cmpq %r9, %rax\n"
+    "    jbe 1b\n"
+    "    vzeroupper\n"
+    "    jmp GuardedCopyTail\n"
+    ".seh_endproc\n"
+    ".seh_proc GuardedCompareAvx2\n"
+    "GuardedCompareAvx2:\n"
+    ".seh_endprologue\n"
+    "    leaq (%rcx,%r8), %r9\n"
+    "    subq %rcx, %rdx\n"
+    "    jmp 2f\n"
+    "GuardedCompareAvx2Load:\n"
+    "1:  vmovdqu (%rcx), %ymm0\n"
+    "GuardedCompareAvx2HighLoad:\n"
+    "    vmovdqu 32(%rcx), %ymm1\n"
+    "    vpcmpeqb (%rcx,%rdx), %ymm0, %ymm0\n"
+    "    vpcmpeqb 32(%rcx,%rdx), %ymm1, %ymm1\n"
+    "    vpand %ymm1, %ymm0, %ymm0\n"
+    "    vpmovmskb %ymm0, %eax\n"
+    "    cmpl $-1, %eax\n"
+    "    jne 3f\n"
+    "    addq $64, %rcx\n"
+    "2:  leaq 64(%rcx), %rax\n"
+    "    cmpq %r9, %rax\n"
+    "    jbe 1b\n"
+    "    vzeroupper\n"
+    "    jmp GuardedCompareTail\n"
+    "3:  vzeroupper\n"
+    "    movl $1, %eax\n"
+    "    ret\n"
+    "GuardedReadFailedAvx2:\n"
+    "    vzeroupper\n"
+    "    movl $-1, %eax\n"
+    "    ret\n"
+    ".seh_endproc\n");
+
+namespace {
+
+bool HasAvx2() {
+    static const bool avx2 = __builtin_cpu_supports("avx2");
+    return avx2;
+}
+
+bool OwnedByGuest(std::uintptr_t address, std::size_t bytes) {
+    const auto& arena = Arena::Get();
+    return arena.Contains(reinterpret_cast<const void*>(address), bytes) && !arena.OverlapsHostRegion(address, bytes);
+}
+
+}
+
+bool GuestArenaCopyOwned_nid_no_patch(std::uintptr_t source, void* destination, std::size_t bytes) {
+    return OwnedByGuest(source, bytes) && (HasAvx2() ? GuardedCopyAvx2 : GuardedCopy)(source, destination, bytes) == 0;
+}
+
+bool GuestArenaCompareOwned_nid_no_patch(std::uintptr_t source, const void* expected, std::size_t bytes, bool* equal) {
+    if (!OwnedByGuest(source, bytes)) return false;
+    const auto result = (HasAvx2() ? GuardedCompareAvx2 : GuardedCompare)(source, expected, bytes);
+    *equal = result == 0;
+    return result >= 0;
+}
+
+bool GuestArenaRecoverRead_nid_no_patch(EXCEPTION_POINTERS* exception) {
+    const auto* record = exception->ExceptionRecord;
+    const auto code = record->ExceptionCode;
+    if ((code != EXCEPTION_ACCESS_VIOLATION && code != EXCEPTION_IN_PAGE_ERROR) || record->NumberParameters < 2 || record->ExceptionInformation[0] != 0) return false;
+    auto* context = exception->ContextRecord;
+    const auto at = reinterpret_cast<const char*>(context->Rip);
+    const bool avx2 = at == GuardedCopyAvx2Load || at == GuardedCopyAvx2HighLoad || at == GuardedCompareAvx2Load || at == GuardedCompareAvx2HighLoad;
+    if (!avx2 && at != GuardedCopyWideLoad && at != GuardedCopyByteLoad && at != GuardedCompareWideLoad && at != GuardedCompareByteLoad) return false;
+    const auto address = static_cast<std::uintptr_t>(record->ExceptionInformation[1]);
+    if (address < context->Rcx || address >= context->R9) return false;
+    context->Rip = reinterpret_cast<DWORD64>(avx2 ? GuardedReadFailedAvx2 : GuardedReadFailed);
+    return true;
+}
+
 void GuestArenaSetProtection_nid_postfix(std::uintptr_t address, std::size_t bytes, std::uint32_t protection) {
     WindowsMappings::Get().SetProtection(address, bytes, protection);
     noteAccess(reinterpret_cast<const void*>(address), bytes);

core/libs/prx/libc/src/specifics/windows/CrashReport.cpp

@@ -352,6 +352,7 @@ bool HandleSse4a(EXCEPTION_POINTERS* info) {
 
 LONG WINAPI ReportCrash(EXCEPTION_POINTERS* info) {
     static std::atomic<bool> reported{false};
+    if (GuestArena::GuestArenaRecoverRead_nid_no_patch(info)) return EXCEPTION_CONTINUE_EXECUTION;
     const auto* fault = info->ExceptionRecord;
     if (fault->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && fault->NumberParameters >= 2 && fault->ExceptionInformation[0] == 1 && GuestArena::GuestArenaHandleWrite_nid_postfix(fault->ExceptionInformation[1])) return EXCEPTION_CONTINUE_EXECUTION;
     if (fault->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && fault->NumberParameters >= 2 && fault->ExceptionInformation[0] == 0 && GuestArena::GuestArenaHandleRead_nid_postfix(fault->ExceptionInformation[1])) return EXCEPTION_CONTINUE_EXECUTION;