fix(agc): allocate Windows page permissions on demand

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -714,6 +714,17 @@ target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel l
 set_target_properties(agc_shared_write_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_shared_write_tests)
 
+if(BUILD_TESTING AND WIN32)
+    add_test_executable(agc_page_state_tests tests/PageStates.cpp)
+    target_include_directories(agc_page_state_tests PRIVATE ${LIBS_INCLUDE_DIR})
+    target_link_libraries(agc_page_state_tests PRIVATE libSceAgcDriver libc)
+    configure_windows_unwind(agc_page_state_tests)
+    add_test(NAME agc_page_states COMMAND agc_page_state_tests)
+    add_test(NAME agc_page_state_metadata COMMAND agc_page_state_tests --metadata)
+    set_tests_properties(agc_page_states agc_page_state_metadata PROPERTIES
+        TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+endif()
+
 add_test_executable(agc_locks_tests EXCLUDE_FROM_ALL tests/Locks.cpp)
 target_include_directories(agc_locks_tests PRIVATE
         ${LIBS_INCLUDE_DIR}

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -22,6 +22,7 @@
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <new>
 #include <stdexcept>
 #include <string>
 #include <tuple>
@@ -372,30 +373,62 @@ struct PageSpan {
     std::size_t size = 0;
     // Per page: access flags, zero while unknown.
     std::uint8_t* pages = nullptr;
+    static constexpr std::size_t ChunkEntries = 65536;
+    struct alignas(8) Chunk { std::array<std::uint8_t, ChunkEntries> entries{}; };
+    std::atomic<Chunk*>* chunks = nullptr;
+
+    std::uint8_t* chunkData(std::size_t index, bool create) const {
+        auto* chunk = chunks[index].load(std::memory_order_acquire);
+        if (chunk == nullptr && create) {
+            auto fresh = std::unique_ptr<Chunk>(new (std::nothrow) Chunk{});
+            if (fresh != nullptr && chunks[index].compare_exchange_strong(chunk, fresh.get(), std::memory_order_acq_rel, std::memory_order_acquire)) chunk = fresh.release();
+        }
+        return chunk == nullptr ? nullptr : chunk->entries.data();
+    }
+
+    std::uint8_t* entry(std::size_t index, bool create) const {
+        if (chunks == nullptr) return pages + index;
+        auto* data = chunkData(index / ChunkEntries, create);
+        return data == nullptr ? nullptr : data + index % ChunkEntries;
+    }
 
-    // Reserves and commits the table; untouched table pages cost nothing until a guest page is recorded.
     bool allocate() {
         if (size == 0) return false;
         const auto count = (size / PageBytes + 8) & ~static_cast<std::size_t>(7);
 #ifdef _WIN32
-        pages = static_cast<std::uint8_t*>(VirtualAlloc(nullptr, count, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
+        const auto chunkCount = (count + ChunkEntries - 1) / ChunkEntries;
+        chunks = new (std::nothrow) std::atomic<Chunk*>[chunkCount]{};
+        if (chunks == nullptr) size = 0;
+        return chunks != nullptr;
 #else
         pages = static_cast<std::uint8_t*>(std::calloc(count, 1));
-#endif
         if (pages == nullptr) size = 0;
         return pages != nullptr;
+#endif
     }
 
     bool covers(std::uintptr_t address) const {
         return size != 0 && address >= base && address - base < size;
     }
 
     std::uint8_t load(std::uintptr_t address) const {
-        return std::atomic_ref<std::uint8_t>(const_cast<std::uint8_t&>(pages[(address - base) / PageBytes])).load(std::memory_order_relaxed);
+        auto* value = entry((address - base) / PageBytes, false);
+        return value == nullptr ? 0 : std::atomic_ref<std::uint8_t>(*value).load(std::memory_order_relaxed);
     }
 
     void store(std::uintptr_t address, std::uint8_t value) {
-        std::atomic_ref<std::uint8_t>(pages[(address - base) / PageBytes]).store(value, std::memory_order_relaxed);
+        if (auto* at = entry((address - base) / PageBytes, value != 0 && value != PageInaccessible)) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+    }
+
+    void store(std::uintptr_t first, std::uintptr_t last, std::uint8_t value) {
+        auto index = (first - base) / PageBytes;
+        const auto stop = (last - 1 - base) / PageBytes + 1;
+        while (index < stop) {
+            const auto next = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
+            if (auto* at = entry(index, value != 0 && value != PageInaccessible)) {
+                for (; index < next; ++index, ++at) std::atomic_ref<std::uint8_t>(*at).store(value, std::memory_order_relaxed);
+            } else index = next;
+        }
     }
 
     // End of the run of pages equal to `value` that starts at `address`, at most `limit`.
@@ -404,11 +437,17 @@ struct PageSpan {
         const auto stop = std::min<std::uintptr_t>((std::min(limit, base + size) - base + PageBytes - 1) / PageBytes, size / PageBytes);
         const std::uint64_t wide = 0x0101010101010101ull * value;
         while (index < stop) {
-            if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(const_cast<std::uint8_t*>(pages + index))).load(std::memory_order_relaxed) == wide) {
+            auto* at = entry(index, false);
+            if (at == nullptr) {
+                if (value != 0) break;
+                index = std::min(stop, (index / ChunkEntries + 1) * ChunkEntries);
+                continue;
+            }
+            if (index % 8 == 0 && index + 8 <= stop && std::atomic_ref<std::uint64_t>(*reinterpret_cast<std::uint64_t*>(at)).load(std::memory_order_relaxed) == wide) {
                 index += 8;
                 continue;
             }
-            if (std::atomic_ref<std::uint8_t>(const_cast<std::uint8_t&>(pages[index])).load(std::memory_order_relaxed) != value) break;
+            if (std::atomic_ref<std::uint8_t>(*at).load(std::memory_order_relaxed) != value) break;
             ++index;
         }
         return std::min(limit, base + index * PageBytes);
@@ -418,7 +457,7 @@ struct PageSpan {
         if (size == 0 || bytes == 0 || address >= base + size || address + bytes <= base) return;
         const auto first = (std::max(address, base) - base) / PageBytes;
         const auto last = (std::min(address + bytes, base + size) - 1 - base) / PageBytes;
-        for (auto page = first; page <= last; ++page) std::atomic_ref<std::uint8_t>(pages[page]).store(0, std::memory_order_relaxed);
+        store(base + first * PageBytes, base + (last + 1) * PageBytes, 0);
     }
 };
 
@@ -449,7 +488,7 @@ struct PageStates {
             const std::uint8_t value = readable ? static_cast<std::uint8_t>(PageReadable | (writable ? PageWritable : 0u)) : PageInaccessible;
             const auto first = std::max(base, span->base);
             const auto last = std::min(regionEnd, span->base + span->size);
-            for (auto at = first; at < last; at += PageBytes) span->store(at, value);
+            span->store(first, last, value);
             if (GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation || accessSerial.load(std::memory_order_acquire) != access) span->forget(first, static_cast<std::size_t>(last - first));
         }
     }

core/libs/prx/libSceAgcDriver/tests/PageStates.cpp

@@ -0,0 +1,145 @@
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cstdio>
+#include <cstring>
+#include <stdexcept>
+#include <string_view>
+#include <thread>
+#include <vector>
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <psapi.h>
+
+namespace {
+using namespace AgcDriver::GuestMemory;
+constexpr std::size_t Unit = 65536;
+constexpr std::size_t ChunkGuestBytes = 256ull << 20u;
+
+void Require(bool condition, const char* message) {
+    if (!condition) throw std::runtime_error(message);
+}
+
+class Range {
+public:
+    explicit Range(std::size_t bytes) : bytes(bytes) {
+        data = static_cast<std::byte*>(GuestArena::GuestArenaAllocate_nid_postfix(bytes, ChunkGuestBytes));
+        Require(data != nullptr, "cannot reserve page-state fixture");
+    }
+
+    ~Range() {
+        {
+            GuestAllocations::Mutation mutation;
+            for (const auto offset : committed) mutation.Remove(data + offset);
+        }
+        GuestArena::GuestArenaReset_nid_postfix(data, bytes);
+        GuestArena::GuestArenaRelease_nid_postfix(data, bytes);
+    }
+
+    std::byte* Commit(std::size_t offset) {
+        auto* at = data + offset;
+        GuestArena::GuestArenaCommit_nid_postfix(at, Unit, PAGE_READWRITE, Unit);
+        std::memset(at, 0x5b, Unit);
+        GuestAllocations::Mutation mutation;
+        mutation.Add(at, Unit, true, true, true);
+        committed.push_back(offset);
+        return at;
+    }
+
+    std::byte* Data() const { return data; }
+
+private:
+    std::byte* data = nullptr;
+    std::size_t bytes;
+    std::vector<std::size_t> committed;
+};
+
+std::size_t PrivateBytes() {
+    PROCESS_MEMORY_COUNTERS_EX counters{};
+    Require(K32GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&counters), sizeof(counters)), "cannot query page-state fixture private commit");
+    return counters.PrivateUsage;
+}
+
+void CheckMetadata() {
+    Range range(3 * ChunkGuestBytes);
+    auto* data = range.Commit(0);
+    const auto before = PrivateBytes();
+    Require(Accessible(data, Unit, true), "committed fixture is inaccessible");
+    const auto after = PrivateBytes();
+    const auto added = after > before ? after - before : 0;
+    std::printf("page-permission initialization: %zu bytes of private commit\n", added);
+    Require(added < (16ull << 20u), "page permissions committed a dense arena table");
+    const auto queried = PrivateBytes();
+    Require(!Accessible(range.Data() + 2 * ChunkGuestBytes, Unit), "reserved fixture tail is accessible");
+    Require(PrivateBytes() < queried + (4ull << 20u), "querying reserved pages allocated untouched metadata chunks");
+}
+
+void CheckBoundaries() {
+    constexpr auto span = 3 * ChunkGuestBytes;
+    Range range(span);
+    const std::array<std::size_t, 4> offsets{0, ChunkGuestBytes - Unit, ChunkGuestBytes, 2 * ChunkGuestBytes + Unit};
+    for (const auto offset : offsets) range.Commit(offset);
+    auto* block = range.Data();
+    std::atomic<bool> begin{false};
+    std::atomic<unsigned> errors{0};
+    std::vector<std::thread> readers;
+    for (unsigned reader = 0; reader < 8; ++reader) {
+        readers.emplace_back([&] {
+            while (!begin.load(std::memory_order_acquire)) std::this_thread::yield();
+            for (unsigned round = 0; round < 500; ++round) {
+                if (!Accessible(block + offsets.back(), Unit, true)) errors.fetch_add(1, std::memory_order_relaxed);
+            }
+        });
+    }
+    begin.store(true, std::memory_order_release);
+    for (auto& reader : readers) reader.join();
+    Require(errors.load() == 0, "concurrent page-state publication changed access");
+    for (const auto offset : offsets) {
+        std::array<std::byte, 16> copy{};
+        Read(reinterpret_cast<std::uint64_t>(block + offset), copy);
+        Require(std::all_of(copy.begin(), copy.end(), [](std::byte value) { return value == std::byte{0x5b}; }), "page cache changed guest bytes");
+    }
+    Require(Accessible(block + ChunkGuestBytes - Unit, 2 * Unit, true), "adjacent chunks lost their boundary pages");
+    const auto described = DescribeCommitted(reinterpret_cast<std::uint64_t>(block), span, true);
+    std::uint64_t total = 0;
+    for (const auto& [first, last] : described.ranges) total += last - first;
+    Require(!described.whole && total == offsets.size() * Unit, "reserved holes were reported committed");
+    auto* changed = block + offsets[1];
+    const auto protect = [&](bool writable) {
+        GuestAllocations::Mutation mutation;
+        mutation.Protect(changed, Unit, true, writable, true, [&] {
+            const DWORD protection = writable ? PAGE_READWRITE : PAGE_READONLY;
+            DWORD previous = 0;
+            Require(VirtualProtect(changed, Unit, protection, &previous) != 0, "cannot protect page-state fixture");
+            GuestArena::GuestArenaSetProtection_nid_postfix(reinterpret_cast<std::uintptr_t>(changed), Unit, protection);
+        });
+    };
+    protect(false);
+    Require(Accessible(changed, Unit) && !Accessible(changed, Unit, true), "protection invalidation retained writable state");
+    protect(true);
+    Require(Accessible(changed, Unit, true), "restored writable state was not discovered");
+    GuestArena::GuestArenaReset_nid_postfix(block, Unit);
+    GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast<std::uintptr_t>(block), Unit);
+    Require(!Accessible(block, Unit), "decommitted pages remained readable");
+    GuestArena::GuestArenaCommit_nid_postfix(block, Unit, PAGE_READWRITE, Unit);
+    Require(Accessible(block, Unit, true), "access observer did not invalidate inaccessible state after commit");
+    std::puts("page-state boundaries, publication, holes and invalidation passed");
+}
+}
+
+int main(int argc, char** argv) {
+    try {
+        Require(GuestArena::GuestArenaAvailable_nid_postfix(), "page-state fixture needs the guest arena");
+        if (argc == 2 && std::string_view(argv[1]) == "--metadata") CheckMetadata();
+        else CheckBoundaries();
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+        return 1;
+    }
+}