refactor(libc): map a run of shared guest pages as one view on Windows

core/libs/CMakeLists.txt

@@ -317,12 +317,20 @@ if(BUILD_TESTING)
     if(WIN32)
         add_test_executable(windows_write_watch_tests tests/WindowsWriteWatch.cpp)
         target_include_directories(windows_write_watch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-        target_compile_definitions(windows_write_watch_tests PRIVATE _WIN32_WINNT=0x0A00)
+        target_compile_definitions(windows_write_watch_tests PRIVATE _WIN32_WINNT=0x0A00 APS5_COUNT_MAPPING_QUERIES=1)
         if(MINGW)
             target_compile_options(windows_write_watch_tests PRIVATE -fasynchronous-unwind-tables)
         endif()
         add_test(NAME windows_write_watch COMMAND windows_write_watch_tests)
         set_tests_properties(windows_write_watch PROPERTIES TIMEOUT 20)
+        add_test_executable(windows_shared_map_cost_tests tests/WindowsSharedMapCost.cpp)
+        target_include_directories(windows_shared_map_cost_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+        target_compile_definitions(windows_shared_map_cost_tests PRIVATE _WIN32_WINNT=0x0A00)
+        if(MINGW)
+            target_compile_options(windows_shared_map_cost_tests PRIVATE -fasynchronous-unwind-tables)
+        endif()
+        add_test(NAME windows_shared_map_cost COMMAND windows_shared_map_cost_tests)
+        set_tests_properties(windows_shared_map_cost PROPERTIES TIMEOUT 120)
     endif()
 
     if(WIN32)

core/libs/prx/libc/include/WindowsMappings.hpp

@@ -5,6 +5,9 @@
 #include <windows.h>
 #include <algorithm>
 #include <array>
+#ifdef APS5_COUNT_MAPPING_QUERIES
+#include <atomic>
+#endif
 #include <cstdio>
 #include <cstddef>
 #include <cstdint>
@@ -28,6 +31,12 @@ class WindowsMappings {
         return *mappings;
     }
 
+#ifdef APS5_COUNT_MAPPING_QUERIES
+    static std::uint64_t MemoryQueries() {
+        return queries.load(std::memory_order_relaxed);
+    }
+#endif
+
     void* Reserve(void* address, std::size_t bytes) {
         return allocate(GetCurrentProcess(), address, bytes, MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0);
     }
@@ -105,6 +114,8 @@ class WindowsMappings {
             const auto shared = slot.page.lock();
             return resident && !slot.stored && (!shared || shared->aliases.empty());
         };
+        const bool perPage = !resident && sectionBytes == SIZE_MAX;
+        if (perPage) commitSection(section, offset, bytes);
         for (std::size_t done = 0; done < bytes;) {
             const auto base = cursor + done;
             if (single(done)) {
@@ -134,21 +145,52 @@ class WindowsMappings {
                 done += runBytes;
                 continue;
             }
-            const auto slot = slotOf(done);
-            const auto shared = pageOf(*slot);
-            const auto aliases = shared->aliases;
-            for (const auto alias : aliases) {
-                if (views.at(alias).resident) demote(alias);
+            if (sectionBytes != SIZE_MAX) {
+                const auto slot = slotOf(done);
+                const auto shared = pageOf(*slot);
+                const auto aliases = shared->aliases;
+                for (const auto alias : aliases) {
+                    if (views.at(alias).resident) demote(alias);
+                }
+                const auto viewBytes = std::min(pageBytes, sectionBytes - done);
+                split(base, pageBytes);
+                mapShared(base, section, offset + done, protection, viewProtection, viewBytes);
+                slot->stored = true;
+                shared->aliases.push_back(base);
+                views.emplace(base, View{shared, protection, {}, 0, owned, offset + done, 0, 0, false, slot, hostPages(viewBytes)});
+                remember(freshRanges, base, base + pageBytes);
+                invalidate(*shared);
+                done += pageBytes;
+                continue;
             }
-            const auto viewBytes = std::min(pageBytes, sectionBytes - done);
-            split(base, pageBytes);
-            mapShared(base, section, offset + done, protection, viewProtection, viewBytes);
-            slot->stored = true;
-            shared->aliases.push_back(base);
-            views.emplace(base, View{shared, protection, {}, 0, owned, offset + done, 0, 0, false, slot, hostPages(viewBytes)});
-            remember(freshRanges, base, base + pageBytes);
-            invalidate(*shared);
-            done += pageBytes;
+            std::vector<std::pair<std::shared_ptr<Physical>, std::shared_ptr<SharedPage>>> pages;
+            pages.reserve((bytes - done) / pageBytes);
+            auto runBytes = std::size_t{0};
+            do {
+                const auto slot = slotOf(done + runBytes);
+                const auto shared = pageOf(*slot);
+                const auto aliases = shared->aliases;
+                for (const auto alias : aliases) {
+                    if (views.at(alias).resident) demote(alias);
+                }
+                pages.emplace_back(slot, shared);
+                runBytes += pageBytes;
+            } while (done + runBytes < bytes && !single(done + runBytes));
+            if (!perPage) commitSection(section, offset + done, runBytes);
+            split(base, runBytes);
+            mapRun(base, runBytes, section, offset + done);
+            DWORD previous;
+            if (!VirtualProtect(reinterpret_cast<void*>(base), runBytes, protection, &previous)) fail("protect shared guest pages");
+            collectableRanges.clear();
+            for (std::size_t at = 0; at < runBytes; at += pageBytes) {
+                auto& [slot, shared] = pages[at / pageBytes];
+                slot->stored = true;
+                shared->aliases.push_back(base + at);
+                invalidate(*shared);
+                views.emplace(base + at, View{std::move(shared), protection, {}, 0, owned, offset + done + at, 0, 0, false, std::move(slot)});
+            }
+            remember(freshRanges, base, base + runBytes);
+            done += runBytes;
         }
     }
 
@@ -694,12 +736,23 @@ class WindowsMappings {
         return (bytes + subpageBytes - 1) & ~(subpageBytes - 1);
     }
 
-    void mapShared(std::uintptr_t base, HANDLE section, std::uint64_t offset, DWORD protection, DWORD viewProtection = PAGE_EXECUTE_READWRITE, std::size_t sectionBytes = pageBytes) {
+    void commitSection(HANDLE section, std::uint64_t offset, std::size_t bytes) {
+        auto window = open(section, offset, bytes);
+        const auto memory = query(reinterpret_cast<std::uintptr_t>(window.bytes));
+        if ((memory.State != MEM_COMMIT || memory.RegionSize < bytes) && !VirtualAlloc(window.bytes, bytes, MEM_COMMIT, PAGE_READWRITE)) {
+            const auto error = GetLastError();
+            close(window);
+            throw std::system_error(static_cast<int>(error), std::system_category(), "commit direct memory pages");
+        }
+        close(window);
+    }
+
+    void mapShared(std::uintptr_t base, HANDLE section, std::uint64_t offset, DWORD protection, DWORD viewProtection = PAGE_EXECUTE_READWRITE, std::size_t sectionBytes = pageBytes, bool committed = false) {
         void* page = reinterpret_cast<void*>(base);
         const auto viewBytes = hostPages(sectionBytes);
         if (viewBytes != pageBytes) split(base, viewBytes);
         if (!map(section, GetCurrentProcess(), page, offset, sectionBytes, MEM_REPLACE_PLACEHOLDER, viewProtection, nullptr, 0)) fail("map shared guest page");
-        if (query(base).State == MEM_RESERVE && !VirtualAlloc(page, viewBytes, MEM_COMMIT, PAGE_EXECUTE_READWRITE)) fail("commit shared guest page");
+        if (!committed && query(base).State == MEM_RESERVE && !VirtualAlloc(page, viewBytes, MEM_COMMIT, PAGE_EXECUTE_READWRITE)) fail("commit shared guest page");
         DWORD previous;
         if (!VirtualProtect(page, viewBytes, protection, &previous)) fail("protect shared guest page");
         if (viewBytes == pageBytes) return;
@@ -708,6 +761,47 @@ class WindowsMappings {
         if (!VirtualProtect(zeros, pageBytes - viewBytes, protection, &previous)) fail("protect guest pages past the end of a file");
     }
 
+    void mapRun(std::uintptr_t base, std::size_t bytes, HANDLE section, std::uint64_t offset) {
+        if (!map(section, GetCurrentProcess(), reinterpret_cast<void*>(base), offset, bytes, MEM_REPLACE_PLACEHOLDER, PAGE_EXECUTE_READWRITE, nullptr, 0)) fail("map a run of shared guest pages");
+    }
+
+    void breakView(std::uintptr_t at) {
+        const auto memory = query(at);
+        if (memory.State == MEM_FREE || memory.Type != MEM_MAPPED) return;
+        const auto start = reinterpret_cast<std::uintptr_t>(memory.AllocationBase);
+        if (start == at) return;
+        const auto first = views.find(start);
+        if (first == views.end()) return;
+        struct Part {
+            std::uintptr_t base;
+            std::uintptr_t end;
+            DWORD protection;
+            bool committed;
+        };
+        std::vector<Part> parts;
+        auto stop = start;
+        for (;;) {
+            const auto part = query(stop);
+            if (part.AllocationBase != memory.AllocationBase) break;
+            const auto partEnd = reinterpret_cast<std::uintptr_t>(part.BaseAddress) + part.RegionSize;
+            parts.push_back({stop, partEnd, part.Protect, part.State == MEM_COMMIT});
+            stop = partEnd;
+        }
+        const auto section = first->second.section;
+        const auto offset = first->second.offset;
+        if (!unmap(GetCurrentProcess(), reinterpret_cast<void*>(start), MEM_PRESERVE_PLACEHOLDER)) fail("unmap a run of shared guest pages to split it");
+        split(start, at - start);
+        mapRun(start, at - start, section->handle, offset);
+        mapRun(at, stop - at, section->handle, offset + (at - start));
+        for (const auto& part : parts) {
+            if (!part.committed) continue;
+            for (const auto [from, to] : {std::pair{part.base, std::min(part.end, at)}, std::pair{std::max(part.base, at), part.end}}) {
+                DWORD previous;
+                if (from < to && !VirtualProtect(reinterpret_cast<void*>(from), to - from, part.protection, &previous)) fail("protect a split run of shared guest pages");
+            }
+        }
+    }
+
     static bool blank(std::uintptr_t page) {
         const auto* words = reinterpret_cast<const std::uint64_t*>(page);
         for (std::size_t i = 0; i < pageBytes / sizeof(std::uint64_t); ++i) {
@@ -890,11 +984,14 @@ class WindowsMappings {
 
     static MEMORY_BASIC_INFORMATION query(std::uintptr_t address) {
         MEMORY_BASIC_INFORMATION memory{};
+#ifdef APS5_COUNT_MAPPING_QUERIES
+        queries.fetch_add(1, std::memory_order_relaxed);
+#endif
         if (VirtualQuery(reinterpret_cast<void*>(address), &memory, sizeof(memory)) != sizeof(memory)) fail("query guest memory");
         return memory;
     }
 
-    static void split(std::uintptr_t address, std::size_t bytes) {
+    static std::uintptr_t split(std::uintptr_t address, std::size_t bytes) {
         auto memory = query(address);
         memory = query(reinterpret_cast<std::uintptr_t>(memory.AllocationBase));
         if (memory.State != MEM_RESERVE) throw std::runtime_error("guest mapping requires a placeholder");
@@ -905,6 +1002,7 @@ class WindowsMappings {
         }
         if (memory.RegionSize < bytes) throw std::runtime_error("guest placeholder is too small");
         if (memory.RegionSize != bytes && !VirtualFree(reinterpret_cast<void*>(address), bytes, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) fail("split guest placeholder suffix");
+        return address + memory.RegionSize;
     }
 
     void reset(std::uintptr_t address, std::size_t bytes) {
@@ -913,6 +1011,8 @@ class WindowsMappings {
         forgetClean(address, end);
         breakRun(address);
         breakRun(end);
+        breakView(address);
+        breakView(end);
         if (const auto run = residentRuns.find(address); run != residentRuns.end() && run->second.end == end) {
             const auto memory = query(address);
             if (reinterpret_cast<std::uintptr_t>(memory.AllocationBase) != address) throw std::runtime_error("cannot release part of a host allocation");
@@ -944,12 +1044,11 @@ class WindowsMappings {
             if (allocationEnd > end || query(allocationEnd).AllocationBase == memory.AllocationBase) throw std::runtime_error("guest release truncates a host allocation");
             if (memory.Type == MEM_MAPPED) {
                 if (!unmap(GetCurrentProcess(), reinterpret_cast<void*>(cursor), MEM_PRESERVE_PLACEHOLDER)) fail("unmap shared guest page");
-                const auto found = views.find(cursor);
-                if (found != views.end()) {
-                    std::erase(found->second.page->aliases, cursor);
-                    views.erase(found);
-                    forget(freshRanges, cursor, cursor + pageBytes);
+                for (auto found = views.lower_bound(cursor); found != views.end() && found->first < allocationEnd;) {
+                    std::erase(found->second.page->aliases, found->first);
+                    found = views.erase(found);
                 }
+                forget(freshRanges, cursor, allocationEnd);
             } else if (memory.Type == MEM_PRIVATE) {
                 if (const auto run = residentRuns.find(cursor); run != residentRuns.end()) evict(run);
                 if (!VirtualFree(reinterpret_cast<void*>(cursor), allocationEnd - cursor, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) fail("release private guest memory");
@@ -1023,6 +1122,9 @@ class WindowsMappings {
     std::map<std::uintptr_t, std::size_t> splitAliases;
     std::size_t granularity = 0x10000;
     bool watchResident = false;
+#ifdef APS5_COUNT_MAPPING_QUERIES
+    static inline std::atomic<std::uint64_t> queries{0};
+#endif
     std::mutex mutex;
     AllocateFunction allocate = nullptr;
     MapFunction map = nullptr;

core/libs/tests/WindowsSharedMapCost.cpp

@@ -0,0 +1,78 @@
+#define NOMINMAX
+#include "prx/libc/include/WindowsMappings.hpp"
+#include <algorithm>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <set>
+#include <vector>
+
+namespace {
+using Mappings = GuestArena::WindowsMappings;
+constexpr std::size_t PageBytes = 16384;
+constexpr int Rounds = 5;
+
+[[noreturn]] void Fail(const char* what) {
+    std::fprintf(stderr, "windows shared map cost: %s (error %lu)\n", what, GetLastError());
+    std::exit(1);
+}
+
+std::size_t MappedViews(const unsigned char* begin, std::size_t bytes) {
+    std::set<void*> bases;
+    for (auto cursor = begin; cursor < begin + bytes;) {
+        MEMORY_BASIC_INFORMATION info{};
+        if (VirtualQuery(cursor, &info, sizeof(info)) != sizeof(info)) Fail("view query failed");
+        if (info.Type == MEM_MAPPED) bases.insert(info.AllocationBase);
+        cursor = static_cast<const unsigned char*>(info.BaseAddress) + info.RegionSize;
+    }
+    return bases.size();
+}
+
+struct Sample {
+    double mapMs = 0;
+    double releaseMs = 0;
+    std::size_t views = 0;
+};
+
+Sample MapOnce(std::size_t pages) {
+    const auto bytes = pages * PageBytes;
+    HANDLE section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE | SEC_RESERVE, static_cast<DWORD>(static_cast<std::uint64_t>(bytes) >> 32), static_cast<DWORD>(bytes), nullptr);
+    auto* memory = static_cast<unsigned char*>(Mappings::Get().Reserve(nullptr, bytes));
+    if (section == nullptr || memory == nullptr) Fail("cannot allocate shared memory");
+    const auto start = std::chrono::steady_clock::now();
+    Mappings::Get().Map(memory, bytes, section, 0, PAGE_READWRITE);
+    Sample sample;
+    sample.mapMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
+    sample.views = MappedViews(memory, bytes);
+    memory[0] = 1;
+    memory[bytes - 1] = 1;
+    const auto released = std::chrono::steady_clock::now();
+    Mappings::Get().Reset(memory, bytes);
+    sample.releaseMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - released).count();
+    VirtualFree(memory, 0, MEM_RELEASE);
+    CloseHandle(section);
+    return sample;
+}
+
+double Median(std::vector<double> values) {
+    std::sort(values.begin(), values.end());
+    return values[values.size() / 2];
+}
+
+}
+
+int main() {
+    for (const std::size_t pages : {std::size_t{256}, std::size_t{16384}, std::size_t{65536}}) {
+        std::vector<double> map;
+        std::vector<double> release;
+        std::size_t views = 0;
+        for (int round = 0; round < Rounds; ++round) {
+            const auto sample = MapOnce(pages);
+            map.push_back(sample.mapMs);
+            release.push_back(sample.releaseMs);
+            views = sample.views;
+        }
+        std::printf("shared map of %zu pages (%zu MiB): map median %.2f ms (%.2f us per page), release median %.2f ms, %zu views\n", pages, pages * PageBytes >> 20, Median(map), Median(map) * 1000.0 / static_cast<double>(pages), Median(release), views);
+    }
+    return 0;
+}