feat(agc): trace ranges that leave the write watch

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1213,6 +1213,10 @@ if(BUILD_TESTING)
         set_tests_properties(agc_storage_kept_unit_settle PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    if(NOT WIN32)
+        add_test(NAME agc_storage_unwalked_import COMMAND agc_driver_recorder_tests --unwalked-import-only)
+        set_tests_properties(agc_storage_unwalked_import PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30)
+    endif()
     add_test(NAME agc_storage_baseline_settled COMMAND agc_driver_recorder_tests --settled-baseline-only)
     add_test(NAME agc_storage_integer_clear COMMAND agc_driver_recorder_tests --integer-clear-only)
     set_tests_properties(agc_storage_integer_clear PROPERTIES SKIP_RETURN_CODE 77)
@@ -1825,6 +1829,11 @@ if(BUILD_TESTING)
     if(WIN32)
         set_tests_properties(agc_driver_partially_resident_pending_stores PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_store_over_untracked_unimported COMMAND agc_driver_store_over_untracked_tests)
+    set_tests_properties(agc_driver_store_over_untracked_unimported PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_HOST_IMPORT=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_store_over_untracked_unimported PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_shader_disk_cache_large_file COMMAND agc_shader_disk_cache_tests --large-file)
     set_tests_properties(agc_shader_disk_cache_large_file PROPERTIES SKIP_RETURN_CODE 77 RUN_SERIAL TRUE WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 240)
     if(WIN32)

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -6,6 +6,7 @@
 #include <cstdint>
 #include <functional>
 #include <mutex>
+#include <optional>
 #include <span>
 #include <utility>
 #include <vector>
@@ -60,6 +61,18 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
 // page tables and are reported with MarkWritten. Without write watching CollectWrites returns 0 and
 // UnchangedSince is always false, so callers fall back to comparing bytes.
 bool WriteWatched();
+enum class WatchExit : std::uint8_t { Unwatch, ImportLeave, ImportStaysUncovered, ImportStaysWalkFailed, MarkWritten };
+const char* WatchExitName(WatchExit exit);
+struct ExclusionRecord {
+    WatchExit site;
+    std::uint64_t generation;
+    bool hooked;
+};
+struct ExclusionView {
+    std::optional<ExclusionRecord> record;
+    bool hostRegion = false;
+};
+ExclusionView ExclusionAt(std::uint64_t address);
 void Unwatch(std::uint64_t address, std::size_t bytes);
 void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges);
 bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched = true);

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -844,6 +844,42 @@ struct WriteTracker {
 #endif
     }
 
+    bool rangeWatched(std::uint64_t address, std::size_t bytes) const {
+#ifdef _WIN32
+        return coverage.Covers(address, bytes);
+#else
+        return GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(static_cast<std::uintptr_t>(address), bytes);
+#endif
+    }
+
+    bool inArena(std::uint64_t address) const {
+#ifdef _WIN32
+        return address >= base && address - base < size;
+#else
+        return address < LeafCount * LeafBlocks * WriteBlockBytes;
+#endif
+    }
+
+    std::vector<std::uint64_t> watchedBlocks(std::uint64_t address, std::size_t bytes) const {
+        std::vector<std::uint64_t> result;
+#ifdef _WIN32
+        const std::uint64_t low = base;
+        const std::uint64_t high = base + size;
+#else
+        const std::uint64_t low = 0;
+        const std::uint64_t high = std::uint64_t{LeafCount} * LeafBlocks * WriteBlockBytes;
+#endif
+        const auto first = std::max<std::uint64_t>(address, low);
+        const auto stop = std::min<std::uint64_t>(address + bytes, high);
+        if (first >= stop) return result;
+        for (auto block = blockOf(first); block <= blockOf(stop - 1); ++block) {
+            const auto from = std::max<std::uint64_t>(first, blockBegin(block));
+            const auto to = std::min<std::uint64_t>(stop, blockBegin(block) + WriteBlockBytes);
+            if (rangeWatched(from, static_cast<std::size_t>(to - from))) result.push_back(block);
+        }
+        return result;
+    }
+
 #ifdef _WIN32
     void startHints(std::uint64_t address, std::size_t bytes) {
         const auto first = std::max<std::uint64_t>(address, base);
@@ -899,6 +935,7 @@ struct WriteTracker {
         }
     };
     std::unordered_map<std::uint64_t, DriverPieces> driverPieces;
+    std::unordered_map<std::uint64_t, ExclusionRecord> exclusions;
 
     struct CpuPages {
         std::array<std::uint32_t, WritePagesPerBlock> pages{};
@@ -1075,6 +1112,7 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_
     tracker.noteCpuStore(address, address + bytes, tracker.generation);
     for (auto block = tracker.blockOf(address); block <= tracker.blockOf(address + bytes - 1); ++block) {
         tracker.driverPieces.erase(block);
+        if (!tracker.exclusions.empty() && tracker.coverage.Covers(tracker.blockBegin(block), 1)) tracker.exclusions.erase(block);
         tracker.stamp(block, tracker.generation, StampKind::Cpu);
     }
     unwatchSerial.fetch_add(1, std::memory_order_release);
@@ -1305,34 +1343,79 @@ namespace {
 
 std::atomic<void (*)(std::uint64_t, std::size_t)> unwatchHook{nullptr};
 
-void beforeUnwatch(std::uint64_t address, std::size_t bytes) {
+bool beforeUnwatch(std::uint64_t address, std::size_t bytes) {
     const auto hook = unwatchHook.load(std::memory_order_acquire);
-    if (hook == nullptr) return;
+    if (hook == nullptr) return false;
 #ifdef _WIN32
     std::uint64_t begin = 0;
     std::uint64_t end = 0;
     {
         auto& tracker = Tracker();
         const auto lock = lockTracker(tracker);
         tracker.initialize();
-        if (!tracker.watched || !tracker.coverage.ExcludedSpan(address, bytes, &begin, &end)) return;
+        if (!tracker.watched || !tracker.coverage.ExcludedSpan(address, bytes, &begin, &end)) return false;
     }
     hook(begin, static_cast<std::size_t>(end - begin));
 #else
     hook(address, bytes);
 #endif
+    return true;
 }
 
-void unwatchLocked(WriteTracker& tracker, std::uint64_t address, std::size_t bytes) {
+bool watchTraceEnabled() {
+    static const bool trace = std::getenv("APS5_TRACE_WATCH") != nullptr;
+    return trace;
+}
+
+bool watchTraceEveryExit() {
+    static const bool every = watchTraceEnabled() && std::strcmp(std::getenv("APS5_TRACE_WATCH"), "unsaved") != 0;
+    return every;
+}
+
+void unwatchLocked(WriteTracker& tracker, std::uint64_t address, std::size_t bytes, WatchExit site, bool hooked) {
     if (!tracker.watched) return;
+    const bool trace = watchTraceEnabled();
+    std::vector<std::uint64_t> fresh;
+    if (trace) fresh = tracker.watchedBlocks(address, bytes);
 #ifdef _WIN32
     tracker.startHints(address, bytes);
     if (tracker.coverage.Exclude(address, bytes, GuestArena::GuestArenaCommitGeneration_nid_postfix())) unwatchSerial.fetch_add(1, std::memory_order_release);
 #else
     if (GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast<const void*>(address), bytes)) unwatchSerial.fetch_add(1, std::memory_order_release);
 #endif
+    if (!trace) return;
+    const std::uint64_t generation = tracker.generation;
+    for (const auto block : fresh) tracker.exclusions.insert_or_assign(block, ExclusionRecord{site, generation, hooked});
+    if (!watchTraceEveryExit() && (hooked || fresh.empty())) return;
+    std::fprintf(stderr, "[watch] exclude 0x%llx+0x%zx by %s hook %s new-blocks %zu generation %llu\n", static_cast<unsigned long long>(address), bytes, WatchExitName(site), hooked ? "ran" : "none", fresh.size(), static_cast<unsigned long long>(generation));
+}
+
+}
+
+const char* WatchExitName(WatchExit exit) {
+    switch (exit) {
+    case WatchExit::Unwatch: return "Unwatch";
+    case WatchExit::ImportLeave: return "ImportLeave";
+    case WatchExit::ImportStaysUncovered: return "ImportStaysUncovered";
+    case WatchExit::ImportStaysWalkFailed: return "ImportStaysWalkFailed";
+    case WatchExit::MarkWritten: return "MarkWritten";
+    }
+    return "unknown";
 }
 
+ExclusionView ExclusionAt(std::uint64_t address) {
+    auto& tracker = Tracker();
+    const auto lock = lockTracker(tracker);
+    tracker.initialize();
+    ExclusionView view;
+    if (!tracker.watched || !tracker.inArena(address)) return view;
+    const auto block = tracker.blockOf(address);
+#ifdef _WIN32
+    view.hostRegion = GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(static_cast<std::uintptr_t>(tracker.blockBegin(block)), WriteBlockBytes);
+#endif
+    const auto found = tracker.exclusions.find(block);
+    if (found != tracker.exclusions.end()) view.record = found->second;
+    return view;
 }
 
 void SetUnwatchHook(void (*hook)(std::uint64_t, std::size_t)) {
@@ -1341,11 +1424,11 @@ void SetUnwatchHook(void (*hook)(std::uint64_t, std::size_t)) {
 
 void Unwatch(std::uint64_t address, std::size_t bytes) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return;
-    beforeUnwatch(address, bytes);
+    const bool hooked = beforeUnwatch(address, bytes);
     auto& tracker = Tracker();
     const auto lock = lockTracker(tracker);
     tracker.initialize();
-    unwatchLocked(tracker, address, bytes);
+    unwatchLocked(tracker, address, bytes, WatchExit::Unwatch, hooked);
 }
 
 void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
@@ -1375,10 +1458,11 @@ void Rewatch(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges) {
 
 bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function<bool()>& import, bool staysWatched) {
     if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return import();
+    bool hooked = false;
     if (!staysWatched) {
         constexpr std::uint64_t page = 4096;
         const auto first = address & ~(page - 1);
-        beforeUnwatch(first, static_cast<std::size_t>(((address + bytes + page - 1) & ~(page - 1)) - first));
+        hooked = beforeUnwatch(first, static_cast<std::size_t>(((address + bytes + page - 1) & ~(page - 1)) - first));
     }
     auto& tracker = Tracker();
     auto lock = lockTracker(tracker);
@@ -1409,6 +1493,13 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
         }
     }
     const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu, true);
+    const auto saveBytes = [&] {
+        lock.unlock();
+        hooked = beforeUnwatch(first, static_cast<std::size_t>(stop - first));
+        lock.lock();
+    };
+    const bool savedEarly = staysWatched && !before;
+    if (savedEarly) saveBytes();
     bool imported = false;
     {
 #ifdef __APPLE__
@@ -1418,12 +1509,15 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
     }
     if (!imported) return false;
     if (!staysWatched) {
-        if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
-        unwatchLocked(tracker, address, bytes);
+        if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first), WatchExit::ImportLeave, hooked);
+        unwatchLocked(tracker, address, bytes, WatchExit::ImportLeave, hooked);
         return true;
     }
     const WindowCopy window{first, saved, slots};
-    if (!before || !walkWrites(tracker, first, stop, StampKind::Cpu, true, &window)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
+    if (!before || !walkWrites(tracker, first, stop, StampKind::Cpu, true, &window)) {
+        if (!savedEarly) saveBytes();
+        unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first), covered ? WatchExit::ImportStaysWalkFailed : WatchExit::ImportStaysUncovered, hooked);
+    }
     return true;
 }
 
@@ -1557,7 +1651,7 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
         const bool complete = GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs);
 #endif
         if (!complete) {
-            unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
+            unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst), WatchExit::MarkWritten, false);
             tracker.noteUnstampedStore(address, address + bytes);
             return 0;
         }

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -564,6 +564,7 @@ class StorageTexture : public std::enable_shared_from_this<StorageTexture> {
     std::uint32_t trackedLayers = 1;
     std::uint64_t trackedLayerBytes = 0;
     std::vector<std::uint64_t> layerGeneration;
+    std::vector<std::uint64_t> layerPendingGeneration;
     std::vector<bool> layerPending;
     bool blockUnits = false;
     // Write-back hysteresis (block units, see writeBack): partial stores in the current window of

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -3221,6 +3221,14 @@ void StorageTexture::markLayersPending(std::uint32_t first, std::uint32_t count)
         anyUnsaved = anyUnsaved || unsaved[layer];
     }
     if (anyUnsaved && !GuestMemory::Watched(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) captureGuestBytes(&unsaved);
+    static const bool traceWatch = std::getenv("APS5_TRACE_WATCH") != nullptr;
+    if (traceWatch) {
+        if (layerPendingGeneration.size() != trackedLayers) layerPendingGeneration.assign(trackedLayers, 0);
+        const auto stamp = GuestMemory::TrackerGeneration();
+        for (std::uint32_t layer = first; layer < first + count; ++layer) {
+            if (!layerPending[layer]) layerPendingGeneration[layer] = stamp;
+        }
+    }
     for (std::uint32_t layer = first; layer < first + count; ++layer) layerPending[layer] = true;
     // Results of this image now cover the alias's results borrowed into these units: this image
     // stores them, the alias no longer has to (its results, unchanged since the borrow, are in
@@ -3422,8 +3430,10 @@ void StorageTexture::forgetSavedOver(std::uint64_t address, std::size_t bytes) {
     // Only these units' bytes changed under the saved copy. Units outside keep it: results still
     // pending there (the flush before the fill or copy stored only the units it overlaps) are
     // compared with it at their store when the memory is outside the write watch.
+    static const bool traceWatch = std::getenv("APS5_TRACE_WATCH") != nullptr;
     for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) {
         if (layerBegin(unit) < address + bytes && address < layerBegin(unit) + layerBytes(unit)) {
+            if (traceWatch && layerPending[unit] && layerOriginal[unit]) std::fprintf(stderr, "[watch] forget image 0x%llx: unit %u 0x%llx+0x%llx pending, saved bytes cleared by 0x%llx+0x%zx\n", static_cast<unsigned long long>(descriptor.baseAddress), unit, static_cast<unsigned long long>(layerBegin(unit)), static_cast<unsigned long long>(layerBytes(unit)), static_cast<unsigned long long>(address), bytes);
             layerOriginal[unit] = false;
             cancelStoredSave(unit);
         }
@@ -3438,7 +3448,10 @@ void StorageTexture::saveBeforeUnwatch() {
     original.resize(static_cast<std::size_t>(guestBytes));
     comparedGuestBytes = {};
     for (const auto& [begin, end] : runs) GuestMemory::ReadCommittedUnsynced(descriptor.baseAddress + begin, std::span(original).subspan(static_cast<std::size_t>(begin), static_cast<std::size_t>(end - begin)));
-    for (const auto& [begin, end] : runs) GuestMemory::CollectWritesUncached(descriptor.baseAddress + begin, static_cast<std::size_t>(end - begin));
+    static const bool traceWatch = std::getenv("APS5_TRACE_WATCH") != nullptr;
+    for (const auto& [begin, end] : runs) {
+        if (GuestMemory::CollectWritesUncached(descriptor.baseAddress + begin, static_cast<std::size_t>(end - begin)) == 0 && traceWatch) std::fprintf(stderr, "[watch] save image 0x%llx: collect failed for 0x%llx+0x%llx\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(descriptor.baseAddress + begin), static_cast<unsigned long long>(end - begin));
+    }
     constexpr std::uint64_t block = 65536;
     for (std::uint32_t unit = 0; unit < trackedLayers; ++unit) {
         if (!units[unit]) continue;
@@ -3455,7 +3468,10 @@ void StorageTexture::saveBeforeUnwatch() {
         const auto spanBlocks = static_cast<std::size_t>((end - spanBegin + block - 1) / block);
         const std::vector<std::uint64_t> generations(spanBlocks, layerGeneration[unit]);
         std::vector<std::uint8_t> changedBlocks(spanBlocks);
-        if (!GuestMemory::ChangedBlocks(begin, static_cast<std::size_t>(end - begin), generations, changedBlocks)) continue;
+        if (!GuestMemory::ChangedBlocks(begin, static_cast<std::size_t>(end - begin), generations, changedBlocks)) {
+            if (traceWatch) std::fprintf(stderr, "[watch] save image 0x%llx: unit %u 0x%llx+0x%llx skipped, blocks not tracked\n", static_cast<unsigned long long>(descriptor.baseAddress), unit, static_cast<unsigned long long>(begin), static_cast<unsigned long long>(end - begin));
+            continue;
+        }
         forgetKeptAtUnwatch(unit);
         for (auto at = spanBegin; at < end; at += block) {
             const auto from = std::max(at, begin);
@@ -4713,12 +4729,34 @@ void StorageTexture::writeBackLayers(const std::vector<bool>& layers) {
             cancelStoredSave(layer);
         }
     };
+    static const bool traceWatch = std::getenv("APS5_TRACE_WATCH") != nullptr;
+    static std::atomic<int> unsavedReports{0};
+    const auto reportUnsaved = [&](std::uint32_t layer) {
+        if (unsavedReports.fetch_add(1) >= 16) return;
+        const auto begin = layerBegin(layer);
+        const auto end = begin + layerBytes(layer);
+        std::array<std::size_t, 5> sites{};
+        std::size_t none = 0;
+        GuestMemory::ExclusionView firstView;
+        for (auto at = begin & ~(block - 1); at < end; at += block) {
+            const auto view = GuestMemory::ExclusionAt(at);
+            if (at == (begin & ~(block - 1))) firstView = view;
+            if (view.record) ++sites[static_cast<std::size_t>(view.record->site)];
+            else ++none;
+        }
+        std::string counts;
+        for (std::size_t site = 0; site < sites.size(); ++site) counts += std::string(GuestMemory::WatchExitName(static_cast<GuestMemory::WatchExit>(site))) + "=" + std::to_string(sites[site]) + " ";
+        std::string first = "no record";
+        if (firstView.record) first = std::string("site ") + GuestMemory::WatchExitName(firstView.record->site) + " generation " + std::to_string(firstView.record->generation) + " hooked " + (firstView.record->hooked ? "ran" : "none");
+        std::fprintf(stderr, "[watch] unsaved image 0x%llx+0x%llx blockUnits %d unit %u 0x%llx+0x%llx layerGeneration %llu layerPendingGeneration %llu reason %s blocks %sNone=%zu first-block %s host-region %d\n", static_cast<unsigned long long>(descriptor.baseAddress), static_cast<unsigned long long>(guestBytes), blockUnits ? 1 : 0, layer, static_cast<unsigned long long>(begin), static_cast<unsigned long long>(end - begin), static_cast<unsigned long long>(layerGeneration[layer]), static_cast<unsigned long long>(layer < layerPendingGeneration.size() ? layerPendingGeneration[layer] : 0), flushReason != nullptr ? flushReason : "none", counts.c_str(), none, first.c_str(), firstView.hostRegion ? 1 : 0);
+    };
     bool unsavedOutsideWatch = false;
     for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
         if (!layers[layer]) continue;
         const auto begin = layerBegin(layer);
         const auto end = begin + layerBytes(layer);
         if (tracked ? layerGeneration[layer] == 0 : !compared[layer]) {
+            if (traceWatch && !tracked) reportUnsaved(layer);
             unsavedOutsideWatch = unsavedOutsideWatch || !tracked;
             keep.emplace_back(begin, end);
             continue;