perf(agc): bypass the draw cache while its hit rate is under 5%

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -3309,6 +3309,14 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_draw_cache_reuse_entry_verify PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_draw_cache_low_hit_bypass COMMAND agc_driver_draw_cache_reuse_tests --low-hit-bypass)
+    set_tests_properties(agc_driver_draw_cache_low_hit_bypass PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    add_test(NAME agc_driver_draw_cache_low_hit_bypass_off COMMAND agc_driver_draw_cache_reuse_tests --low-hit-bypass)
+    set_tests_properties(agc_driver_draw_cache_low_hit_bypass_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_CACHE_BYPASS_PERCENT=0" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    if(WIN32)
+        set_tests_properties(agc_driver_draw_cache_low_hit_bypass agc_driver_draw_cache_low_hit_bypass_off PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_draw_cache_reuse_inline COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_inline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0;APS5_VERIFY_DRAW_RECIPE=1;APS5_VERIFY_REGISTER_KEYS=1" SKIP_RETURN_CODE 77)
     set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_inline PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:")

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -95,6 +95,7 @@ struct DrawEntryCounters {
     std::array<std::uint64_t, static_cast<std::size_t>(DrawMiss::Count)> misses{};
     std::array<std::uint64_t, MaxDispatchVariants> variantHitsByRank{};
     double validateUs = 0;
+    std::uint64_t bypassed = 0;
 
     std::uint64_t registerKeyLookups = 0, registerKeyHits = 0, decodeSkipped = 0, decodePartial = 0, facadeMismatches = 0, verifyDecodes = 0, verifyDecodeMismatches = 0;
     double keyUs = 0;
@@ -161,6 +162,19 @@ struct DrawRelocationCounters {
     std::uint64_t lookups = 0, donors = 0, refused = 0, hits = 0, evicted = 0;
 };
 
+class DrawCacheBypass {
+public:
+    static constexpr std::uint32_t Window = 2048, BypassDraws = 16384;
+    explicit DrawCacheBypass(std::uint32_t percent);
+    std::uint32_t Percent() const { return percent; }
+    bool Skip();
+    void Note(bool hit);
+
+private:
+    std::uint32_t percent, minimumHits;
+    std::atomic<std::uint32_t> lookups{0}, hits{0}, left{0};
+};
+
 bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry& registry);
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -166,6 +166,7 @@ class Driver {
     void dispatch(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::uint64_t indirectArguments = 0);
     void verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, ShaderRecompiler::RecompileRequest request, std::span<const ShaderRecompiler::MemoryRegion> memory, std::uint64_t address, const DispatchVariant& variant, std::span<const std::uint32_t> liveWords, const ShaderRecompiler::RecompileResult& patched);
     static bool drawEntries();
+    static std::uint32_t drawCacheBypassPercent();
     static bool drawDataHits();
     static bool verifyDrawDataHits();
     static bool drawStageDataHits();
@@ -427,6 +428,7 @@ class Driver {
     DrawRelocationPending drawRelocationPending;
     std::unordered_map<std::uint64_t, DrawRelocationTrial> drawRelocationTrials;
     DrawRelocationCounters drawRelocationCounters;
+    DrawCacheBypass drawCacheBypass{drawCacheBypassPercent()};
     AgcDriver::Mutex drawCacheMutex;
     std::uint64_t drawCacheHits = 0, drawCacheEvictions = 0, drawCacheVariants = 0, drawCacheVariantBytes = 0;
     std::optional<DrawKeySightings> drawKeySightings;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -386,7 +386,12 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     timing.Mark("device_setup");
     phaseTiming.Phase(DrawRowVectors);
 
-    const bool useDrawEntries = drawEntries() && !ShaderRecompiler::DebugProbeActive() && dumpTarget == 0 && dumpSlot1 == 0;
+    const bool bypassed = drawEntries() && drawCacheBypass.Skip();
+    if (bypassed) {
+        std::lock_guard cacheLock(drawCacheMutex);
+        ++drawEntryCounters.bypassed;
+    }
+    const bool useDrawEntries = drawEntries() && !bypassed && !ShaderRecompiler::DebugProbeActive() && dumpTarget == 0 && dumpSlot1 == 0;
     tail.registerKey = useDrawEntries && registerKeyEnabled();
     const bool registerKey = tail.registerKey;
     auto& drawKey = tail.drawKey;
@@ -656,6 +661,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         TraceDrawAttemptException(traceToken, "lookup", submission.serial, submission.queue, packetOffset, std::current_exception());
         throw;
     }
+    if (stageEntries) drawCacheBypass.Note(drawHit || verifyHit);
     for (std::size_t i = 0; i < matched.size(); ++i) {
         if (matched[i] == nullptr || matched[i]->shaderDataPatches.empty()) continue;
         applyShaderDataPatches(*matched[i]);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp

@@ -26,6 +26,33 @@ bool Driver::drawEntries() {
     return entries;
 }
 
+std::uint32_t Driver::drawCacheBypassPercent() {
+    static const std::uint32_t percent = [] {
+        const char* text = std::getenv("APS5_DRAW_CACHE_BYPASS_PERCENT");
+        return text != nullptr ? static_cast<std::uint32_t>(std::min<unsigned long long>(std::strtoull(text, nullptr, 10), 100)) : 5u;
+    }();
+    return percent;
+}
+
+DrawCacheBypass::DrawCacheBypass(std::uint32_t percent) : percent(std::min(percent, 100u)), minimumHits(Window * std::min(percent, 100u) / 100u) {}
+
+bool DrawCacheBypass::Skip() {
+    auto remaining = left.load(std::memory_order_relaxed);
+    while (remaining != 0) {
+        if (left.compare_exchange_weak(remaining, remaining - 1, std::memory_order_relaxed)) return true;
+    }
+    return false;
+}
+
+void DrawCacheBypass::Note(bool hit) {
+    if (minimumHits == 0) return;
+    if (hit) hits.fetch_add(1, std::memory_order_relaxed);
+    if (lookups.fetch_add(1, std::memory_order_relaxed) + 1 < Window) return;
+    const auto windowHits = hits.exchange(0, std::memory_order_relaxed);
+    lookups.store(0, std::memory_order_relaxed);
+    if (windowHits < minimumHits) left.store(BypassDraws, std::memory_order_relaxed);
+}
+
 bool Driver::drawDataHits() {
     static const bool hits = std::getenv("APS5_NO_DRAW_DATA_HITS") == nullptr;
     return hits;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCacheStatistics.cpp

@@ -77,6 +77,7 @@ std::string Driver::profileDrawCacheLines(const DrawEntryCounters& counters) {
     }
     append("[draw-cache] %llu lookups (10 s): %llu no entry, %llu validated in %.1f us each (key, lookup and every stage): %llu hits (every stage equal; stage hits by variant rank 1..k %s), misses by reason: front stage differing %llu, fragment differing %llu, other stage differing %llu, layout (push offset) %llu, gate %llu, stage count %llu; %llu stage validations (%llu equal, %.2f variants compared each); %llu inserts (%llu shape updates, %llu variants inserted, %llu evicted beyond k, %llu already present, %llu unstable), %zu entries (%llu variants, ~%.1f MiB of a %.1f MiB budget), %llu evictions in total (%llu over the byte budget), %llu LRU moves; verify: %llu hits captured again, %llu stages differed\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast<double>(validated) : 0.0, count(counters.hits), ranks.c_str(), miss(DrawMiss::FrontDiffering), miss(DrawMiss::FragmentDiffering), miss(DrawMiss::OtherDiffering), miss(DrawMiss::Layout), miss(DrawMiss::Gate), miss(DrawMiss::Stages), count(counters.stageValidations), count(counters.stageEqual), counters.stageValidations != 0 ? static_cast<double>(counters.variantsCompared) / static_cast<double>(counters.stageValidations) : 0.0, count(counters.inserts), count(counters.shapeUpdates), count(counters.variantsInserted), count(counters.variantsEvicted), count(counters.present), count(counters.unstable), drawCache.size(), count(drawCacheVariants), static_cast<double>(drawCacheVariantBytes) / (1024.0 * 1024.0), static_cast<double>(DrawCacheBytes()) / (1024.0 * 1024.0), count(drawCacheEvictions), count(drawCacheByteEvictions), count(counters.touches), count(counters.verifyHits), count(counters.verifyMismatches));
     append("[draw-cache] register key (10 s): %llu lookups, %llu hits, key build %.1f us each; decode skipped %llu, partial (state/pixel/programs from the entry) %llu; facade log != table %llu; verify: %llu decodes compared, %llu differed\n", count(counters.registerKeyLookups), count(counters.registerKeyHits), counters.registerKeyLookups != 0 ? counters.keyUs / static_cast<double>(counters.registerKeyLookups) : 0.0, count(counters.decodeSkipped), count(counters.decodePartial), count(counters.facadeMismatches), count(counters.verifyDecodes), count(counters.verifyDecodeMismatches));
+    if (counters.bypassed != 0) append("[draw-cache] bypassed %llu draws (10 s): the hit rate stayed under %u%%, entries are neither looked up nor inserted\n", count(counters.bypassed), drawCacheBypass.Percent());
     append("[draw-cache] stage data hits (10 s): %llu stages (%llu words refreshed), verified %llu; stage inserts with data positions %llu (%.1f positions each)\n", count(counters.stageDataHits), count(counters.stageDataWords), count(counters.stageDataVerified), count(counters.stageDataInserts), counters.stageDataInserts != 0 ? static_cast<double>(counters.stageDataPositions) / static_cast<double>(counters.stageDataInserts) : 0.0);
     return text;
 }

core/libs/prx/libSceAgcDriver/tests/execution/DrawCacheReuse.cpp

@@ -241,9 +241,68 @@ void SightingsKeepCollidingKeys() {
     Require(sightings.SeenBefore(12) && sightings.SeenBefore(13) && sightings.SeenBefore(14), "a full bucket lost a key newer than the one it replaced");
 }
 
+std::vector<std::uint32_t> ScaledDraws(const std::vector<float>& scales) {
+    auto commands = Draws(32.0f, VertexCode.data(), 1, PixelCode.data(), 1, {});
+    Append(commands, 0x76, {0x00c, std::bit_cast<std::uint32_t>(1.0f)});
+    for (const auto scale : scales) {
+        Append(commands, 0x69, {0x10f, std::bit_cast<std::uint32_t>(scale)});
+        Append(commands, 0x69, {0x110, std::bit_cast<std::uint32_t>(scale)});
+        Append(commands, 0x2d, {3, 2});
+    }
+    return commands;
+}
+
+std::vector<float> HitWindow(std::size_t draws, float repeated, std::size_t repeats, float& distinct) {
+    std::vector<float> scales(repeats, repeated);
+    while (scales.size() < draws) scales.push_back(distinct += 1.0f);
+    return scales;
+}
+
+void LowHitRateBypassesTheCache() {
+    constexpr std::size_t Window = 2048, BypassDraws = 16384;
+    const char* percent = std::getenv("APS5_DRAW_CACHE_BYPASS_PERCENT");
+    const bool bypass = percent == nullptr || std::strtoul(percent, nullptr, 10) != 0;
+    const auto mode = std::string(bypass ? "with the default 5% bypass" : "with the bypass off");
+    Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2);
+    Registered(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1);
+    Registered(TriangleVertexHeader, TriangleVertexCode.data(), sizeof(TriangleVertexCode), 2);
+    Registered(ConstantPixelHeader, ConstantPixelCode.data(), sizeof(ConstantPixelCode), 1);
+    ColorMemory.fill(std::byte{0});
+    PixelConstant[0] = 0.25f;
+    const auto cached = SubmitConstantDraws(3);
+    Require(cached.lookups == 3 && cached.hits == 1, mode + ": three constant draws did not miss twice and hit once");
+    const auto cachedColor = BrightestColorByte();
+    Require(cachedColor == 64, mode + ": the cached draws of constant 0.25 wrote " + std::to_string(cachedColor) + ", expected 64");
+    float distinct = 1000.0f;
+    const auto even = Submit(ScaledDraws(HitWindow(Window - 3, 32.0f, 103, distinct)));
+    Require(even.lookups == Window && even.hits == 102, mode + ": the first window made " + std::to_string(even.lookups) + " lookups with " + std::to_string(even.hits) + " hits, expected 2048 with 102");
+    const auto armed = Submit(ScaledDraws(HitWindow(Window, 16.0f, 103, distinct)));
+    Require(armed.lookups == 2 * Window && armed.hits == 203, mode + ": a window of 102 hits in 2048 lookups bypassed the draw cache, which needs fewer than 5%");
+    ColorMemory.fill(std::byte{0});
+    const auto bypassed = SubmitConstantDraws(1);
+    const auto bypassedColor = BrightestColorByte();
+    PixelConstant[0] = 1.0f;
+    ColorMemory.fill(std::byte{0});
+    const auto refreshed = SubmitConstantDraws(1);
+    Require(bypassedColor == cachedColor && BrightestColorByte() == 255, mode + ": draws after a window of 101 hits wrote " + std::to_string(bypassedColor) + " and " + std::to_string(BrightestColorByte()) + ", expected the cached " + std::to_string(cachedColor) + " and 255");
+    if (!bypass) {
+        Require(refreshed.lookups == armed.lookups + 2, mode + ": draws after a window of 101 hits were not looked up");
+        return;
+    }
+    Require(bypassed.lookups == armed.lookups && refreshed.lookups == armed.lookups && refreshed.inserts == armed.inserts, mode + ": draws after a window of 101 hits in 2048 lookups were still looked up or inserted");
+    const auto rest = Submit(ScaledDraws(std::vector<float>(BypassDraws - 2, 32.0f)));
+    Require(rest.lookups == armed.lookups && rest.inserts == armed.inserts, mode + ": the bypass ended before 16384 draws");
+    const auto resumed = Submit(ScaledDraws({32.0f}));
+    Require(resumed.lookups == armed.lookups + 1 && resumed.hits == armed.hits + 1, mode + ": the draw after 16384 bypassed draws did not look up the cached draw again");
+    const auto recovered = Submit(ScaledDraws(std::vector<float>(Window - 1, 32.0f)));
+    Require(recovered.hits - resumed.hits == Window - 1, mode + ": a window of repeated draws did not hit the cache");
+    const auto kept = Submit(ScaledDraws(std::vector<float>(10, 32.0f)));
+    Require(kept.lookups == recovered.lookups + 10, mode + ": a window of 2048 hits bypassed the draw cache");
+}
+
 }
 
-int main() {
+int main(int argc, char** argv) {
     try {
         SightingsKeepCollidingKeys();
         bool libraries = false;
@@ -253,7 +312,9 @@ int main() {
             libraries = device->GraphicsPipelineLibraries();
         }
         if (!libraries && std::getenv("APS5_NO_GPL") == nullptr) return VulkanTestSkipped;
-        if (std::getenv("APS5_VERIFY_DRAW_ENTRIES") != nullptr) {
+        if (argc > 1 && std::string(argv[1]) == "--low-hit-bypass") {
+            LowHitRateBypassesTheCache();
+        } else if (std::getenv("APS5_VERIFY_DRAW_ENTRIES") != nullptr) {
             RefreshedStageDataVerifies();
         } else {
             ThirdDrawHitsTheCache(libraries);

docs/dev/ENVIRONMENT.md

@@ -132,6 +132,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_DISPMOVE_MAX_KIB` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | 1024 | upper bound in KiB on the read-only buffers a dispatch-cache hit moves into its template, above which the dispatch is prepared again |
 | `APS5_DRAIN_ALL` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp` |  | drains every packet, disabling the batched recording path |
 | `APS5_DRAIN_COMPLETION_LABELS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp` |  | records completion labels immediately instead of deferring them with the batch, and drains a label recorded while a copied write-back is pending instead of storing it behind the write-backs |
+| `APS5_DRAW_CACHE_BYPASS_PERCENT` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | 5 | when fewer than this percent of a window of 2048 draw-cache lookups hit, the next 16384 draws neither look up nor insert draw entries, then lookups resume and a new window is measured; 0 never bypasses, and values above 100 count as 100 |
 | `APS5_DRAW_CACHE_BYTES` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | 268435456 | size of the draw cache in bytes, evicting least-recently-used entries past it (0 keeps the default of 256 MiB) |
 | `APS5_DRAW_CACHE_ENTRIES` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | 4096 | number of entries in the draw cache; 0 keeps the default 4096 |
 | `APS5_DRAW_DRAIN` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp` |  | treats draw packets as drains, waiting for them instead of recording ahead |