perf(agc): bypass the draw cache while its hit rate is under 5%

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -95,6 +95,7 @@ struct DrawEntryCounters {
     std::array<std::uint64_t, static_cast<std::size_t>(DrawMiss::Count)> misses{};
     std::array<std::uint64_t, MaxDispatchVariants> variantHitsByRank{};
     double validateUs = 0;
+    std::uint64_t bypassed = 0;
 
     std::uint64_t registerKeyLookups = 0, registerKeyHits = 0, decodeSkipped = 0, decodePartial = 0, facadeMismatches = 0, verifyDecodes = 0, verifyDecodeMismatches = 0;
     double keyUs = 0;
@@ -111,8 +112,11 @@ class DrawKeySightings {
     static constexpr std::size_t Ways = 4;
     explicit DrawKeySightings(std::size_t keys);
     bool SeenBefore(std::uint64_t key);
+    bool Holds(std::uint64_t key) const;
 
 private:
+    std::size_t bucketOf(std::uint64_t key) const;
+
     std::vector<std::uint64_t> keys;
     std::vector<std::uint8_t> filled;
     std::vector<std::uint8_t> next;
@@ -163,6 +167,25 @@ struct DrawRelocationCounters {
     std::uint64_t lookups = 0, donors = 0, refused = 0, hits = 0, evicted = 0;
 };
 
+class DrawCacheBypass {
+public:
+    static constexpr std::uint32_t Window = 2048, LowWindowsToBypass = 2, FirstBypassDraws = 2048, BypassDraws = 16384;
+    explicit DrawCacheBypass(std::uint32_t percent);
+    std::uint32_t Percent() const { return percent; }
+    bool Skip();
+    void Note(bool useful);
+    std::uint64_t TakeBypassed();
+
+private:
+    static constexpr std::uint32_t CountBits = 12, LowShift = 2 * CountBits, LowBits = 3, LeftShift = 32;
+    static constexpr std::uint64_t CountMask = (std::uint64_t{1} << CountBits) - 1u, LowMask = (std::uint64_t{1} << LowBits) - 1u;
+    static_assert(Window <= CountMask + 1u && LowWindowsToBypass + 3u <= LowMask && LowShift + LowBits <= LeftShift);
+
+    std::uint32_t percent, minimumHits;
+    std::atomic<std::uint64_t> state{0};
+    std::atomic<std::uint64_t> bypassed{0};
+};
+
 bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry& registry);
 
 }

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -175,6 +175,7 @@ class Driver {
     void dispatch(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::uint64_t indirectArguments = 0);
     void verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, ShaderRecompiler::RecompileRequest request, std::span<const ShaderRecompiler::MemoryRegion> memory, std::uint64_t address, const DispatchVariant& variant, std::span<const std::uint32_t> liveWords, const ShaderRecompiler::RecompileResult& patched);
     static bool drawEntries();
+    static std::uint32_t drawCacheBypassPercent();
     static bool drawDataHits();
     static bool verifyDrawDataHits();
     static bool drawStageDataHits();
@@ -186,6 +187,7 @@ class Driver {
     void accountDrawVariant(const DispatchVariant& variant, bool added);
     void insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::shared_ptr<const DrawDecode> decode, std::uint64_t shape = 0, bool replaceShape = false);
     bool admitDrawKey(std::uint64_t key, std::uint64_t shape);
+    bool drawKeySeen(std::uint64_t key);
     void eraseDrawEntry(std::uint64_t key, const std::shared_ptr<DrawEntry>& expected);
     std::shared_ptr<const DrawRecipe> findDrawRecipe(std::uint64_t key, const std::vector<std::shared_ptr<DispatchVariant>>& stages);
     void attachDrawRecipe(std::uint64_t key, const std::vector<std::shared_ptr<DispatchVariant>>& stages, std::shared_ptr<const DrawRecipe> recipe);
@@ -437,6 +439,7 @@ class Driver {
     DrawRelocationPending drawRelocationPending;
     std::unordered_map<std::uint64_t, DrawRelocationTrial> drawRelocationTrials;
     DrawRelocationCounters drawRelocationCounters;
+    DrawCacheBypass drawCacheBypass{drawCacheBypassPercent()};
     AgcDriver::Mutex drawCacheMutex;
     std::uint64_t drawCacheHits = 0, drawCacheEvictions = 0, drawCacheVariants = 0, drawCacheVariantBytes = 0;
     std::optional<DrawKeySightings> drawKeySightings;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -386,7 +386,8 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
     timing.Mark("device_setup");
     phaseTiming.Phase(DrawRowVectors);
 
-    const bool useDrawEntries = drawEntries() && !ShaderRecompiler::DebugProbeActive() && dumpTarget == 0 && dumpSlot1 == 0;
+    const bool bypassed = drawEntries() && drawCacheBypass.Skip();
+    const bool useDrawEntries = drawEntries() && !bypassed && !ShaderRecompiler::DebugProbeActive() && dumpTarget == 0 && dumpSlot1 == 0;
     tail.registerKey = useDrawEntries && registerKeyEnabled();
     const bool registerKey = tail.registerKey;
     auto& drawKey = tail.drawKey;
@@ -656,6 +657,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
         TraceDrawAttemptException(traceToken, "lookup", submission.serial, submission.queue, packetOffset, std::current_exception());
         throw;
     }
+    if (stageEntries) drawCacheBypass.Note(drawHit || verifyHit || std::any_of(dataCandidates.begin(), dataCandidates.end(), [](const auto& candidates) { return !candidates.empty(); }) || drawKeySeen(drawKey));
     for (std::size_t i = 0; i < matched.size(); ++i) {
         if (matched[i] == nullptr || (matched[i]->shaderDataPatches.empty() && matched[i]->descriptorPatches.empty())) continue;
         applyShaderDataPatches(*matched[i]);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp

@@ -26,6 +26,48 @@ bool Driver::drawEntries() {
     return entries;
 }
 
+std::uint32_t Driver::drawCacheBypassPercent() {
+    static const std::uint32_t percent = [] {
+        const char* text = std::getenv("APS5_DRAW_CACHE_BYPASS_PERCENT");
+        return text != nullptr ? static_cast<std::uint32_t>(std::min<unsigned long long>(std::strtoull(text, nullptr, 10), 100)) : 5u;
+    }();
+    return percent;
+}
+
+DrawCacheBypass::DrawCacheBypass(std::uint32_t percent) : percent(std::min(percent, 100u)), minimumHits((Window * std::min(percent, 100u) + 99u) / 100u) {}
+
+bool DrawCacheBypass::Skip() {
+    auto current = state.load(std::memory_order_relaxed);
+    while ((current >> LeftShift) != 0) {
+        if (state.compare_exchange_weak(current, current - (std::uint64_t{1} << LeftShift), std::memory_order_relaxed)) {
+            bypassed.fetch_add(1, std::memory_order_relaxed);
+            return true;
+        }
+    }
+    return false;
+}
+
+std::uint64_t DrawCacheBypass::TakeBypassed() {
+    return bypassed.exchange(0, std::memory_order_relaxed);
+}
+
+void DrawCacheBypass::Note(bool useful) {
+    if (minimumHits == 0) return;
+    auto current = state.load(std::memory_order_relaxed);
+    std::uint64_t next = 0;
+    do {
+        const auto lookups = (current & CountMask) + 1u;
+        const auto usefulLookups = ((current >> CountBits) & CountMask) + (useful ? 1u : 0u);
+        if (lookups < Window) {
+            next = (current & ~((CountMask << CountBits) | CountMask)) | (usefulLookups << CountBits) | lookups;
+            continue;
+        }
+        const auto low = usefulLookups >= minimumHits ? 0u : std::min<std::uint64_t>(((current >> LowShift) & LowMask) + 1u, LowWindowsToBypass + 3u);
+        const auto remaining = low >= LowWindowsToBypass ? std::uint64_t{std::min(BypassDraws, FirstBypassDraws << (low - LowWindowsToBypass))} : current >> LeftShift;
+        next = (remaining << LeftShift) | (low << LowShift);
+    } while (!state.compare_exchange_weak(current, next, std::memory_order_relaxed));
+}
+
 bool Driver::drawDataHits() {
     static const bool hits = std::getenv("APS5_NO_DRAW_DATA_HITS") == nullptr;
     return hits;
@@ -98,12 +140,16 @@ DrawKeySightings::DrawKeySightings(std::size_t keyCount) {
     bucketMask = buckets - 1;
 }
 
-bool DrawKeySightings::SeenBefore(std::uint64_t key) {
+std::size_t DrawKeySightings::bucketOf(std::uint64_t key) const {
     auto mixed = key;
     mixed = (mixed ^ (mixed >> 30u)) * 0xbf58476d1ce4e5b9ull;
     mixed = (mixed ^ (mixed >> 27u)) * 0x94d049bb133111ebull;
     mixed ^= mixed >> 31u;
-    const auto bucket = static_cast<std::size_t>(mixed) & bucketMask;
+    return static_cast<std::size_t>(mixed) & bucketMask;
+}
+
+bool DrawKeySightings::SeenBefore(std::uint64_t key) {
+    const auto bucket = bucketOf(key);
     auto* ways = keys.data() + bucket * Ways;
     if (std::find(ways, ways + filled[bucket], key) != ways + filled[bucket]) return true;
     ways[next[bucket]] = key;
@@ -112,6 +158,17 @@ bool DrawKeySightings::SeenBefore(std::uint64_t key) {
     return false;
 }
 
+bool DrawKeySightings::Holds(std::uint64_t key) const {
+    const auto bucket = bucketOf(key);
+    const auto* ways = keys.data() + bucket * Ways;
+    return std::find(ways, ways + filled[bucket], key) != ways + filled[bucket];
+}
+
+bool Driver::drawKeySeen(std::uint64_t key) {
+    std::lock_guard cacheLock(drawCacheMutex);
+    return drawKeySightings && drawKeySightings->Holds(key);
+}
+
 bool Driver::admitDrawKey(std::uint64_t key, std::uint64_t shape) {
     std::lock_guard cacheLock(drawCacheMutex);
     if (drawCache.contains(key)) return true;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCacheStatistics.cpp

@@ -37,6 +37,7 @@ void Driver::reportDrawCache(DrawEntryCounters& counters, bool profile) {
         std::snprintf(line, sizeof(line), "[draw-cache] register-key miss component changes (10 s): stable registers %llu, user data %llu, auxiliary registers %llu, device %llu, first shader set %llu, unchanged components %llu\n", count(counters.registerKeyMissChanges[0]), count(counters.registerKeyMissChanges[1]), count(counters.registerKeyMissChanges[2]), count(counters.registerKeyMissChanges[3]), count(counters.registerKeyMissChanges[4]), count(counters.registerKeyMissChanges[5]));
         text += line;
     }
+    counters.bypassed = drawCacheBypass.TakeBypassed();
     if (profile) text += profileDrawCacheLines(counters);
     AgcDriver::ProfilePrint_nid_no_patch("%s", text.c_str());
     counters = DrawEntryCounters{};
@@ -77,6 +78,7 @@ std::string Driver::profileDrawCacheLines(const DrawEntryCounters& counters) {
     }
     append("[draw-cache] %llu lookups (10 s): %llu no entry, %llu validated in %.1f us each (key, lookup and every stage): %llu hits (every stage equal; stage hits by variant rank 1..k %s), misses by reason: front stage differing %llu, fragment differing %llu, other stage differing %llu, layout (push offset) %llu, gate %llu, stage count %llu; %llu stage validations (%llu equal, %.2f variants compared each); %llu inserts (%llu shape updates, %llu variants inserted, %llu evicted beyond k, %llu already present, %llu unstable), %zu entries (%llu variants, ~%.1f MiB of a %.1f MiB budget), %llu evictions in total (%llu over the byte budget), %llu LRU moves; verify: %llu hits captured again, %llu stages differed\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast<double>(validated) : 0.0, count(counters.hits), ranks.c_str(), miss(DrawMiss::FrontDiffering), miss(DrawMiss::FragmentDiffering), miss(DrawMiss::OtherDiffering), miss(DrawMiss::Layout), miss(DrawMiss::Gate), miss(DrawMiss::Stages), count(counters.stageValidations), count(counters.stageEqual), counters.stageValidations != 0 ? static_cast<double>(counters.variantsCompared) / static_cast<double>(counters.stageValidations) : 0.0, count(counters.inserts), count(counters.shapeUpdates), count(counters.variantsInserted), count(counters.variantsEvicted), count(counters.present), count(counters.unstable), drawCache.size(), count(drawCacheVariants), static_cast<double>(drawCacheVariantBytes) / (1024.0 * 1024.0), static_cast<double>(DrawCacheBytes()) / (1024.0 * 1024.0), count(drawCacheEvictions), count(drawCacheByteEvictions), count(counters.touches), count(counters.verifyHits), count(counters.verifyMismatches));
     append("[draw-cache] register key (10 s): %llu lookups, %llu hits, key build %.1f us each; decode skipped %llu, partial (state/pixel/programs from the entry) %llu; facade log != table %llu; verify: %llu decodes compared, %llu differed\n", count(counters.registerKeyLookups), count(counters.registerKeyHits), counters.registerKeyLookups != 0 ? counters.keyUs / static_cast<double>(counters.registerKeyLookups) : 0.0, count(counters.decodeSkipped), count(counters.decodePartial), count(counters.facadeMismatches), count(counters.verifyDecodes), count(counters.verifyDecodeMismatches));
+    if (counters.bypassed != 0) append("[draw-cache] bypassed %llu draws (10 s): useful lookups stayed under %u%% for two or more windows in a row, entries are neither looked up nor inserted\n", count(counters.bypassed), drawCacheBypass.Percent());
     append("[draw-cache] stage data hits (10 s): %llu stages (%llu words refreshed), verified %llu; stage inserts with data positions %llu (%.1f positions each)\n", count(counters.stageDataHits), count(counters.stageDataWords), count(counters.stageDataVerified), count(counters.stageDataInserts), counters.stageDataInserts != 0 ? static_cast<double>(counters.stageDataPositions) / static_cast<double>(counters.stageDataInserts) : 0.0);
     return text;
 }

core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_reuse_tests.cmake

@@ -11,6 +11,18 @@ if(BUILD_TESTING)
         set_tests_properties(agc_driver_draw_cache_reuse_entry_verify PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
+    add_test(NAME agc_driver_draw_cache_low_hit_bypass COMMAND agc_driver_draw_cache_reuse_tests --low-hit-bypass)
+    set_tests_properties(agc_driver_draw_cache_low_hit_bypass PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    add_test(NAME agc_driver_draw_cache_low_hit_bypass_off COMMAND agc_driver_draw_cache_reuse_tests --low-hit-bypass)
+    set_tests_properties(agc_driver_draw_cache_low_hit_bypass_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_CACHE_BYPASS_PERCENT=0" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    add_test(NAME agc_driver_draw_cache_cold_scene COMMAND agc_driver_draw_cache_reuse_tests --cold-scene)
+    set_tests_properties(agc_driver_draw_cache_cold_scene PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    add_test(NAME agc_driver_draw_cache_cold_scene_off COMMAND agc_driver_draw_cache_reuse_tests --cold-scene)
+    set_tests_properties(agc_driver_draw_cache_cold_scene_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_CACHE_BYPASS_PERCENT=0" SKIP_RETURN_CODE 77 FAIL_REGULAR_EXPRESSION "FATAL:")
+    if(WIN32)
+        set_tests_properties(agc_driver_draw_cache_low_hit_bypass agc_driver_draw_cache_low_hit_bypass_off agc_driver_draw_cache_cold_scene agc_driver_draw_cache_cold_scene_off PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
     add_test(NAME agc_driver_draw_cache_reuse_inline COMMAND agc_driver_draw_cache_reuse_tests)
     set_tests_properties(agc_driver_draw_cache_reuse_inline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=0;APS5_VERIFY_DRAW_RECIPE=1;APS5_VERIFY_REGISTER_KEYS=1" SKIP_RETURN_CODE 77)
     add_test(NAME agc_driver_draw_cache_reuse_indirect COMMAND agc_driver_draw_cache_reuse_tests)