fix(agc): drop resource-cache entries that hold released sampled textures

core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp

@@ -216,6 +216,7 @@ class ShaderResources {
     std::vector<std::pair<std::uint64_t, std::uint64_t>> DeviceReads() const { return guestMemory.DeviceReads(); }
     bool ReadsImage(const StorageTexture* image) const;
     bool HasCachedStorageImages() const;
+    bool HoldsReleasedTexture() const;
     std::uint32_t TableElements() const { return tableElements[0] + tableElements[1]; }
     const std::vector<std::uint32_t>& LayoutKey() const { return layoutKey; }
     const std::vector<std::uint32_t>& FragmentLayoutKey() const { return fragmentLayoutKey; }
@@ -665,6 +666,7 @@ class ResourceCache {
 private:
     void erase(const Key& key);
     void noteMiss(const Key& key);
+    void dropReleasedTextures(std::vector<std::shared_ptr<ShaderResources>>* evicted);
     void noteEvicted(const Key& key);
     void noteReturn(const Key& key);
     void noteUse(const Key& key);

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -110,6 +110,8 @@ class Texture {
     PerUseCopy CopyKind() const { return perUseCopy; }
     bool RefreshedPerUse() const { return perUseCopy == PerUseCopy::RemadePerUse; }
     void MarkRefreshedPerUse() { perUseCopy = PerUseCopy::RemadePerUse; }
+    bool Released() const { return released.load(std::memory_order_acquire); }
+    void MarkReleased() { released.store(true, std::memory_order_release); }
 
 private:
     void release() noexcept;
@@ -143,6 +145,7 @@ class Texture {
     std::shared_ptr<StorageTexture> storageSource;
     std::unique_ptr<CommandBatch> upload;
     PerUseCopy perUseCopy = PerUseCopy::None;
+    std::atomic<bool> released{false};
     bool copiedOnce = false;
     std::uint64_t copiedVersion = 0;
     VkFormat depthFormat = VK_FORMAT_UNDEFINED;

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -194,6 +194,16 @@ void noteUncached(const std::shared_ptr<StorageTexture>& image) {
     uncached.any.store(true, std::memory_order_release);
 }
 
+std::atomic<bool>& TexturesReleased() {
+    static std::atomic<bool> released{false};
+    return released;
+}
+
+void noteReleased(const std::shared_ptr<Texture>& texture) {
+    texture->MarkReleased();
+    TexturesReleased().store(true, std::memory_order_release);
+}
+
 bool SamplerCacheDisabled() {
     static const bool disabled = std::getenv("APS5_NO_SAMPLER_CACHE") != nullptr;
     return disabled;
@@ -225,6 +235,7 @@ void eraseTexture(TextureCache& cache, std::list<CachedTexture>::iterator it) {
             }
         }
     }
+    if (it->texture.use_count() > 1) noteReleased(it->texture);
     cache.index.erase(it->key);
     cache.entries.erase(it);
     ++cache.erasures;
@@ -827,6 +838,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
     const auto budget = sampledBudget(context, cache);
     auto texture = entry.texture;
     if (!admitTexture(cache, key, entry.accounted, budget)) {
+        noteReleased(texture);
         logLookup({texture.get(), resource, guestBytes, *keys, generation, source.get()});
         reportCacheCounts(cache, budget);
         reportTextureCounters();
@@ -3482,7 +3494,8 @@ void ResourceCache::noteMiss(const Key& key) {
 
 void ResourceCache::Insert(const Key& key, std::shared_ptr<ShaderResources> resources, std::vector<std::shared_ptr<ShaderResources>>* evicted) {
     std::lock_guard lock(mutex);
-    if (!resources->HasCachedStorageImages()) return;
+    dropReleasedTextures(evicted);
+    if (!resources->HasCachedStorageImages() || resources->HoldsReleasedTexture()) return;
     if (const auto found = index.find(key); found != index.end()) {
         if (evicted != nullptr) evicted->push_back(std::move(found->second->second));
         entries.erase(found->second);
@@ -3541,6 +3554,19 @@ void ResourceCache::RemoveImage(const StorageTexture* image, std::vector<std::sh
     }
 }
 
+void ResourceCache::dropReleasedTextures(std::vector<std::shared_ptr<ShaderResources>>* evicted) {
+    if (!TexturesReleased().exchange(false, std::memory_order_acq_rel)) return;
+    for (auto it = entries.begin(); it != entries.end();) {
+        if (!it->second->HoldsReleasedTexture()) {
+            ++it;
+            continue;
+        }
+        if (evicted != nullptr) evicted->push_back(std::move(it->second));
+        index.erase(it->first);
+        it = entries.erase(it);
+    }
+}
+
 bool ResourceCache::Touch(const Key& key) {
     resourceCacheTouches.fetch_add(1, std::memory_order_relaxed);
     std::lock_guard lock(mutex);
@@ -5119,6 +5145,10 @@ bool ShaderResources::HasCachedStorageImages() const {
     return std::none_of(textures.begin(), textures.end(), [](const auto& texture) { return texture != nullptr && texture->StorageSource() != nullptr && !texture->StorageSource()->Cached(); });
 }
 
+bool ShaderResources::HoldsReleasedTexture() const {
+    return std::any_of(textures.begin(), textures.end(), [](const auto& texture) { return texture != nullptr && texture->Released(); });
+}
+
 bool ShaderResources::ReadsImage(const StorageTexture* image) const {
     if (image == nullptr) return false;
     for (const auto& texture : textures) {

core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp

@@ -469,6 +469,48 @@ void heldTests(const Context& context) {
     Require(TextureCacheUsage().sampledEntries == capacity && TextureCacheUsage().sampledBytes <= Budget, "released textures were not evicted down to the budget");
 }
 
+void releasedTests(Context context) {
+    PlaceholderImages placeholders(context);
+    context.placeholderImages = &placeholders;
+    TextureCache textures(context);
+    context.textureCache = &textures;
+    ClearCachedTextures(context.device);
+    auto& cache = SharedResourceCache();
+    cache.Clear();
+    try {
+        Block block(context);
+        auto pinned = ImageResources(context, block, 0, 1, true);
+        Require(pinned->SampledTextures().size() == 1 && !pinned->SampledTextures().front()->ViewsStorageImage(), "the pinned descriptor set does not hold a sampled snapshot");
+        const std::weak_ptr<Texture> texture = pinned->SampledTextures().front();
+        const std::weak_ptr<ShaderResources> pinnedSet = pinned;
+        cache.Insert({1}, pinned);
+        Require(cache.Find({1}) == pinned, "the resource cache rejected a descriptor set whose texture is cached");
+        pinned.reset();
+        ClearCachedTextures(context.device);
+        Require(!texture.expired() && texture.lock()->Released(), "the sampled texture cache dropped a texture the resource cache holds without marking it released");
+        auto live = ImageResources(context, block, 1, 1, true);
+        std::vector<std::shared_ptr<ShaderResources>> displaced;
+        cache.Insert({2}, live, &displaced);
+        Require(cache.Find({1}) == nullptr && cache.Find({2}) == live && cache.Size() == 1, "the resource cache kept an entry holding a released texture");
+        Require(displaced.size() == 1 && displaced.front() == pinnedSet.lock(), "the entry holding a released texture was not handed back to the caller");
+        displaced.clear();
+        Require(pinnedSet.expired() && texture.expired(), "the resource cache kept a released texture alive");
+        auto stale = ImageResources(context, block, 2, 1, true);
+        ClearCachedTextures(context.device);
+        Require(stale->SampledTextures().front()->Released(), "a held texture the sampled cache dropped is not marked released");
+        cache.Insert({3}, stale, &displaced);
+        Require(cache.Find({3}) == nullptr, "a descriptor set holding a released texture entered the resource cache");
+        Require(cache.Size() == 0 && displaced.size() == 1 && displaced.front() == live, "the live entry whose texture was released stayed in the resource cache");
+        displaced.clear();
+    } catch (...) {
+        cache.Clear();
+        throw;
+    }
+    cache.Clear();
+    ClearCachedTextures(context.device);
+    std::puts("resource cache released texture tests passed");
+}
+
 void rescanTests(const Context& context) {
     constexpr std::uint32_t SmallSide = 128;
     constexpr std::size_t SmallBytes = std::size_t{SmallSide} * SmallSide * 4u;
@@ -713,6 +755,7 @@ int main(int argc, char** argv) {
             reportedBudgetTests(context);
             snapshotTests(context);
             heldTests(context);
+            releasedTests(context);
             rescanTests(context);
             viewEvictionTests(context);
             viewTests(context);

docs/dev/debt/libSceAgcDriver.md

@@ -172,6 +172,7 @@
 - [Flip packets raise the graphics end-of-pipe event](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) (libSceAgcDriver) - a flip delivers the queue's end-of-pipe event (`sceAgcDriverAddEqEvent`) once the work recorded before it completes, as a `RELEASE_MEM` with `INT_SEL` does. Minecraft (PPSA17221) registers the graphics event, never requests an interrupt in its releases, and waits on the event after every flip while holding its render lock, so without this it never presents a second frame. Whether the console raises the event from the flip itself or from an interrupt its flip sequence contains is not measured; a title that waits only on release interrupts sees one extra event per flip, with `data` counting it.
 - [Host import windows and budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): a registered range larger than 64 MiB is imported one 64 MiB window at a time for buffer and fill accesses; an access across window edges imports the cells it touches, grown to the union of the range's imports it overlaps. An address-based build (`AcquireRegistered`) imports the range whole and replaces its windows. Live imports stay under a budget: `APS5_HOST_IMPORT_MIB`, else the larger of 6 GiB and half the physical memory, at most the largest host heap less 2 GiB (imports and type-3 mirrors share it; #167's rule). A new import beyond it retires the least recently used imports not used in the last 256 lookups; one that would not fit even then is refused before anything is retired, and imports made when a range is mapped never evict. A refused import is tried again at its next use. The 64 MiB window, the 256-lookup protection and the 2 GiB reserve are choices, not measured.
 - [Registration preparation at submit](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) - `Driver::Submit` waits for every registration-time shader preparation queued before the call, not only those of the shaders the commands use, so a submission behind a long queue of unrelated preparations waits for all of them on the submitting thread. Waiting there instead of in the queue worker keeps the GPU side of a submission short: a title that sees a submission take seconds treats the GPU as hung (Goat Simulator 3, PPSA04159, then stops submitting and its garbage collector fails a 10 s timeout). On the console shaders are compiled ahead of time, so neither wait exists; how long a title tolerates a slow submit call is not known.
+- [Resource cache entries holding released textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a sampled texture the texture cache drops while something else holds it, or makes without caching (`admitTexture` refusing a first use), is marked released, and `ResourceCache::Insert` refuses descriptor sets that hold one and removes the entries that do. The removal waits for the next insert: until then `Find` still returns those entries and their textures stay alive. One process-wide flag starts the sweep, so while the sampled cache is over its budget every insert scans all entries (1,024 at start, up to 16,384 after growth, or `APS5_RESOURCE_CACHE_ENTRIES`, each with its textures); that cost is not measured. When `ReleaseTextureCaches` runs on device memory exhaustion, the sampled textures these entries hold are only marked released and are freed at the next insert, so the immediate allocation retry gets no memory back from them. A draw or dispatch whose textures the cache refused builds its descriptor sets again on each use for as long as the cache stays full; the count bounds (`APS5_RESOURCE_CACHE_ENTRIES`, `APS5_RESOURCE_CACHE_TABLE_ELEMENTS`) still bound nothing by bytes, and sampled textures made outside the texture cache (the disabled cache, depth-surface copies) are never marked. Measured on one title on one RTX 4090: with the default bounds, sampled textures held outside the cache peaked at 2,836 MiB (7,830 MiB before) and device-local heap 0 stayed under its 22.9 GB budget for the whole run (26,357 MiB before, which aborted with `VK_ERROR_OUT_OF_DEVICE_MEMORY` at the wolf scene). Whether the extra descriptor-set builds cost frame time elsewhere is not measured.
 - [Unclamped normalized color targets](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - CB_COLOR_INFO targets with BLEND_CLAMP clear (bit 15) on UNORM and SNORM formats are accepted when blending is disabled (ENABLE clear or BLEND_BYPASS set). Vulkan converts unblended shader values and clamps to the normalized range automatically; blending into an unclamped normalized target remains unmodeled because Vulkan always clamps inputs prior to blending.
 
 - [Thick SW_4KB_S_X volumes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp) use AMD addrlib's GFX10 non-RB+ 16-pipe `GFX10_SW_4K_S3_X_PATINFO` equations for 1, 2, 4, 8 and 16-byte elements, including packed mip tails. The layout follows [AMD PAL c5e80007](https://github.com/GPUOpen-Drivers/pal/blob/c5e800072a32f68b6ccc4422936d96167c6e0728/src/core/imported/addrlib/src/gfx10/gfx10addrlib.cpp); it has not been measured on console hardware. Other thick swizzle modes outside the supported families still throw. Mipmapped 3D color targets remain refused by the color-register decoder.