diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 7e7d22b0211d..2647cdce3746 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1489,6 +1489,13 @@ if(BUILD_TESTING) set_tests_properties(agc_import_watch_other_family PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME agc_import_settle_deferred COMMAND agc_driver_recorder_tests --deferred-settle-only) add_test(NAME agc_import_settle_deferred_other_family COMMAND agc_driver_recorder_tests --deferred-settle-other-family-only) + add_test(NAME agc_import_settle_write_order COMMAND agc_driver_recorder_tests --settle-write-order-only) + add_test(NAME agc_import_settle_write_order_other_family COMMAND agc_driver_recorder_tests --settle-write-order-other-family-only) + add_test(NAME agc_import_settle_wait_failure COMMAND agc_driver_recorder_tests --settle-wait-failure-only) + set_tests_properties(agc_import_settle_write_order agc_import_settle_write_order_other_family agc_import_settle_wait_failure PROPERTIES + SKIP_RETURN_CODE 77 TIMEOUT 30 + FAIL_REGULAR_EXPRESSION "SYNC-HAZARD-|VUID-" + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") add_test(NAME agc_import_settle_first_use COMMAND agc_driver_recorder_tests --settle-first-use-only) add_test(NAME agc_import_settle_earlier_import COMMAND agc_driver_recorder_tests --settle-earlier-import-only) add_test(NAME agc_import_settle_retire COMMAND agc_driver_recorder_tests --settle-retire-only) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 5dcf54c727d0..495f80ee0a0c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -73,6 +73,7 @@ class BufferPool { std::size_t RetainedImages(); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; + VkDeviceSize ReclaimIdleDevice(VkDeviceSize bytes, std::chrono::steady_clock::time_point now); static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom); static VkDeviceSize SlabBlockBytes(std::size_t capacity); SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable); @@ -168,6 +169,10 @@ class BufferPool { std::uint64_t imageEvictions = 0; std::list imageOrder; std::uint64_t clock = 0; + std::optional idleRead; + std::uint64_t idleClock = 0; + std::uint64_t reclaimBefore = 0; + std::uint64_t scannedIdleClock = 0; AgcDriver::Mutex slabMutex; std::unordered_map slabs; std::unordered_map> slabBlocks; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 128970a898fa..47d1c1af2379 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -454,6 +454,56 @@ bool BufferPool::evictOldestImage(std::vector& evicted) { return true; } +VkDeviceSize BufferPool::ReclaimIdleDevice(VkDeviceSize bytes, std::chrono::steady_clock::time_point now) { + if (bytes == 0) return 0; + const auto heaps = std::min(memoryProperties.memoryHeapCount, VK_MAX_MEMORY_HEAPS); + if (std::count_if(memoryProperties.memoryHeaps, memoryProperties.memoryHeaps + heaps, [](const auto& heap) { return (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0; }) != 1) return 0; + std::vector evicted; + { + std::lock_guard lock(mutex); + if (!idleRead) { + idleRead = now; + idleClock = clock; + return 0; + } + if (now < *idleRead) return 0; + if (now - *idleRead >= std::chrono::seconds(1)) { + reclaimBefore = idleClock; + idleClock = clock; + idleRead = now; + } + if (reclaimBefore <= scannedIdleClock) return 0; + VkDeviceSize selected = 0; + while (selected < bytes) { + auto oldest = deviceTier.free.end(); + for (auto it = deviceTier.free.begin(); it != deviceTier.free.end(); ++it) { + if (it->second.empty()) continue; + const auto& slot = it->second.front(); + if (slot.allocation.buffer == VK_NULL_HANDLE || slot.allocation.slab || slot.lastUse > reclaimBefore) continue; + if (oldest == deviceTier.free.end() || slot.lastUse < oldest->second.front().lastUse) oldest = it; + } + if (oldest == deviceTier.free.end()) { + scannedIdleClock = reclaimBefore; + break; + } + if (evicted.empty()) evicted.reserve(deviceTier.slots); + const auto allocation = oldest->second.front().allocation; + evicted.push_back(allocation); + selected += allocation.allocationBytes; + deviceTier.retainedBytes -= allocation.allocationBytes; + oldest->second.pop_front(); + if (oldest->second.empty()) deviceTier.free.erase(oldest); + --deviceTier.slots; + ++deviceTier.evictions; + } + } + VkDeviceSize released = 0; + for (const auto& allocation : evicted) released += destroy(allocation); + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (profile && released != 0) AgcDriver::ProfilePrint_nid_no_patch("[bufferpool] idle device reclaim: %llu bytes released for %llu bytes of storage-cache pressure\n", static_cast(released), static_cast(bytes)); + return released; +} + std::size_t BufferPool::MaxSlots() { // APS5_BUFFER_POOL_SLOTS= bounds the retained allocations; 64 is the capacity the pool had // before least-recently-used retention, for A/B runs. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 0c7f40cd5c4b..d2e39b540e65 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -293,7 +293,7 @@ HostImports& Imports() { #ifdef _WIN32 void waitSettleBatch(const ImportSettle& settle, VkDevice device, const SettleBatch& batch) { - if (batch.busy && settle.waitForFences != nullptr) static_cast(settle.waitForFences(device, 1, &batch.fence, VK_TRUE, 10'000'000'000ull)); + if (batch.busy && settle.waitForFences != nullptr) Check(settle.waitForFences(device, 1, &batch.fence, VK_TRUE, 60'000'000'000ull), "vkWaitForFences (settle batch)"); } bool settleBatchRunning(const ImportSettle& settle, VkDevice device, const SettleBatch& batch) { @@ -987,7 +987,7 @@ const char* prepareSettleBatch(const Context& context, ImportSettle& settle, Set } if (batch.commands == VK_NULL_HANDLE) { VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - allocate.commandPool = context.settlePool; + allocate.commandPool = context.pool; allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; allocate.commandBufferCount = 1; if ((result = context.Function("vkAllocateCommandBuffers")(context.device, &allocate, &batch.commands)) != VK_SUCCESS) { @@ -1037,9 +1037,7 @@ void submitSettleBatch(const Context& context, HostImports& state) { begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; if ((result = context.Function("vkBeginCommandBuffer")(batch.commands, &begin)) != VK_SUCCESS) return fail("vkBeginCommandBuffer (settle batch)"); const auto copy = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); - const bool otherFamily = context.settleFamily != context.queueFamily; - const auto createBuffer = otherFamily ? context.Function("vkCreateBuffer") : nullptr; - const auto bindBufferMemory = otherFamily ? context.Function("vkBindBufferMemory") : nullptr; + RecordMemoryBarrier(context, batch.commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); std::size_t consumed = 0; for (; consumed < state.settleWanted.size() && batch.bases.size() < settleBatchCopies; ++consumed) { const auto base = state.settleWanted[consumed]; @@ -1050,38 +1048,19 @@ void submitSettleBatch(const Context& context, HostImports& state) { state.pendingSettles.erase(settle); continue; } - VkBuffer source = import->second.buffer; - if (otherFamily) { - VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; - external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT; - VkBufferCreateInfo aliasInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; - aliasInfo.size = std::min(import->second.bytes, 65536); - aliasInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; - VkBuffer alias = VK_NULL_HANDLE; - if (createBuffer(context.device, &aliasInfo, nullptr, &alias) != VK_SUCCESS) { - state.pendingSettles.erase(settle); - continue; - } - if (bindBufferMemory(context.device, alias, import->second.memory, 0) != VK_SUCCESS) { - destroyBuffer(context.device, alias, nullptr); - state.pendingSettles.erase(settle); - continue; - } - batch.aliases.push_back(alias); - source = alias; - } const VkBufferCopy region{0, (VkDeviceSize{index} * settleBatchCopies + batch.bases.size()) * 4u, 4}; - copy(batch.commands, source, deviceSettle.batchTarget, 1, ®ion); + copy(batch.commands, import->second.buffer, deviceSettle.batchTarget, 1, ®ion); settle->second = PendingSettle{SettlePhase::Submitted, index, context.device}; batch.bases.push_back(base); } state.settleWanted.erase(state.settleWanted.begin(), state.settleWanted.begin() + static_cast(consumed)); + RecordMemoryBarrier(context, batch.commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_WRITE_BIT); if ((result = context.Function("vkEndCommandBuffer")(batch.commands)) != VK_SUCCESS) return fail("vkEndCommandBuffer (settle batch)"); if (batch.bases.empty()) return; VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; submit.commandBufferCount = 1; submit.pCommandBuffers = &batch.commands; - if ((result = context.Function("vkQueueSubmit")(context.settleQueue, 1, &submit, batch.fence)) != VK_SUCCESS) return fail("vkQueueSubmit (settle batch)"); + if ((result = context.Function("vkQueueSubmit")(context.queue, 1, &submit, batch.fence)) != VK_SUCCESS) return fail("vkQueueSubmit (settle batch)"); batch.busy = true; state.settlesSubmitted += batch.bases.size(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 863d6cd9cfb1..a7177e41f78c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -9,6 +9,7 @@ #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" @@ -572,7 +573,7 @@ std::uint64_t forcedCacheBudget() { return forced; } -std::uint64_t storageBudget(const Context& context) { +std::uint64_t storageBudget(const Context& context, std::uint64_t required = 0) { const auto forced = forcedCacheBudget(); if (forced != 0) return forced; if (context.memoryProperties2 == nullptr || context.physical == VK_NULL_HANDLE) return TextureCacheBudget(context.memory); @@ -581,17 +582,26 @@ std::uint64_t storageBudget(const Context& context) { VkPhysicalDevice device{VK_NULL_HANDLE}; std::chrono::steady_clock::time_point read{}; std::uint64_t budget{0}; + std::uint64_t ceiling{0}; } cached; const auto now = std::chrono::steady_clock::now(); std::lock_guard lock(cached.mutex); - if (cached.device == context.physical && now - cached.read < std::chrono::seconds(1)) return cached.budget; - VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; - VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2}; - properties.pNext = &reported; - context.memoryProperties2(context.physical, &properties); - cached.device = context.physical; - cached.read = now; - cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes()); + const auto read = [&] { + VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2}; + properties.pNext = &reported; + context.memoryProperties2(context.physical, &properties); + cached.device = context.physical; + cached.read = now; + cached.budget = StorageTextureBudget(context.memory, &reported, SampledTextureMemory(), storageCacheBytes()); + cached.ceiling = cached.budget; + if (const auto heap = largestDeviceLocalHeap(context.memory); heap && reported.heapBudget[*heap] != 0) { + cached.ceiling = std::max(512ull << 20u, std::min(TextureCacheBudget(context.memory), reported.heapBudget[*heap] / 4u)); + } + }; + if (cached.device != context.physical || now - cached.read >= std::chrono::seconds(1)) read(); + const auto needed = std::min(required, cached.ceiling); + if (needed > cached.budget && context.bufferPool != nullptr && context.bufferPool->ReclaimIdleDevice(needed - cached.budget, now) != 0) read(); return cached.budget; } @@ -1182,7 +1192,7 @@ std::shared_ptr cachedStorageTexture(const Context& context, std static const bool keepNew = std::getenv("APS5_NO_KEEP_NEW_STORAGE") == nullptr; if (auto* recorder = Recorder::Active(); keepNew && recorder != nullptr && GuestMemory::GpuMutex().HeldByThisThread() && recorder->Recording()) recorder->Keep(entry.texture); entry.accounted = heldBytes(*entry.texture); - const auto budget = storageBudget(context); + const auto budget = storageBudget(context, cache.bytes + entry.accounted); while (!cache.entries.empty() && cache.bytes + entry.accounted > budget) evicted.push_back(evictStorage(cache, std::prev(cache.entries.end()))); cache.bytes += entry.accounted; texture = entry.texture; diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index abdf8f0166f4..9547cfb6c2c2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -832,6 +832,103 @@ void AllocationFailureReportsHeapBudgets() { Expect(diagnosticBudgetQueries == 0, "missing-extension fallback attempted a memory budget query"); } +Context reclaimContext() { + auto context = mockContext(); + context.memory.memoryHeapCount = 2; + context.memory.memoryHeaps[0] = {4096 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + context.memory.memoryHeaps[1] = {8192 * MiB, 0}; + context.memory.memoryTypes[1].heapIndex = 1; + return context; +} + +void IdleDeviceReclaim() { + mock = MockDevice{}; + { + auto context = reclaimContext(); + VkBuffer cold; + VkBuffer nextCold; + VkBuffer hot; + VkBuffer host; + VkBuffer small; + { DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); cold = buffer.Handle(); } + { DeviceBuffer buffer(context, 3 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); nextCold = buffer.Handle(); } + { DeviceBuffer buffer(context, 4 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); hot = buffer.Handle(); } + { Buffer buffer(context, 8 * MiB, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); host = buffer.Handle(); } + { Buffer buffer(context, 1024, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); small = buffer.Handle(); } + DeviceBuffer active(context, 16 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto& pool = *context.bufferPool; + const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)}; + Expect(pool.ReclaimIdleDevice(MiB, start) == 0, "the first pressure observation reclaimed a buffer without an idle interval"); + Expect(pool.ReclaimIdleDevice(MiB, start + std::chrono::milliseconds(999)) == 0, "a partial idle interval reclaimed a buffer"); + { + DeviceBuffer reused(context, 4 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + Expect(reused.Handle() == hot, "the warm device buffer was not reused"); + } + const auto next = start + std::chrono::seconds(1); + SetAllocationFailureCountdown(0); + bool threw = false; + try { + pool.ReclaimIdleDevice(MiB, next); + } catch (const std::bad_alloc&) { + threw = true; + } + SetAllocationFailureCountdown(-1); + Expect(threw && mock.liveBuffers.contains(cold) && mock.liveBuffers.contains(nextCold), "a failed reclaim collection lost ownership of an allocation"); + Expect(pool.ReclaimIdleDevice(MiB, next) == 2 * MiB, "reclaim did not release exactly the oldest complete idle allocation"); + Expect(!mock.liveBuffers.contains(cold) && mock.liveBuffers.contains(nextCold), "bounded reclaim released the wrong allocation or exceeded its request"); + Expect(pool.ReclaimIdleDevice(0, next) == 0, "zero pressure released a buffer"); + Expect(pool.ReclaimIdleDevice(32 * MiB, next) == 3 * MiB, "reclaim counted a hot, live, host or slab allocation as idle device memory"); + for (const auto buffer : {hot, host, small, active.Handle()}) Expect(mock.liveBuffers.contains(buffer), "reclaim destroyed a protected buffer"); + SetAllocationFailureCountdown(0); + const auto empty = pool.ReclaimIdleDevice(32 * MiB, next); + SetAllocationFailureCountdown(-1); + Expect(empty == 0, "an unchanged empty eligible set did not stay empty"); + Expect(pool.ReclaimIdleDevice(32 * MiB, start) == 0, "a backwards time observation reclaimed a buffer"); + Expect(pool.ReclaimIdleDevice(32 * MiB, next + std::chrono::seconds(1)) == 4 * MiB, "a device buffer did not become reclaimable after a complete idle interval"); + Expect(mock.liveBuffers.contains(active.Handle()), "reclaim touched an allocation still in use"); + Expect(pool.ReclaimIdleDevice(32 * MiB, next + std::chrono::seconds(2)) == 0, "an empty device tier reported reclaimed memory"); + } + Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "idle device reclaim leaked an allocation"); +} + +void IdleReclaimPreservesWorkingSet() { + mock = MockDevice{}; + { + auto context = reclaimContext(); + const auto useWorkingSet = [&] { + std::vector> buffers; + for (unsigned i = 0; i < 4; ++i) buffers.push_back(std::make_unique(context, 96 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT)); + }; + useWorkingSet(); + const auto allocations = mock.allocations; + const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)}; + auto& pool = *context.bufferPool; + Expect(pool.ReclaimIdleDevice(1024 * MiB, start) == 0, "initial pressure discarded the working set"); + for (unsigned second = 1; second <= 4; ++second) { + useWorkingSet(); + Expect(pool.ReclaimIdleDevice(1024 * MiB, start + std::chrono::seconds(second)) == 0, "pressure discarded buffers reused during the observation interval"); + Expect(mock.allocations == allocations && mock.liveBytes == 384 * MiB, "a reusable working set larger than 128 MiB was recreated or shrunk"); + } + Expect(pool.ReclaimIdleDevice(1024 * MiB, start + std::chrono::seconds(5)) == 384 * MiB, "the former working set remained retained after becoming idle"); + Expect(mock.liveBytes == 0, "reclaim did not return the idle working set to the device"); + } + Expect(mock.liveBuffers.empty(), "working-set reclaim leaked a buffer"); + for (const auto heaps : {0u, 2u}) { + mock = MockDevice{}; + { + auto context = reclaimContext(); + context.memory.memoryHeapCount = heaps; + context.memory.memoryHeaps[1].flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; + { DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); } + auto& pool = *context.bufferPool; + const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)}; + Expect(pool.ReclaimIdleDevice(4 * MiB, start) == 0 && pool.ReclaimIdleDevice(4 * MiB, start + std::chrono::seconds(1)) == 0, "reclaim assumed a device heap without unambiguous ownership"); + Expect(mock.liveBytes == 2 * MiB, "reclaim released memory from an unknown or different local heap"); + } + Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "unsupported-heap reclaim leaked an allocation"); + } +} + void SuccessfulAllocationKeepsPool() { mock = MockDevice{}; { @@ -1001,10 +1098,50 @@ void ImageMemoryReuse() { Expect(mock.frees == 35 && context.bufferPool->RetainedImages() == 1, "the pool retains image memory again after a trim"); } +void IdleReclaimPreservesImages() { + mock = MockDevice{}; + { + auto context = reclaimContext(); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = 4 * MiB; + allocation.memoryTypeIndex = 0; + std::uint32_t type = 99; + const auto image = AllocateImageMemory(context, allocation, 1u, "test idle image", &type); + ReleaseImageMemory(context, image, 4 * MiB, type); + VkBuffer buffer; + { DeviceBuffer idle(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); buffer = idle.Handle(); } + auto& pool = *context.bufferPool; + const auto start = std::chrono::steady_clock::time_point{std::chrono::seconds(1)}; + Expect(pool.RetainedImages() == 1 && mock.liveBytes == 6 * MiB, "the mixed pool fixture was not retained"); + Expect(pool.ReclaimIdleDevice(MiB, start) == 0, "initial pressure reclaimed mixed pool memory"); + const auto next = start + std::chrono::seconds(1); + const auto released = pool.ReclaimIdleDevice(MiB, next); + Expect(released == 2 * MiB, "idle buffer reclaim selected image memory instead of the oldest buffer"); + if (released != 2 * MiB) return; + Expect(!mock.liveBuffers.contains(buffer) && mock.liveBytes == 4 * MiB && pool.RetainedImages() == 1, "buffer reclaim changed image ownership or accounting"); + Expect(pool.ReclaimIdleDevice(32 * MiB, next) == 0, "an image-only pool was reclaimed as idle buffers"); + SetAllocationFailureCountdown(0); + const auto empty = pool.ReclaimIdleDevice(32 * MiB, next); + SetAllocationFailureCountdown(-1); + Expect(empty == 0, "a repeated image-only scan allocated or reclaimed memory"); + const auto allocations = mock.allocations; + const auto reused = AllocateImageMemory(context, allocation, 1u, "test idle image", &type); + Expect(reused == image && mock.allocations == allocations && pool.RetainedImages() == 0, "the retained image was not reusable after buffer reclaim"); + ReleaseImageMemory(context, reused, 4 * MiB, type); + Expect(pool.RetainedImages() == 1 && mock.liveBytes == 4 * MiB, "the reused image was not retained exactly once"); + } + Expect(mock.liveBytes == 0 && mock.liveBuffers.empty() && mock.frees == 2, "mixed pool cleanup leaked or freed an allocation twice"); +} + + int main(int argc, char** argv) { try { if (argc == 2 && std::string_view(argv[1]) == "--small-device-retention") { SmallDeviceRetention(); + if (std::getenv("APS5_STAGING_POOL_MIB") == nullptr) { + IdleDeviceReclaim(); + IdleReclaimPreservesWorkingSet(); + } return failures == 0 ? 0 : 1; } if (argc == 2) { @@ -1040,6 +1177,9 @@ int main(int argc, char** argv) { AllocationFailure(VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_DEVICE_LOST, true); AllocationFailureReportsHeapBudgets(); SuccessfulAllocationKeepsPool(); + IdleDeviceReclaim(); + IdleReclaimPreservesWorkingSet(); + IdleReclaimPreservesImages(); PartialSlabDoesNotRetry(); CacheReclaimRecovery(); ImageCacheReclaim(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 851b36e40481..cbffabd1a4d2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -2027,7 +2027,10 @@ bool settleFirstUseRaced(const Context& context, int copies) { const auto* import = HostImportFor(context, address, unit); Require(import != nullptr && import->unwatched && !Watched(address, unit), "(f) a 64 KiB import was not compared at first"); const auto commands = recorder.Commands(); - for (int copy = 0; copy < copies; ++copy) CopyBuffer(context, commands, import->buffer, 0, scratch.Handle(), 0, unit); + for (int copy = 0; copy < copies; ++copy) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + CopyBuffer(context, commands, import->buffer, 0, scratch.Handle(), 0, unit); + } recorder.Submit(); const auto submittedAt = std::chrono::steady_clock::now(); const auto deadline = submittedAt + std::chrono::seconds(5); @@ -2066,6 +2069,177 @@ bool settleFirstUseRaced(const Context& context, int copies) { return racing; } +bool settleWriteOrderTests(const Device& device) { + using namespace AgcDriver::GuestMemory; + const auto& context = device.GetContext(); + constexpr std::size_t unit = 65536; + if (context.hostImportAlignment == 0 || context.hostImportAlignment > unit || context.settleQueue == VK_NULL_HANDLE || !WriteWatched()) { + std::cout << "host imports within 64 KiB, a settle queue or write watching unavailable: settle write order not tested\n"; + return false; + } + std::unique_lock gpu(GpuMutex()); + PrepareImportWatch(context); + if (!settledImportsWatchPrivate(context)) { + std::cout << "this device compares every Windows import: settle write order not tested\n"; + return false; + } + void* block = AllocateWatched(unit, unit); + struct Release { + void* block; + ~Release() { ReleaseWatched(block, unit); } + } release{block}; + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, unit, true, true, true); + } + Recorder recorder(context); + recorder.Activate(); + const auto* import = HostImportFor(context, address, unit); + Require(import != nullptr && import->unwatched, "the write-order import was not initially compared"); + const auto fill = [&](std::uint32_t value) { + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdFillBuffer")(commands, import->buffer, HostImportOffset(*import, address), unit, value); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT); + recorder.NotePendingWrite(address, unit); + recorder.Submit(); + }; + const auto before = HostImportSettles().submitted; + fill(0x22222222u); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (HostImportSettles().submitted == before && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(HostImportFor(context, address, unit) == import, "the write-order import changed while settling"); + } + Require(HostImportSettles().submitted > before, "the write-order import did not submit a settle"); + fill(0x33333333u); + recorder.Sync(); + while (!Watched(address, unit) && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + HostImportFor(context, address, unit); + } + Require(Watched(address, unit), "the write-order import did not return to the write watch"); + const auto* bytes = static_cast(block); + Require(std::all_of(bytes, bytes + unit, [](std::uint8_t value) { return value == 0x33; }), "the GPU store after the settle did not preserve its bytes"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, unit); + std::cout << "settle read between main-queue GPU writes passed\n"; + return true; +} + +struct SettleWaitFailure { + PFN_vkGetDeviceProcAddr resolve; + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkResult result = VK_SUCCESS; + std::uint64_t timeout = 0; + unsigned waits = 0; + unsigned destroys = 0; + unsigned frees = 0; +}; + +SettleWaitFailure* settleWaitFailure = nullptr; + +VKAPI_ATTR VkResult VKAPI_CALL settleWaitForFences(VkDevice device, std::uint32_t count, const VkFence* fences, VkBool32 all, std::uint64_t timeout) { + auto& failure = *settleWaitFailure; + if (failure.result != VK_SUCCESS) { + failure.timeout = timeout; + ++failure.waits; + return failure.result; + } + return reinterpret_cast(failure.resolve(device, "vkWaitForFences"))(device, count, fences, all, timeout); +} + +VKAPI_ATTR void VKAPI_CALL settleDestroyBuffer(VkDevice device, VkBuffer buffer, const VkAllocationCallbacks* allocator) { + auto& failure = *settleWaitFailure; + if (buffer == failure.buffer) ++failure.destroys; + reinterpret_cast(failure.resolve(device, "vkDestroyBuffer"))(device, buffer, allocator); +} + +VKAPI_ATTR void VKAPI_CALL settleFreeMemory(VkDevice device, VkDeviceMemory memory, const VkAllocationCallbacks* allocator) { + auto& failure = *settleWaitFailure; + if (memory == failure.memory) ++failure.frees; + reinterpret_cast(failure.resolve(device, "vkFreeMemory"))(device, memory, allocator); +} + +PFN_vkVoidFunction VKAPI_CALL settleWaitDeviceProc(VkDevice device, const char* name) { + if (std::strcmp(name, "vkWaitForFences") == 0) return reinterpret_cast(settleWaitForFences); + if (std::strcmp(name, "vkDestroyBuffer") == 0) return reinterpret_cast(settleDestroyBuffer); + if (std::strcmp(name, "vkFreeMemory") == 0) return reinterpret_cast(settleFreeMemory); + return settleWaitFailure->resolve(device, name); +} + +bool settleWaitFailureTests(const Device& device) { + using namespace AgcDriver::GuestMemory; + auto context = device.GetContext(); + constexpr std::size_t unit = 65536; + if (context.hostImportAlignment == 0 || context.hostImportAlignment > unit || context.settleQueue == VK_NULL_HANDLE || !WriteWatched()) return false; + std::lock_guard gpu(GpuMutex()); + PrepareImportWatch(context); + if (!settledImportsWatchPrivate(context)) return false; + ClearHostImports(context.device); + SettleWaitFailure failure{context.deviceProc}; + settleWaitFailure = &failure; + context.deviceProc = settleWaitDeviceProc; + context.functions = nullptr; + void* block = AllocateWatched(unit, unit); + Require(block != nullptr, "no watched block for settle wait failure"); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, unit, true, true, true); + } + struct Release { + Context& context; + SettleWaitFailure& failure; + void* block; + ~Release() { + failure.result = VK_SUCCESS; + ClearHostImports(context.device); + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + ReleaseWatched(block, unit); + settleWaitFailure = nullptr; + } + } release{context, failure, block}; + const auto address = reinterpret_cast(block); + const auto* import = HostImportFor(context, address, unit); + Require(import != nullptr && import->unwatched, "wait failure import was not initially compared"); + failure.buffer = import->buffer; + failure.memory = import->memory; + const auto before = HostImportSettles().submitted; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (HostImportSettles().submitted == before && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + HostImportFor(context, address, unit); + } + Require(HostImportSettles().submitted > before, "wait failure did not submit a settle"); + Check(context.Function("vkDeviceWaitIdle")(context.device), "vkDeviceWaitIdle before settle failure"); + for (const auto result : {VK_TIMEOUT, VK_ERROR_DEVICE_LOST}) { + failure.result = result; + const auto waits = failure.waits; + std::string error; + try { + ClearHostImports(context.device); + } catch (const std::runtime_error& caught) { + error = caught.what(); + } + failure.result = VK_SUCCESS; + Require(failure.waits == waits + 1 && failure.timeout > 0 && failure.timeout <= 60'000'000'000ull, "settle retirement did not use a bounded fence wait"); + Require(error == "AGC graphics: vkWaitForFences (settle batch): Vulkan result " + std::to_string(result), "settle retirement did not propagate its Vulkan failure"); + Require(failure.destroys == 0 && failure.frees == 0 && HostImportCovers(context, address, unit), "failed settle wait released its host import"); + } + ClearHostImports(context.device); + Require(failure.destroys == 1 && failure.frees == 1, "successful settle retirement did not release the retained import exactly once"); + std::cout << "bounded settle wait preserves imports on timeout and device loss; successful retry releases once\n"; + return true; +} + bool settleFirstUseTests(const Device& device) { using namespace AgcDriver::GuestMemory; const auto& context = device.GetContext(); @@ -2080,15 +2254,9 @@ bool settleFirstUseTests(const Device& device) { std::cout << "this device compares every Windows import: the first use of a settled import not tested\n"; return false; } - for (int copies = 4096; copies <= 65536; copies *= 4) { - if (settleFirstUseRaced(context, copies)) { - std::cout << "settled import first use tests passed\n"; - return true; - } - std::cout << "(f) the main-queue copies had finished before the import returned to the write watch, so its read did not race them\n"; - } - std::cout << "(f) the main-queue copies finished first in every round: the first use of a settled import not tested\n"; - return false; + Require(!settleFirstUseRaced(context, 4096), "(f) the import returned to the write watch while its earlier main-queue reads were still in flight"); + std::cout << "settled import first use ordering tests passed\n"; + return true; } bool settleEarlierImportTests(const Device& device) { @@ -2127,7 +2295,10 @@ bool settleEarlierImportTests(const Device& device) { const auto* earlier = HostImportFor(context, address, 16); Require(earlier != nullptr && earlier->base == address && earlier->bytes == unit && earlier->unwatched, "(e) the first window of the range was not imported compared"); const auto commands = recorder.Commands(); - for (int copy = 0; copy < copies; ++copy) CopyBuffer(context, commands, earlier->buffer, 0, scratch.Handle(), 0, unit); + for (int copy = 0; copy < copies; ++copy) { + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + CopyBuffer(context, commands, earlier->buffer, 0, scratch.Handle(), 0, unit); + } const auto* whole = HostImportFor(context, address + unit - 16, 32); Require(whole != nullptr && whole->base == address && whole->bytes == 2 * unit && whole->unwatched, "(e) an access across the windows did not replace the first window's import"); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); @@ -10482,7 +10653,7 @@ int main(int argc, char** argv) { } std::unique_ptr created; try { - created = std::make_unique(argc == 2 && (std::string_view(argv[1]) == "--import-watch-other-family-only" || std::string_view(argv[1]) == "--deferred-settle-other-family-only" || std::string_view(argv[1]) == "--host-import-lifetime-other-family-only")); + created = std::make_unique(argc == 2 && (std::string_view(argv[1]) == "--import-watch-other-family-only" || std::string_view(argv[1]) == "--deferred-settle-other-family-only" || std::string_view(argv[1]) == "--host-import-lifetime-other-family-only" || std::string_view(argv[1]) == "--settle-write-order-other-family-only")); } catch (const std::exception& error) { if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; std::cout << "skipped, no usable Vulkan device: " << error.what() << '\n'; @@ -10521,6 +10692,14 @@ int main(int argc, char** argv) { } return deferredSettleTests(device) ? 0 : 77; } + if (argc == 2 && (std::string_view(argv[1]) == "--settle-write-order-only" || std::string_view(argv[1]) == "--settle-write-order-other-family-only")) { + if (std::string_view(argv[1]) == "--settle-write-order-other-family-only" && device.GetContext().settleFamily == device.GetContext().queueFamily) { + std::cout << "skipped, no other queue family to settle imports on\n"; + return 77; + } + return settleWriteOrderTests(device) ? 0 : 77; + } + if (argc == 2 && std::string_view(argv[1]) == "--settle-wait-failure-only") return settleWaitFailureTests(device) ? 0 : 77; if (argc == 2 && std::string_view(argv[1]) == "--settle-first-use-only") return settleFirstUseTests(device) ? 0 : 77; if (argc == 2 && std::string_view(argv[1]) == "--settle-earlier-import-only") return settleEarlierImportTests(device) ? 0 : 77; if (argc == 2 && std::string_view(argv[1]) == "--settle-retire-only") return settleRetireTests(device) ? 0 : 77; diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 8676f9acffeb..77bf4fd9af24 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -145,7 +145,7 @@ flowchart LR - Guest memory reaches the GPU through host imports (`VK_EXT_external_memory_host`) where the driver accepts them, and through copies otherwise ([`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). A registered range larger than the import window (64 MiB) is imported per window for buffer and fill accesses and whole for an address-based build. Live imports stay under a budget, with the least recently used import retired first (see TechnicalDebt, Host import windows and budget). - A guest protect or unmap that overlaps a plain host-pointer import retires the whole import before changing the guest range (`WaitForLeasesAndImports` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp)). Later accesses import surviving registered ranges again; imports never keep the pages across a cut pinned in a trimmed state. `GuestAllocations::Range::backingSerial` distinguishes the original pages from replacements: untouched pieces of a split keep their serial, newly mapped or protected pieces get a fresh one, and every span passed to `RequireUnpinned` (including the other mappings of remapped direct memory and the resident runs touched by a 2 MiB unmap or remap) gets a fresh serial when the mutation ends. - A draw binds its vertex and index ranges in place from a host import, except with NVIDIA's Windows driver, which reads a host import of guest memory far slower than a buffer it allocated (Astro Bot at Crash Site on an RTX 5070 Ti: 164 against 45 ms of draws per frame), so they are copied there (`DrawInputsInPlaceUsable`, `Context::drawInputsInPlace`). `APS5_DRAW_INPUTS_IN_PLACE=1` binds them in place and `APS5_NO_DRAW_INPUTS_IN_PLACE=1` copies them on any device; the second wins. -- On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the settle queue, and once the batch's fence has signalled a later lookup returns the import's whole 64 KiB blocks to the write watch (`GuestMemory::Rewatch` drops their marks and stamps every block as a CPU store), unless a live storage image overlaps them. Such an import stays compared: an image that took its generation before the stamp would take the stamp for a CPU store over the results it produces after it, and drop them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. `APS5_NO_DEFERRED_SETTLES` (any value) keeps every import under 1 MiB compared. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. On Linux the settle queue is made when the device imports dma-buf memory, and every dma-buf import of 1 MiB or more is settled right after it is made, for another reason: the kernel maps an import's pages for the GPU at the first submission that uses it, which took about 2 s for a 3 GiB import on RADV and held up the title's frame. The import's buffer is shared with the settle queue's family instead of aliased, since a sparse import is bound to many memory objects. +- On Windows, a host import of private `MEM_WRITE_WATCH` memory (singly mapped direct memory) stays on `GetWriteWatch` when `ProbeImportWriteProtection` shows the driver reports nothing written after the second GPU read and a GPU write, and a CPU store is collected (`decideImportWatch`). The driver marks every page of such an import written, asynchronously, until a GPU use of it completes (a read on the settle queue ends it even while main-queue work that used the import first is still running, and the first use of an earlier import of the same pages after that read marks nothing), so `settleImport` in [`GuestBufferMemory.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) copies 4 bytes out of the new import on a second queue of the graphics family, or on a queue of another copy-capable family when the graphics family has only one (`Context::settleQueue`, own pool and mutex) inside `ImportWatched`'s window and waits for it, for imports of 1 MiB or more; the main queue can't be used there because `importAllocation` holds the imports lock and `GpuMutex` comes before it in the lock order. A smaller import is compared when it is made, so a title that maps, imports and unmaps a small buffer around every submission does not block its mapping thread. Its 4-byte read is queued instead: the next import lookup that holds `GpuMutex` while the imports match the registry (`settleUnwatchedImports`) records the queued reads into one batch on the main queue, with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU, and once the batch's fence has signalled a later lookup returns the import's whole 64 KiB blocks to the write watch (`GuestMemory::Rewatch` drops their marks and stamps every block as a CPU store), unless a live storage image overlaps them. Such an import stays compared: an image that took its generation before the stamp would take the stamp for a CPU store over the results it produces after it, and drop them. A settled import whose range had already left the watch returns the same way, and an import retired while its read is in flight is kept with that batch and freed once the batch has completed, without waiting for it; a guest unmap of its memory waits for the batch instead. `APS5_NO_DEFERRED_SETTLES` (any value) keeps every import under 1 MiB compared. Imports through a section alias keep tracking on the guest views, and every other Windows host import is compared. A device without such a queue keeps the comparisons. On Linux the settle queue is made when the device imports dma-buf memory, and every dma-buf import of 1 MiB or more is settled right after it is made, for another reason: the kernel maps an import's pages for the GPU at the first submission that uses it, which took about 2 s for a 3 GiB import on RADV and held up the title's frame. The import's buffer is shared with the settle queue's family instead of aliased, since a sparse import is bound to many memory objects. ### Log output diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 6de88a50abbb..67b3d2b74f04 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -102,6 +102,7 @@ - [Covered-centroid raster state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) accepts PA_SC_AA_CONFIG.COVERED_CENTROID_IS_CENTER only when the pixel program is skipped or SPI_PS_INPUT_ENA and SPI_PS_INPUT_ADDR have no active perspective or linear centroid input. Bit 29 follows [PS5_Mesa GFX10.3 registers](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/registers/gfx103.json); [RADV](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/vulkan/radv_cmd_buffer.c) sets it on GFX10.3 and later. Active centroid interpolation and unspecified input registers still throw. Console covered-pixel centroid results were not measured. - [Compute LDS beyond the device's shared-memory limit](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) lives in a coherent storage buffer with one slice of `LDS bytes / 4 + 1` dwords, rounded up to an even count so that 64-bit atomics stay 8-byte aligned, per workgroup (the dword after the LDS is the atomic lock on a device without `shaderBufferInt64Atomics`), shared by all such dispatches and grown in 1 MiB steps up to `maxStorageBufferRange`; a dispatch whose slices do not fit throws. It is slower than shared memory and its cost was not measured. An indirect dispatch of such a program always reads its group counts on the CPU. Its results are pinned against the in-limit semantics in `agc_driver_lds_beyond_device_limit`, not measured on hardware. - [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request. +- [Storage-cache pressure and idle device buffers](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp): the one-second idle observation interval is a retention choice, not a console rule; buffers reused less often can be recreated under pressure. Before a storage-cache eviction, independently allocated device-tier buffers unused throughout that interval can be released, oldest first, for the cache's shortfall below its existing ceiling. Live buffers, host tiers and slab allocations are not reclaimed this way. Reclamation requires exactly one device-local heap because retained buffers do not record their heap. The heap budget is read again after an actual release; a delayed driver report can still cause an eviction. This is not a shared budget for all retained GPU resources, and its effect on titles with intermittent large buffer use is unmeasured. - [Host placement after a device-memory refusal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) (libSceAgcDriver): when the driver refuses a device-local allocation with `VK_ERROR_OUT_OF_DEVICE_MEMORY` after the pool's trim and the texture-cache release, device buffers and images (textures, storage textures, block, repacked, depth, depth plane and stencil copies, render, depth and multisampled targets and attachment proxies) are allocated in a host memory type their requirements allow. A host-placed device buffer leaves the pool on release. A host-placed image keeps that placement until it is destroyed and is never moved back to device memory, so every access to it crosses PCIe for its lifetime. The placements are counted in a rate-limited `[bufferpool]` line with the cumulative MiB, not the resident bytes. The 1x1 denormal clear probe still allocates device memory directly and throws on a refusal. - [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): the journal is append-only and never compacted, so a module a later run no longer builds pipelines from stays in the file until it is deleted, and the whole file is read into memory while the replay runs (its modules reference that buffer until the last replayed pipeline was built). Only the first `APS5_PIPELINE_JOURNAL_ENTRIES` pipelines recorded are replayed (a record is never rewritten, so a pipeline past the bound stays past it), and the pipeline store evicts the least recently used entry past `APS5_PIPELINE_STORE_ENTRIES` (1024 each): a title with more configurations rebuilds the rest at the draw as before. A compute pipeline is replayed over the set layout `DescribeSetLayouts` derives from the result's bindings, which a dispatch that finds it resident checks against its `ShaderResources` layout only under `APS5_VERIFY_SET_LAYOUTS=1`. Recording and replay are best effort: a record whose write fails is lost, an entry whose build fails is skipped, and both are counted in the `[pipeline-journal]` line rather than stopping the run. A replayed pipeline that the title never asks for again costs its build time and its driver memory (its optimized relink is deferred to its first draw, so a run whose first frames bind many replayed pipelines queues their relinks then); nothing ages recipes out of the file. Two runs sharing the file take turns under its lock file, but each only knows the modules it loaded or wrote itself, so a module the other run appended meanwhile may be appended again (the loader keeps the first copy). Records a run appends after another run's load are replayed by that run only after its next start. The file header's format number is the only guard against a change of the recipe codec, `CanonicalPipelineState` or the store key that keeps the structures' sizes (the recompiler's source version does not cover the driver's sources). - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline @@ -114,7 +115,8 @@ - [`APS5_SKIP_REFUSED_DRAWS`](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp) (libSceAgcDriver) drops a draw whose graphics stage the recompiler refuses instead of failing the queue, so the frame lacks that draw, and the stage is compiled and refused again at each such draw. Compute refusals still fail. - [Pending storage-image results over memory outside the write watch](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) leave to the CPU the 64 KiB blocks whose bytes differ from the ones saved at the upload, the fill, the image copy, the previous store or the moment their range left the watch (`Unwatch`, or a host import that does not stay watched), and the blocks the watch saw the CPU write before that moment. A store made on the GPU saves its bytes through a host-visible copy that becomes the saved bytes when its batch completes; a store or a refresh that needs them earlier waits for that batch. A unit without a write generation saves the guest bytes when results land in it outside the watch. One whose range leaves the watch while its results are pending saves its bytes through the unwatch hook, which also runs when a failed collect takes a range out of the watch. A range that leaves the watch without the hook running over it (an import fallback that unwatched without saving) leaves its pending results without saved bytes; storing such results throws. A unit whose results were stored while its blocks were watched keeps only a generation; when its blocks leave the watch while nothing is pending, the hook saves nothing, and results that land in it afterwards have neither saved bytes nor a watch, so storing them throws too. - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched, except writable imports of private `MEM_WRITE_WATCH` memory when the import probe keeps the write watch and a settle queue exists: those of 1 MiB or more stay watched, and smaller ones return to the watch once their settle has completed, unless a storage image overlaps them (see Pages under an import window): a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. Excluded blocks keep their write-watch marks as a hint only (`UntrackedHint`): a storage image or a sampled texture snapshot over them compares its bytes again after a mark in its range, a driver store, a mapping change or a recorded GPU write instead of at every collect epoch or lookup, and a mark the driver makes without a CPU store costs one comparison. This preserves CPU edits through byte comparisons but limits snapshot reuse on excluded blocks to draws of one collect epoch and disables unit shadows for them; restoring tracking on import retirement without replacing the backing remains unimplemented. -- [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) (libSceAgcDriver) whose write protection was resolved while a host import that leaves its range watched was made are compared with the bytes they held before the import and count as stored over only when those differ. A CPU store of the bytes already there within the window cannot be told apart from an import that resolves the write protection without storing, as the NVIDIA driver (610.57.04) does for every imported page, so such a store does not win over pending results. Where host-pointer and dma-buf imports differ in whether they stay watched, a failed host-pointer import and the dma-buf import after it are two windows, and the pages the failed one resolved count as stored over. On Windows, the NVIDIA driver (617.42) marks every page of a host import of private `MEM_WRITE_WATCH` memory written, asynchronously, until a GPU use of the import completes; a 4-byte read of it on the settle queue ends that even while main-queue work that used the import first is still running, and an earlier import of the same pages whose first use runs after that read marks none of them (`agc_import_settle_first_use`, `agc_import_settle_earlier_import`). Such an import stays watched only after a 4-byte read of it on the settle queue has completed inside the window. Each one waits for that read, which keeps its window open for one round trip on the settle queue, so a same-value CPU store from another thread in that time counts as unchanged too. An import made where no settle queue exists is compared instead. An import of under 1 MiB is not settled inside its window, because the settle blocks the importing thread and a title that imports a 64 KiB buffer around every submission lost about 19 % of its frame rate to that wait (Dripl1x, #6060). It is compared until its read, submitted in a batch by a later import lookup under `GpuMutex` while the imports match the registry, has completed; a later lookup then returns its whole blocks to the write watch, each stamped as a CPU store, so whatever was derived from them while they were compared reads as changed once. Its blocks stay compared while storage results are pending over them, and those of an import retired before that stay compared. +- [Pages under an import window](../../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) whose protection an import resolves count as stored only when their bytes change. A same-value CPU store during that window cannot be distinguished from driver activity and does not win over pending results. On Windows, NVIDIA 617.42 marks every private MEM_WRITE_WATCH page after import until a GPU use completes. The driver behavior was measured on this driver; other drivers are unverified. Imports of at least 1 MiB wait for a four-byte read on the separate settle queue inside the window. Smaller imports stay compared until a deferred read completes; a later lookup returns whole blocks to write watching with a new CPU-store stamp. Blocks overlapping a live storage image, blocks of an import retired before rewatching, and imports without a settle queue stay compared. Shared-memory imports retain their separate-alias tracking. +- [Deferred host-import reads](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) use the main queue with barriers ordering earlier and later writes around the reads. Submission does not wait on the CPU. The latency cost of placing batches behind main-queue work is unmeasured. The first-use test now checks that earlier main-queue reads finish before the import returns to write watching; the original two-queue canary measured driver marking while those reads were still in flight. The earlier-import canary still checks later use of another import of the same pages. - [Settled dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a dma-buf import of 1 MiB or more is read by a 4-byte copy on the settle queue right after it is made (`settleImport`), so the kernel maps its pages for the GPU there and not in the first submission that uses it (about 2 s for a 3 GiB import on RADV). The import's buffer is shared with the settle queue's family. An import made at map time settles on the guest thread that maps the range, inside its map call, so that thread waits while the kernel maps the pages: 0.47 to 2.2 s for the 3 GiB range of Hades (PPSA03355) and 0.49 to 4.3 s for the 4.5 GiB range of ASTRO BOT (PPSA21567) on RADV. Both titles map these ranges at start-up; a title that maps large ranges during gameplay stalls its mapping thread for as long. An import made from a submission settles under the GPU mutex, so its frame still pays for it; a smaller import, and every import on a device with no settle queue (one without a copy-capable queue besides the main one), pays in its first submission. A range imported through a host pointer, as lavapipe imports shared memory, is not settled - [Sparse dma-buf imports](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): on Linux a shared-backed range larger than one 64 MiB udmabuf buffer is imported as one sparse buffer bound to the 64 MiB buffers, on a queue of its own when the device has a second family that binds sparse memory. The path is decided before anything is imported, and a failure after that throws. A range whose offset in the backing file is not a multiple of 64 KiB starts its sparse buffer at the 64 KiB block below it (`HostImport::lead`). A range takes the single udmabuf import instead, which udmabuf's `size_limit_mb` refuses above 64 MiB by default so the range is copied, when the buffer's sparse block size is larger than 64 KiB (it is 64 KiB on RADV), when the sparse buffer does not accept every memory type a chunk import can have, when the backing file ends inside its last chunk, or when it is imported at map time on a device with no second family (the bind then needs the GPU mutex); the NVIDIA proprietary driver imports no dma-buf memory - [Unnormalized S# coordinates](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view.