diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 657c5351c49d..68b48ced0187 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -44,9 +44,6 @@ struct SlabSlot { // builds no longer sweeps the large ones out. Free slots are kept per size, usage and memory // properties, so a lookup does not scan the other slots. Reuse is safe because a Buffer is only released once // the GPU work using it completed (kept until the batch fence, or after a CommandBatch wait). -// APS5_NO_BUFFER_CLASSES=1 matches exact sizes only, as before; APS5_BUFFER_POOL_SLOTS= sets the -// slot count (the bound applies to each tier below, and is held to a sixth of the device's -// maxMemoryAllocationCount so the three tiers keep half of it free). // // The size classes and the exact-size allocations are retained in two tiers with a budget each: // the multi-MiB texture scratch buffers that come back from a batch (GPU-direct uploads and @@ -123,8 +120,6 @@ class BufferPool { std::uint64_t misses = 0; std::uint64_t evictions = 0; }; - // The tier a buffer of `capacity` and `properties` is retained in (the large one for everything - // when shared; the device tier for device-local memory while it has a budget). Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties); static bool DeviceTierEnabled(); static VkDeviceSize deviceBudget(const Context& context); @@ -157,6 +152,7 @@ class BufferPool { Tier smallTier; Tier largeTier; Tier deviceTier; + Tier smallDeviceTier; std::chrono::steady_clock::time_point heapRead{}; bool heapOver = false; VkDeviceSize imageBudget = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 128970a898fa..cc9e4cd7122f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -38,10 +38,11 @@ constexpr VkBufferUsageFlags deviceUsage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | } -BufferPool::BufferPool(const Context& context) : maxSlots(std::max(1, std::min(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))), cacheReclaim(context.cacheReclaim), device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")), allocateMemory(context.Function("vkAllocateMemory")), mapMemory(context.Function("vkMapMemory")), physical(context.physical), memoryProperties(context.memory), memoryProperties2(context.memoryProperties2) { +BufferPool::BufferPool(const Context& context) : maxSlots(std::max(1, std::min(MaxSlots(), context.limits.maxMemoryAllocationCount / 8u))), cacheReclaim(context.cacheReclaim), device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")), allocateMemory(context.Function("vkAllocateMemory")), mapMemory(context.Function("vkMapMemory")), physical(context.physical), memoryProperties(context.memory), memoryProperties2(context.memoryProperties2) { smallTier.budget = smallBudget; largeTier.budget = budget; deviceTier.budget = deviceBudget(context); + smallDeviceTier.budget = std::max(smallBudget, deviceTier.budget / 8u); memoryProperties = context.memory; physical = context.physical; memoryProperties2 = context.memoryProperties2; @@ -49,7 +50,7 @@ BufferPool::BufferPool(const Context& context) : maxSlots(std::max( } BufferPool::~BufferPool() { - for (auto* tier : {&smallTier, &largeTier, &deviceTier}) { + for (auto* tier : {&smallTier, &largeTier, &deviceTier, &smallDeviceTier}) { for (const auto& [key, slots] : tier->free) { for (const auto& slot : slots) destroy(slot.allocation); } @@ -384,7 +385,7 @@ VkBufferUsageFlags BufferPool::Usage(VkBufferUsageFlags usage, VkMemoryPropertyF } BufferPool::Tier& BufferPool::tierFor(std::size_t capacity, VkMemoryPropertyFlags properties) { - if (DeviceTiered(properties) && (sharedTiers() || capacity >= classLimit)) return deviceTier; + if (DeviceTiered(properties)) return sharedTiers() || capacity >= classLimit ? deviceTier : smallDeviceTier; return !sharedTiers() && capacity < classLimit ? smallTier : largeTier; } @@ -398,11 +399,11 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag const auto now = std::chrono::steady_clock::now(); if (now - lastReport > std::chrono::seconds(10)) { lastReport = now; - AgcDriver::ProfilePrint_nid_no_patch("[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s; device: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu); images: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu)\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.slots, smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.slots, largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : "", static_cast(deviceTier.hits), static_cast(deviceTier.misses), static_cast(deviceTier.evictions), deviceTier.slots, deviceTier.retainedBytes / 1048576.0, static_cast(deviceTier.budget >> 20u), static_cast(imageHits), static_cast(imageMisses), static_cast(imageEvictions), imageSlots, imageBytes / 1048576.0, static_cast(imageBudget >> 20u)); + AgcDriver::ProfilePrint_nid_no_patch("[bufferpool] small: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB); large: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB)%s; device: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu); small device: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu); images: %llu hits, %llu misses, %llu evictions, %zu retained (%.0f MiB of %llu)\n", static_cast(smallTier.hits), static_cast(smallTier.misses), static_cast(smallTier.evictions), smallTier.slots, smallTier.retainedBytes / 1048576.0, static_cast(largeTier.hits), static_cast(largeTier.misses), static_cast(largeTier.evictions), largeTier.slots, largeTier.retainedBytes / 1048576.0, sharedTiers() ? " (shared)" : "", static_cast(deviceTier.hits), static_cast(deviceTier.misses), static_cast(deviceTier.evictions), deviceTier.slots, deviceTier.retainedBytes / 1048576.0, static_cast(deviceTier.budget >> 20u), static_cast(smallDeviceTier.hits), static_cast(smallDeviceTier.misses), static_cast(smallDeviceTier.evictions), smallDeviceTier.slots, smallDeviceTier.retainedBytes / 1048576.0, static_cast(smallDeviceTier.budget >> 20u), static_cast(imageHits), static_cast(imageMisses), static_cast(imageEvictions), imageSlots, imageBytes / 1048576.0, static_cast(imageBudget >> 20u)); } } auto& tier = tierFor(capacity, properties); - const auto largest = &tier == &deviceTier && !exactSizes() && bytes <= std::numeric_limits::max() / 2 ? std::max(capacity, bytes * 2) : capacity; + const auto largest = (&tier == &deviceTier || &tier == &smallDeviceTier) && !exactSizes() && bytes <= std::numeric_limits::max() / 2 ? std::max(capacity, bytes * 2) : capacity; auto found = tier.free.end(); for (auto size = capacity; size <= largest && found == tier.free.end(); size = NextCapacity(size)) found = tier.free.find({size, usage, properties}); if (found == tier.free.end() || found->second.empty()) { @@ -469,9 +470,10 @@ VkDeviceSize BufferPool::trim() { Tier retiredSmall; Tier retiredLarge; Tier retiredDevice; + Tier retiredSmallDevice; { std::lock_guard lock(mutex); - for (const auto& [tier, retired] : {std::pair{&smallTier, &retiredSmall}, std::pair{&largeTier, &retiredLarge}, std::pair{&deviceTier, &retiredDevice}}) { + for (const auto& [tier, retired] : {std::pair{&smallTier, &retiredSmall}, std::pair{&largeTier, &retiredLarge}, std::pair{&deviceTier, &retiredDevice}, std::pair{&smallDeviceTier, &retiredSmallDevice}}) { tier->free.swap(retired->free); tier->evictions += tier->slots; tier->retainedBytes = 0; @@ -483,7 +485,7 @@ VkDeviceSize BufferPool::trim() { imageOrder.clear(); } VkDeviceSize bytes = 0; - for (const auto* tier : {&retiredSmall, &retiredLarge, &retiredDevice}) { + for (const auto* tier : {&retiredSmall, &retiredLarge, &retiredDevice, &retiredSmallDevice}) { for (const auto& [key, slots] : tier->free) { for (const auto& slot : slots) bytes += destroy(slot.allocation); } diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index abdf8f0166f4..d9bb3d3f8e65 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -324,6 +324,26 @@ void SmallDeviceRetention() { Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "device pool tier retention leaked Vulkan objects"); } +void SmallDeviceOwnTier() { + mock = MockDevice{}; + { + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkDeviceMemory small = VK_NULL_HANDLE; + { + DeviceBuffer buffer(context, 256 * 1024, usage); + small = MemoryOf(buffer.Handle()); + } + std::vector> host; + for (int i = 0; i < 160; ++i) host.push_back(std::make_unique(context, 512 * 1024, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible)); + host.clear(); + const auto made = mock.allocations; + DeviceBuffer again(context, 256 * 1024, usage); + Expect(MemoryOf(again.Handle()) == small && mock.allocations == made, "small host buffer churn evicted a retained small device buffer"); + } + Expect(mock.liveBytes == 0 && mock.liveBuffers.empty(), "the small device tier leaked Vulkan objects"); +} + void LargerClass() { mock = MockDevice{}; auto context = mockContext(); @@ -1021,6 +1041,7 @@ int main(int argc, char** argv) { LargerClass(); KeptUntilFence(); SmallDeviceRetention(); + SmallDeviceOwnTier(); Budget(); AddressAndHostUnchanged(); SmallDeviceClasses();