diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 21a08c936111..ae801a9a604f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -267,6 +267,10 @@ struct Context { throw std::runtime_error("AGC graphics: required Vulkan memory type is unavailable"); } + static bool HeapFits(const VkPhysicalDeviceMemoryBudgetPropertiesEXT& reported, std::uint32_t heap, VkDeviceSize bytes) { + return reported.heapBudget[heap] != 0 && reported.heapUsage[heap] < reported.heapBudget[heap] && bytes <= reported.heapBudget[heap] - reported.heapUsage[heap]; + } + std::uint32_t BufferMemoryType(std::uint32_t mask, VkMemoryPropertyFlags flags, VkDeviceSize bytes) const { constexpr VkMemoryPropertyFlags coherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; auto type = MemoryType(mask, flags); @@ -283,13 +287,10 @@ struct Context { VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &reported}; memoryProperties2(physical, &properties); const auto heap = memory.memoryTypes[type].heapIndex; - const auto fits = [&](std::uint32_t candidate) { - return reported.heapBudget[candidate] != 0 && reported.heapUsage[candidate] < reported.heapBudget[candidate] && bytes <= reported.heapBudget[candidate] - reported.heapUsage[candidate]; - }; - if (reported.heapBudget[heap] == 0 || fits(heap)) return type; + if (reported.heapBudget[heap] == 0 || HeapFits(reported, heap, bytes)) return type; for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) { const auto& candidate = memory.memoryTypes[i]; - if ((mask & (1u << i)) != 0 && (candidate.propertyFlags & flags) == flags && candidate.heapIndex != heap && fits(candidate.heapIndex)) return i; + if ((mask & (1u << i)) != 0 && (candidate.propertyFlags & flags) == flags && candidate.heapIndex != heap && HeapFits(reported, candidate.heapIndex, bytes)) return i; } return type; } @@ -304,13 +305,21 @@ struct Context { const auto heap = memory.memoryTypes[type].heapIndex; const auto budget = reported.heapBudget[heap]; const auto usage = reported.heapUsage[heap]; - if (budget == 0 || (usage < budget && bytes <= budget - usage)) return type; - const auto host = HostMemoryType(mask); - if (!host) return type; + if (budget == 0 || HeapFits(reported, heap, bytes)) return type; + const auto host = HostMemoryTypeWithRoom(mask, reported, bytes); + if (!host) { + if (!HostMemoryType(mask)) return type; + static std::atomic kept{0}, keptBytes{0}; + const auto count = kept.fetch_add(1) + 1; + const auto total = keptBytes.fetch_add(bytes) + bytes; + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget and no allowed host heap has room, %llu bytes kept in device-local memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), type, static_cast(count), static_cast(total >> 20u)); + return type; + } + const auto hostHeap = memory.memoryTypes[*host].heapIndex; static std::atomic placements{0}, placedBytes{0}; const auto count = placements.fetch_add(1) + 1; const auto total = placedBytes.fetch_add(bytes) + bytes; - if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), *host, static_cast(count), static_cast(total >> 20u)); + if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB; host heap %u uses %llu of its %llu MiB budget\n", what, heap, static_cast(usage >> 20u), static_cast(budget >> 20u), static_cast(bytes), *host, static_cast(count), static_cast(total >> 20u), hostHeap, static_cast(reported.heapUsage[hostHeap] >> 20u), static_cast(reported.heapBudget[hostHeap] >> 20u)); return *host; } @@ -321,6 +330,14 @@ struct Context { return std::nullopt; } + std::optional HostMemoryTypeWithRoom(std::uint32_t mask, const VkPhysicalDeviceMemoryBudgetPropertiesEXT& reported, VkDeviceSize bytes) const { + for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) { + const auto heap = memory.memoryTypes[i].heapIndex; + if ((mask & (1u << i)) != 0 && (memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0 && HeapFits(reported, heap, bytes)) return i; + } + return std::nullopt; + } + std::optional RefusedDeviceMemoryFallback(std::uint32_t mask, std::uint32_t refused) const { if (refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt; return HostMemoryType(mask); diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp index 50356ef05225..c8483f967d38 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -52,6 +52,8 @@ struct MockDevice { VkDeviceSize limit = 0; VkResult pressureResult = VK_ERROR_OUT_OF_DEVICE_MEMORY; std::uint64_t attempts = 0; + bool traceAttempts = false; + std::vector attemptedTypes; VkResult mapResult = VK_SUCCESS; VkResult bindResult = VK_SUCCESS; unsigned binds = 0; @@ -75,6 +77,7 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { ++mock.attempts; + if (mock.traceAttempts) mock.attemptedTypes.push_back(info->memoryTypeIndex); auto result = VK_SUCCESS; if (!mock.results.empty()) { result = mock.results.front(); @@ -170,11 +173,15 @@ Context mockContext() { VkDeviceSize reportedBudget = 0; VkDeviceSize reportedUsage = 0; +VkDeviceSize reportedHostBudget = 8000 * MiB; +VkDeviceSize reportedHostUsage = 2000 * MiB; VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) { auto* reported = static_cast(properties->pNext); reported->heapBudget[0] = reportedBudget; reported->heapUsage[0] = reportedUsage; + reported->heapBudget[1] = reportedHostBudget; + reported->heapUsage[1] = reportedHostUsage; } Context budgetContext() { @@ -240,6 +247,45 @@ void RefusedDevicePlacement() { Expect(mock.liveBuffers.empty(), "a failed device buffer left its VkBuffer alive"); } +void BothHeapsOverBudget() { + mock = MockDevice{}; + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + reportedBudget = 1620 * MiB; + reportedUsage = 1600 * MiB; + reportedHostUsage = reportedHostBudget - 32 * MiB; + { + auto context = budgetContext(); + DeviceBuffer kept(context, 176 * MiB, usage); + Expect(mock.memoryTypes.at(MemoryOf(kept.Handle())) == 0, "with both heaps over their budgets, a device buffer was placed in a host heap without room instead of device-local memory"); + } + { + mock = MockDevice{}; + mock.traceAttempts = true; + auto context = budgetContext(); + mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY}; + DeviceBuffer refused(context, 176 * MiB, usage); + Expect(mock.memoryTypes.at(MemoryOf(refused.Handle())) == 1, "with both heaps over their budgets, a device buffer refused device-local memory was not placed in the host memory type"); + Expect(mock.attemptedTypes == std::vector{0, 1}, "with both heaps over their budgets, a device buffer did not try device-local memory before host memory"); + } + { + mock = MockDevice{}; + mock.traceAttempts = true; + auto context = budgetContext(); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = 64 * MiB; + allocation.memoryTypeIndex = context.DeviceMemoryType(3, allocation.allocationSize, "test image"); + const auto kept = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory test image"); + Expect(mock.memoryTypes.at(kept) == 0 && mock.attemptedTypes == std::vector{0}, "with both heaps over their budgets, an image was placed in host memory before the driver refused device-local memory"); + mockFreeMemory(VK_NULL_HANDLE, kept, nullptr); + mock.attemptedTypes.clear(); + mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY}; + const auto refused = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory test image"); + Expect(mock.memoryTypes.at(refused) == 1 && mock.attemptedTypes == std::vector{0, 1}, "with both heaps over their budgets, a refused image did not try device-local memory before host memory"); + mockFreeMemory(VK_NULL_HANDLE, refused, nullptr); + } + reportedHostUsage = 2000 * MiB; +} + void AttachmentReclaim() { mock = MockDevice{}; auto context = budgetContext(); @@ -874,6 +920,7 @@ int main(int argc, char** argv) { ImageCacheReclaim(); HostPlacement(); RefusedDevicePlacement(); + BothHeapsOverBudget(); AttachmentReclaim(); } catch (const std::exception& error) { std::fprintf(stderr, "FAIL: %s\n", error.what()); diff --git a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp index a5ff02af5d70..a0855ca1d3c4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp @@ -18,12 +18,15 @@ void Expect(bool condition, const std::string& what) { constexpr std::uint32_t HostType = 0; constexpr std::uint32_t DeviceType = 1; +constexpr std::uint32_t SecondHostType = 2; constexpr std::uint64_t MiB = 1ull << 20u; std::uint64_t reportedBudget = 0; std::uint64_t reportedUsage = 0; std::uint64_t reportedHostBudget = 0; std::uint64_t reportedHostUsage = 0; +std::uint64_t reportedSecondHostBudget = 0; +std::uint64_t reportedSecondHostUsage = 0; VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) { auto* reported = static_cast(properties->pNext); @@ -31,6 +34,8 @@ VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDev reported->heapUsage[0] = reportedUsage; reported->heapBudget[1] = reportedHostBudget; reported->heapUsage[1] = reportedHostUsage; + reported->heapBudget[2] = reportedSecondHostBudget; + reported->heapUsage[2] = reportedSecondHostUsage; } Context mockContext(bool budgetExtension) { @@ -53,6 +58,46 @@ void Choose(const char* what, bool budgetExtension, std::uint64_t budget, std::u Expect(type == expected, std::string(what) + ": chose memory type " + std::to_string(type) + ", expected " + std::to_string(expected)); } +void HostHeapTests() { + const std::uint32_t both = (1u << HostType) | (1u << DeviceType); + reportedHostBudget = 8000 * MiB; + reportedHostUsage = 2000 * MiB; + Choose("host heap has room", true, 4096 * MiB, 4096 * MiB, both, HostType); + reportedHostUsage = 8000 * MiB - 64 * MiB; + Choose("allocation fills the host heap exactly", true, 4096 * MiB, 4096 * MiB, both, HostType); + reportedHostUsage = 8000 * MiB - 32 * MiB; + Choose("host heap nearly full keeps the device-local type", true, 4096 * MiB, 4096 * MiB, both, DeviceType); + reportedHostUsage = 9000 * MiB; + Choose("host heap over its budget keeps the device-local type", true, 4096 * MiB, 4096 * MiB, both, DeviceType); + reportedHostBudget = 0; + reportedHostUsage = 0; + Choose("unreported host budget is not assumed free", true, 4096 * MiB, 4096 * MiB, both, DeviceType); +} + +void SecondHostHeapTests() { + auto context = mockContext(true); + context.memory.memoryHeapCount = 3; + context.memory.memoryHeaps[2] = {16384 * MiB, 0}; + context.memory.memoryTypeCount = 3; + context.memory.memoryTypes[SecondHostType] = {0, 2}; + reportedBudget = 4096 * MiB; + reportedUsage = 4096 * MiB; + reportedHostBudget = 8000 * MiB; + reportedHostUsage = 8000 * MiB - 32 * MiB; + reportedSecondHostBudget = 8000 * MiB; + reportedSecondHostUsage = 2000 * MiB; + const auto all = (1u << HostType) | (1u << DeviceType) | (1u << SecondHostType); + const auto type = context.DeviceMemoryType(all, 64 * MiB, "test"); + Expect(type == SecondHostType, "a full first host heap stopped the search for a host heap with room: chose memory type " + std::to_string(type)); + const auto masked = context.DeviceMemoryType((1u << HostType) | (1u << DeviceType), 64 * MiB, "test"); + Expect(masked == DeviceType, "with the only allowed host heap full, memory type " + std::to_string(masked) + " was chosen instead of the device-local type"); + reportedSecondHostUsage = 9000 * MiB; + const auto bothFull = context.DeviceMemoryType(all, 64 * MiB, "test"); + Expect(bothFull == DeviceType, "with every host heap full, a host type was chosen: memory type " + std::to_string(bothFull)); + reportedSecondHostBudget = 0; + reportedSecondHostUsage = 0; +} + void RefusedFallbackTests() { const auto context = mockContext(true); const std::uint32_t both = (1u << HostType) | (1u << DeviceType); @@ -108,6 +153,8 @@ void BufferPlacementTests() { int main() { try { const std::uint32_t both = (1u << HostType) | (1u << DeviceType); + reportedHostBudget = 8000 * MiB; + reportedHostUsage = 2000 * MiB; Choose("no VK_EXT_memory_budget", false, 0, 0, both, DeviceType); Choose("budget not reported", true, 0, 4000 * MiB, both, DeviceType); Choose("room in the device-local budget", true, 3000 * MiB, 2000 * MiB, both, DeviceType); @@ -115,6 +162,8 @@ int main() { Choose("allocation exceeds the budget", true, 3000 * MiB, 2937 * MiB, both, HostType); Choose("usage already over the budget", true, 2952 * MiB, 3633 * MiB, both, HostType); Choose("no host type allowed", true, 3000 * MiB, 3000 * MiB, 1u << DeviceType, DeviceType); + HostHeapTests(); + SecondHostHeapTests(); BufferPlacementTests(); RefusedFallbackTests(); } catch (const std::exception& error) {