perf(agc): keep draw snapshots in resizable BAR memory

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1856,6 +1856,8 @@ if(BUILD_TESTING)
     set_tests_properties(agc_draw_inputs_in_place PROPERTIES SKIP_RETURN_CODE 77)
     add_test(NAME agc_draw_inputs_copied COMMAND agc_driver_recorder_tests --draw-inputs-copied-only)
     set_tests_properties(agc_draw_inputs_copied PROPERTIES SKIP_RETURN_CODE 77)
+    add_test(NAME agc_draw_input_memory COMMAND agc_driver_recorder_tests --draw-input-memory-only)
+    set_tests_properties(agc_draw_input_memory PROPERTIES SKIP_RETURN_CODE 77)
     add_test(NAME agc_resident_present COMMAND agc_driver_recorder_tests --resident-present-only)
     set_tests_properties(agc_resident_present PROPERTIES SKIP_RETURN_CODE 77)
     add_test(NAME agc_queued_write_groups COMMAND agc_driver_recorder_tests --queued-write-groups-only)
@@ -1866,7 +1868,7 @@ if(BUILD_TESTING)
     add_test(NAME agc_metadata_pass COMMAND agc_driver_recorder_tests --metadata-pass-only)
     set_tests_properties(agc_metadata_pass PROPERTIES SKIP_RETURN_CODE 77)
     if(WIN32)
-        set_tests_properties(agc_recorder_read_list agc_host_import_unmap agc_parked_import_unmap agc_host_import_retirement agc_sparse_run_import agc_sparse_run_import_copied agc_unit_shadow agc_driver_recorder agc_idle_publish_wait agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_draw_snapshot_epoch agc_multisample_guest_access agc_draw_inputs_in_place agc_draw_inputs_copied agc_resident_present agc_slice_dirty agc_queued_write_groups agc_metadata_pass agc_tiled_resolve PROPERTIES ENVIRONMENT_MODIFICATION
+        set_tests_properties(agc_recorder_read_list agc_host_import_unmap agc_parked_import_unmap agc_host_import_retirement agc_sparse_run_import agc_sparse_run_import_copied agc_unit_shadow agc_driver_recorder agc_idle_publish_wait agc_descriptor_pool_recycling agc_draw_snapshot_cap agc_draw_snapshot_epoch agc_multisample_guest_access agc_draw_inputs_in_place agc_draw_inputs_copied agc_draw_input_memory agc_resident_present agc_slice_dirty agc_queued_write_groups agc_metadata_pass agc_tiled_resolve PROPERTIES ENVIRONMENT_MODIFICATION
             "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
         add_test(NAME agc_host_import_second_mapping COMMAND agc_driver_recorder_tests --host-import-second-mapping-only)
         set_tests_properties(agc_host_import_second_mapping PROPERTIES SKIP_RETURN_CODE 77 ENVIRONMENT_MODIFICATION

core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp

@@ -34,6 +34,7 @@ struct SlabSlot {
     VkDeviceMemory memory;
     VkDeviceSize offset;
     void* mapping;
+    std::uint32_t memoryType = 0;
 };
 
 // Released buffer allocations kept for reuse, since creating, binding and mapping one costs tens of
@@ -76,7 +77,7 @@ class BufferPool {
     VkDeviceSize ReclaimIdleDevice(VkDeviceSize bytes, std::chrono::steady_clock::time_point now);
     static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom);
     static VkDeviceSize SlabBlockBytes(std::size_t capacity);
-    SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable);
+    SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable, std::optional<std::uint32_t> hostType = std::nullopt);
     VkDeviceSize PutSlot(VkDeviceMemory memory, VkDeviceSize offset) noexcept;
     std::size_t SlabBlocks();
 
@@ -88,6 +89,7 @@ class BufferPool {
         std::vector<std::uint32_t> free;
         std::uint32_t used = 0;
         std::uint64_t slab = 0;
+        std::uint32_t memoryType = 0;
     };
     struct Slab {
         std::vector<SlabBlock*> available;

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -10,6 +10,7 @@
 #include <atomic>
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 #include <functional>
 #include <memory>
 #include <mutex>
@@ -151,6 +152,19 @@ inline std::uint64_t& DeviceProcLookups() {
 
 inline constexpr std::size_t EmptyBufferBytes = 16;
 
+inline constexpr VkMemoryPropertyFlags HostBufferProperties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
+inline constexpr VkMemoryPropertyFlags MappedDeviceProperties = HostBufferProperties | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+
+inline VkMemoryPropertyFlags SnapshotMemoryProperties(const VkPhysicalDeviceMemoryProperties& memory, bool disabled) {
+    constexpr VkDeviceSize ResizableBarBytes = VkDeviceSize{4} << 30u;
+    if (disabled) return HostBufferProperties;
+    for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
+        const auto& type = memory.memoryTypes[i];
+        if ((type.propertyFlags & MappedDeviceProperties) == MappedDeviceProperties && type.heapIndex < memory.memoryHeapCount && memory.memoryHeaps[type.heapIndex].size >= ResizableBarBytes) return MappedDeviceProperties;
+    }
+    return HostBufferProperties;
+}
+
 struct Context {
     VkDevice device;
     VkPhysicalDevice physical;
@@ -299,10 +313,20 @@ struct Context {
         return Function<TFunction>(name);
     }
 
-    std::uint32_t MemoryType(std::uint32_t mask, VkMemoryPropertyFlags flags) const {
+    VkMemoryPropertyFlags SnapshotProperties() const {
+        static const bool disabled = std::getenv("APS5_NO_BAR_SNAPSHOTS") != nullptr;
+        return SnapshotMemoryProperties(memory, disabled);
+    }
+
+    std::optional<std::uint32_t> MatchingMemoryType(std::uint32_t mask, VkMemoryPropertyFlags flags) const {
         for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
             if ((mask & (1u << i)) != 0 && (memory.memoryTypes[i].propertyFlags & flags) == flags) return i;
         }
+        return std::nullopt;
+    }
+
+    std::uint32_t MemoryType(std::uint32_t mask, VkMemoryPropertyFlags flags) const {
+        if (const auto type = MatchingMemoryType(mask, flags)) return *type;
         throw std::runtime_error("AGC graphics: required Vulkan memory type is unavailable");
     }
 
@@ -364,6 +388,43 @@ struct Context {
         if (refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt;
         return HostMemoryType(mask);
     }
+
+    VkMemoryPropertyFlags MappedBufferProperties(std::uint32_t mask, VkMemoryPropertyFlags flags, VkDeviceSize bytes) const {
+        if ((flags & MappedDeviceProperties) != MappedDeviceProperties) return flags;
+        const auto mapped = MatchingMemoryType(mask, flags);
+        if (!mapped) {
+            static std::atomic<std::uint64_t> placements{0};
+            const auto count = placements.fetch_add(1) + 1;
+            if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] mapped buffer: none of the buffer's memory types (mask 0x%x) is mapped device-local, placed in host memory; %llu such allocations\n", mask, static_cast<unsigned long long>(count));
+            return HostBufferProperties;
+        }
+        if (memoryProperties2 == nullptr || physical == VK_NULL_HANDLE) return flags;
+        VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT};
+        VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2};
+        properties.pNext = &reported;
+        memoryProperties2(physical, &properties);
+        const auto heap = memory.memoryTypes[*mapped].heapIndex;
+        const auto budget = reported.heapBudget[heap];
+        const auto usage = reported.heapUsage[heap];
+        if (budget == 0 || (usage < budget && bytes <= budget - usage)) return flags;
+        static std::atomic<std::uint64_t> placements{0}, placedBytes{0};
+        const auto count = placements.fetch_add(1) + 1;
+        const auto total = placedBytes.fetch_add(bytes) + bytes;
+        if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] mapped buffer: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory; %llu such allocations, %llu MiB\n", heap, static_cast<unsigned long long>(usage >> 20u), static_cast<unsigned long long>(budget >> 20u), static_cast<unsigned long long>(bytes), static_cast<unsigned long long>(count), static_cast<unsigned long long>(total >> 20u));
+        return HostBufferProperties;
+    }
+
+    std::optional<std::uint32_t> RefusedMappedMemoryFallback(std::uint32_t mask, VkMemoryPropertyFlags flags, std::uint32_t refused) const {
+        if ((flags & MappedDeviceProperties) != MappedDeviceProperties || refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt;
+        std::optional<std::uint32_t> host;
+        for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
+            const auto& type = memory.memoryTypes[i];
+            if ((mask & (1u << i)) == 0 || (type.propertyFlags & HostBufferProperties) != HostBufferProperties || (memory.memoryHeaps[type.heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) continue;
+            if ((type.propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0) return i;
+            if (!host) host = i;
+        }
+        return host;
+    }
 };
 
 inline VkShaderModule CreateShaderModule(const Context& context, const std::uint32_t* code, std::size_t bytes, const char* operation) {

core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp

@@ -18,6 +18,7 @@ class Buffer {
     // shadows of GuestBufferMemory) has no mapping and its bytes move by GPU copies alone.
     std::span<std::byte> Bytes();
     bool Mapped() const { return mapping != nullptr; }
+    VkMemoryPropertyFlags Properties() const { return properties; }
     void Invalidate();
 
 private:

core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp

@@ -238,20 +238,24 @@ std::uint64_t BufferPool::SlabKey(std::uint32_t memoryType, std::size_t capacity
     return (static_cast<std::uint64_t>(memoryType) << 40u) | (static_cast<std::uint64_t>(addressable ? 1u : 0u) << 39u) | static_cast<std::uint64_t>(capacity);
 }
 
-SlabSlot BufferPool::TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable) {
+SlabSlot BufferPool::TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable, std::optional<std::uint32_t> hostType) {
     const auto key = SlabKey(memoryType, capacity, addressable);
     const auto takeFrom = [&](SlabBlock& block, Slab& slab) {
         const auto index = block.free.back();
         block.free.pop_back();
         if (block.used++ == 0) --slab.emptyBlocks;
         if (block.free.empty()) slab.available.erase(std::find(slab.available.begin(), slab.available.end(), &block));
         const auto offset = static_cast<VkDeviceSize>(index) * block.slotBytes;
-        return SlabSlot{block.memory, offset, block.mapping != nullptr ? block.mapping + offset : nullptr};
+        return SlabSlot{block.memory, offset, block.mapping != nullptr ? block.mapping + offset : nullptr, block.memoryType};
     };
     {
         std::lock_guard lock(slabMutex);
         auto& slab = slabs[key];
         if (!slab.available.empty()) return takeFrom(*slab.available.back(), slab);
+        if (hostType.has_value() && *hostType != memoryType) {
+            auto& host = slabs[SlabKey(*hostType, capacity, addressable)];
+            if (!host.available.empty()) return takeFrom(*host.available.back(), host);
+        }
     }
     const auto blockBytes = SlabBlockBytes(capacity);
     const auto release = [this](SlabBlock* block) noexcept {
@@ -260,14 +264,14 @@ SlabSlot BufferPool::TakeSlot(const Context& context, std::uint32_t memoryType,
     };
     std::unique_ptr<SlabBlock, decltype(release)> block(new SlabBlock, release);
     block->slotBytes = capacity;
-    block->slab = key;
     VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
     const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0};
     if (addressable) allocation.pNext = &flags;
     allocation.allocationSize = blockBytes;
     allocation.memoryTypeIndex = memoryType;
-    block->memory = AllocateMemory(allocation, "vkAllocateMemory buffer slab");
-    if ((context.memory.memoryTypes[memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) {
+    block->memory = AllocateMemory(allocation, "vkAllocateMemory buffer slab", hostType, &block->memoryType);
+    block->slab = block->memoryType == memoryType ? key : SlabKey(block->memoryType, capacity, addressable);
+    if ((context.memory.memoryTypes[block->memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) {
         void* mapping = nullptr;
         const auto mapped = mapMemory(device, block->memory, 0, VK_WHOLE_SIZE, 0, &mapping);
         Check(mapped, "vkMapMemory buffer slab");
@@ -277,7 +281,7 @@ SlabSlot BufferPool::TakeSlot(const Context& context, std::uint32_t memoryType,
     block->free.reserve(slots);
     for (std::uint32_t index = slots; index-- != 0;) block->free.push_back(index);
     std::lock_guard lock(slabMutex);
-    auto& slab = slabs[key];
+    auto& slab = slabs[block->slab];
     if (slab.blocks == slab.available.capacity()) slab.available.reserve(std::max<std::size_t>(1, slab.available.capacity() * 2));
     auto& inserted = *block;
     const auto owner = slabBlocks.try_emplace(inserted.memory).first;

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -1197,7 +1197,7 @@ DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uin
             return copy;
         }
     }
-    copy.buffer = std::make_shared<Buffer>(context, bytes, use == Recorder::SnapshotUse::Vertex ? VK_BUFFER_USAGE_VERTEX_BUFFER_BIT : VK_BUFFER_USAGE_INDEX_BUFFER_BIT);
+    copy.buffer = use == Recorder::SnapshotUse::Vertex ? std::make_shared<Buffer>(context, bytes, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, context.SnapshotProperties()) : std::make_shared<Buffer>(context, bytes, VK_BUFFER_USAGE_INDEX_BUFFER_BIT);
     GuestMemory::Read(address, copy.buffer->Bytes(), alignment);
     return copy;
 }

core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp

@@ -2790,7 +2790,7 @@ Recorder::UploadSlice Recorder::DrawUpload(std::size_t bytes) {
     const VkDeviceSize alignment = std::max<VkDeviceSize>(context.limits.minStorageBufferOffsetAlignment, 16);
     auto offset = (drawUploadUsed + alignment - 1) & ~(alignment - 1);
     if (drawUpload == nullptr || offset + bytes > drawUpload->Bytes().size()) {
-        drawUpload = std::make_shared<Buffer>(context, std::max(DrawUploadChunk, bytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+        drawUpload = std::make_shared<Buffer>(context, std::max(DrawUploadChunk, bytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, context.SnapshotProperties());
         offset = 0;
     }
     drawUploadUsed = offset + bytes;
@@ -2895,7 +2895,7 @@ bool Recorder::refreshDrawSnapshot(DrawSnapshot& entry, std::uint64_t address, s
         spans.push_back({held.data() + offset, reinterpret_cast<const void*>(from), static_cast<std::size_t>(to - from)});
     }
     equal.assign(spans.size(), 0);
-    CompareSpans(spans, equal);
+    if ((entry.buffer->Properties() & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) == 0) CompareSpans(spans, equal);
     for (std::size_t k = 0; k < spans.size(); ++k) {
         if (equal[k] != 0) continue;
         const auto offset = static_cast<std::size_t>(static_cast<const std::byte*>(spans[k].first) - held.data());

core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp

@@ -83,9 +83,13 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag
         // The CPU reads most of these buffers back (write-back, diffs), which is very slow from
         // write-combined memory, so the default host properties prefer cached host memory.
         const bool useSlab = (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 && BufferPool::SlabEligible(capacity, requirements.alignment, requirements.size, context.limits.nonCoherentAtomSize);
-        allocation.memoryTypeIndex = context.BufferMemoryType(requirements.memoryTypeBits, properties, useSlab ? BufferPool::SlabBlockBytes(capacity) : allocation.allocationSize);
+        const auto placedBytes = useSlab ? BufferPool::SlabBlockBytes(capacity) : allocation.allocationSize;
+        this->properties = context.MappedBufferProperties(requirements.memoryTypeBits, properties, placedBytes);
+        allocation.memoryTypeIndex = context.BufferMemoryType(requirements.memoryTypeBits, this->properties, placedBytes);
+        const auto refused = context.RefusedMappedMemoryFallback(requirements.memoryTypeBits, this->properties, allocation.memoryTypeIndex);
         if (useSlab) {
-            const auto slot = cache->TakeSlot(context, allocation.memoryTypeIndex, capacity, addressable);
+            const auto slot = cache->TakeSlot(context, allocation.memoryTypeIndex, capacity, addressable, refused);
+            if (slot.memoryType != allocation.memoryTypeIndex) this->properties = HostBufferProperties;
             memory = slot.memory;
             offset = slot.offset;
             slab = true;
@@ -96,7 +100,9 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag
             ready = true;
             return;
         }
-        memory = cache->AllocateMemory(allocation, "vkAllocateMemory buffer");
+        std::uint32_t allocatedType = allocation.memoryTypeIndex;
+        memory = cache->AllocateMemory(allocation, "vkAllocateMemory buffer", refused, &allocatedType);
+        if (allocatedType != allocation.memoryTypeIndex) this->properties = HostBufferProperties;
         Check(context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory");
         initializeAddress(usage);
         if ((properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) Check(context.Function<PFN_vkMapMemory>("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory");

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -5694,7 +5694,7 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
                 const auto generation = GuestMemory::CollectWrites(begin, bytes);
                 buffer = recorder.ReusableDrawSnapshot(begin, bytes, Recorder::SnapshotUse::Storage, nullptr, generation, &offset);
                 if (buffer == nullptr) {
-                    buffer = std::make_shared<Buffer>(context, bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+                    buffer = std::make_shared<Buffer>(context, bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, context.SnapshotProperties());
                     std::memcpy(buffer->Bytes().data(), reinterpret_cast<const void*>(begin), bytes);
                     recorder.KeepDrawSnapshot(begin, bytes, generation, registryGeneration, buffer);
                 }