diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 476d738b21ed..487f38fcb808 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -345,6 +345,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/GuestBufferMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ResidentMirror.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ReferenceStorage.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BdaResources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BufferAddress.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderValidation.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 2531eccf35cf..859a0922e4c6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -520,6 +520,7 @@ class GuestBufferMemory { // the barriers that order them after earlier recorded writes and before the shaders reading them. void recordGpuCopies(std::span copies, bool addressable); void takeHeapReferences(); + void releaseHeapReferences() noexcept; Context context; bool stagingAllowed = false; bool addressStaging = true; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp new file mode 100644 index 000000000000..5acc836b1dea --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp @@ -0,0 +1,46 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_REFERENCESTORAGE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_REFERENCESTORAGE_HPP + +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +class ReferenceStorage { +public: + static constexpr std::size_t Slots = 8; + + explicit ReferenceStorage(std::size_t capacityBytes) noexcept; + ReferenceStorage(const ReferenceStorage&) = delete; + ReferenceStorage& operator=(const ReferenceStorage&) = delete; + + using Clock = std::chrono::steady_clock; + static constexpr std::chrono::milliseconds IdleLimit{2000}; + + std::vector Take(std::size_t bytes, Clock::time_point now = Clock::now()) noexcept; + void Return(std::vector&& storage, Clock::time_point now = Clock::now()) noexcept; + std::size_t ReleaseIdle(Clock::time_point now = Clock::now()) noexcept; + std::size_t IdleBytes() const noexcept; + std::size_t IdleCount() const noexcept; + + static ReferenceStorage& Shared() noexcept; + +private: + std::size_t releaseIdle(Clock::time_point now, std::array, Slots>& released) noexcept; + + mutable std::mutex mutex; + std::array, Slots> idle; + std::array returned{}; + std::array returnedAt{}; + std::uint64_t returns = 0; + std::size_t bytes = 0; + std::size_t capacityBytes; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 2e891ddddcfe..a548ae3474c4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -10,6 +10,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/ResidentMirror.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -3328,6 +3329,7 @@ std::shared_ptr findMirror(const Context& context, std::uint64_t be } void sweepMirrors(std::uint64_t neededBytes, std::uint64_t protectAddress) { + ReferenceStorage::Shared().ReleaseIdle(); const bool released = releaseExpiredJoinedHeaps(); auto space = Spaces().current.load(); auto& state = Mirrors(); @@ -5351,6 +5353,7 @@ void ShutdownGuestBufferWorkers() { } GuestBufferMemory::~GuestBufferMemory() { + releaseHeapReferences(); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; if (!profile) return; // A staged region whose last use recorded its copy-in but no copy-back (the use threw before @@ -6465,9 +6468,14 @@ std::vector> mergeWrites(std::vectormirror == nullptr || !found->mirror->heap || !found->mirror->writable) continue; @@ -6476,10 +6484,16 @@ void GuestBufferMemory::takeHeapReferences() { const auto served = std::min(end, servedEnd(*found)); if (begin >= served) continue; growResident(mirror, {{begin, served}}); - std::vector bytes(static_cast(served - begin)); - if (mirror.resident != nullptr) mirror.resident->Read(begin, bytes); - else std::memcpy(bytes.data(), mirror.buffer->Bytes().data() + (begin - mirror.base), bytes.size()); - heapReferences.emplace_back(begin, std::move(bytes)); + const auto size = static_cast(served - begin); + auto storage = ReferenceStorage::Shared().Take(size); + if (mirror.resident != nullptr) { + storage.resize(size); + mirror.resident->Read(begin, storage); + } else { + const auto bytes = mirror.buffer->Bytes().subspan(static_cast(begin - mirror.base), size); + storage.assign(bytes.begin(), bytes.end()); + } + heapReferences.emplace_back(begin, std::move(storage)); } } @@ -6580,8 +6594,7 @@ void GuestBufferMemory::WriteBack() { committed = true; lease.clear(); space.reset(); - heapReferences.clear(); - heapReadbacks.clear(); + releaseHeapReferences(); } std::optional>> GuestBufferMemory::DirectRegions() const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 0d0f4742450d..2eecabd1b24c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Execution/include/Mutex.hpp" #include "prx/libc/include/HostThreadLocal.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libkernel/Time/include/SleepWake.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -2024,6 +2025,7 @@ void Recorder::NoteAccess(CommandClass which, const Access& access) { void Recorder::CountPresent() { timingPresents.fetch_add(1, std::memory_order_relaxed); presentSerial.fetch_add(1, std::memory_order_relaxed); + ReferenceStorage::Shared().ReleaseIdle(); } std::uint64_t Recorder::Presents() { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ReferenceStorage.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ReferenceStorage.cpp new file mode 100644 index 000000000000..6ef11ade919d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ReferenceStorage.cpp @@ -0,0 +1,89 @@ +#include "prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp" + +#include + +namespace AgcDriver::Graphics { + +ReferenceStorage::ReferenceStorage(std::size_t capacityBytes) noexcept : capacityBytes(capacityBytes) {} + +std::size_t ReferenceStorage::releaseIdle(Clock::time_point now, std::array, Slots>& released) noexcept { + std::size_t freed = 0; + for (std::size_t slot = 0; slot < Slots; ++slot) { + if (idle[slot].capacity() == 0 || now - returnedAt[slot] < IdleLimit) continue; + freed += idle[slot].capacity(); + bytes -= idle[slot].capacity(); + released[slot] = std::move(idle[slot]); + idle[slot] = std::vector(); + } + return freed; +} + +std::size_t ReferenceStorage::ReleaseIdle(Clock::time_point now) noexcept { + std::array, Slots> released; + std::lock_guard lock(mutex); + return releaseIdle(now, released); +} + +std::vector ReferenceStorage::Take(std::size_t requested, Clock::time_point now) noexcept { + std::array, Slots> released; + std::lock_guard lock(mutex); + releaseIdle(now, released); + std::size_t best = Slots; + for (std::size_t slot = 0; slot < Slots; ++slot) { + const auto capacity = idle[slot].capacity(); + if (capacity == 0 || capacity < requested) continue; + if (best == Slots || capacity < idle[best].capacity()) best = slot; + } + if (best == Slots) return {}; + auto storage = std::move(idle[best]); + idle[best] = std::vector(); + bytes -= storage.capacity(); + return storage; +} + +void ReferenceStorage::Return(std::vector&& storage, Clock::time_point now) noexcept { + auto kept = std::move(storage); + const auto capacity = kept.capacity(); + if (capacity == 0 || capacity > capacityBytes) return; + std::array, Slots> evicted; + std::array, Slots> released; + std::lock_guard lock(mutex); + releaseIdle(now, released); + std::size_t free = Slots; + for (std::size_t slot = 0; slot < Slots; ++slot) { + if (idle[slot].capacity() == 0) free = slot; + } + while (free == Slots || bytes + capacity > capacityBytes) { + std::size_t oldest = Slots; + for (std::size_t slot = 0; slot < Slots; ++slot) { + if (idle[slot].capacity() != 0 && (oldest == Slots || returned[slot] < returned[oldest])) oldest = slot; + } + bytes -= idle[oldest].capacity(); + evicted[oldest] = std::move(idle[oldest]); + idle[oldest] = std::vector(); + free = oldest; + } + bytes += capacity; + returned[free] = ++returns; + returnedAt[free] = now; + idle[free] = std::move(kept); +} + +std::size_t ReferenceStorage::IdleBytes() const noexcept { + std::lock_guard lock(mutex); + return bytes; +} + +std::size_t ReferenceStorage::IdleCount() const noexcept { + std::lock_guard lock(mutex); + std::size_t count = 0; + for (const auto& storage : idle) count += storage.capacity() != 0 ? 1 : 0; + return count; +} + +ReferenceStorage& ReferenceStorage::Shared() noexcept { + static ReferenceStorage storage(std::size_t{2} << 30u); + return storage; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index a00cc6e740a9..24ca6b769b5c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -3585,6 +3585,67 @@ void RunSparseHeapMirrorBudgetTests(const Context& context, const BdaTestAccess& std::cout << "a sparse heap whose mirror would pass the heap mirror budget is copied\n"; } +void RunHeapMirrorReferenceSizeTests(const Context& context, const BdaTestAccess& access) { + constexpr std::size_t block = 65536; + constexpr std::size_t heapBytes = 4 * block; +#ifdef _WIN32 + if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { + std::cout << "guest arena unavailable or not write-watched: heap mirror reference sizes not tested\n"; + return; + } + auto* guest = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(heapBytes, block)); + Require(guest != nullptr, "cannot allocate the heap in the guest arena"); + GuestArena::GuestArenaCommit_nid_postfix(guest, heapBytes, PAGE_READWRITE, heapBytes); +#else + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::cout << "write watch unavailable: heap mirror reference sizes not tested\n"; + return; + } + void* mapping = mmap(nullptr, heapBytes + block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(mapping != MAP_FAILED, "cannot map the heap"); + auto* guest = reinterpret_cast((reinterpret_cast(mapping) + block - 1) & ~std::uintptr_t{block - 1}); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(guest, heapBytes); +#endif + std::memset(guest, 0x11, heapBytes); + const auto address = reinterpret_cast(guest); + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, guest, heapBytes, true, true, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, guest); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + registry(true); + const auto before = MirrorCounters(); + const auto store = [&](std::uint64_t written, std::size_t bytes, std::size_t at, std::byte value) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.AddWritable(written, bytes); + leased.Upload(true); + std::uint32_t adjustment = 0; + const auto view = leased.Descriptor(written, static_cast(bytes), adjustment); + access.bytes(view.buffer)[view.offset + adjustment + at] = value; + leased.WriteBack(); + }; + store(address + 4096, 8192, 8000, std::byte{0x7a}); + Require(MirrorCounters().heapMirrors == before.heapMirrors + 1, "the writable heap was not mirrored"); + Require(guest[4096 + 8000] == 0x7a, "the larger write-back lost the GPU's store"); + store(address + 16384, 2048, 2000, std::byte{0x7b}); + Require(guest[16384 + 2000] == 0x7b && guest[4096 + 8000] == 0x7a, "a smaller write-back after a larger one lost the GPU's store"); + store(address + 32768, 8192, 100, std::byte{0x7c}); + Require(guest[32768 + 100] == 0x7c && guest[16384 + 2000] == 0x7b, "a write-back after a smaller one lost the GPU's store"); + registry(false); + ClearImageMirrors(context.device); + ClearSnapshotCopies(context.device); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(guest, heapBytes); + GuestArena::GuestArenaRelease_nid_postfix(guest, heapBytes); +#else + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(guest, heapBytes); + munmap(mapping, heapBytes + block); +#endif + std::cout << "a writable heap mirror writes back a smaller range after a larger one\n"; +} + void RunHeapMirrorRunUnwatchTests(const Context& context, const BdaTestAccess& access) { constexpr std::size_t block = 65536; constexpr std::size_t heapBytes = 32 * block; diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp index 08ac6006e927..8b6a0c6ce716 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp @@ -22,6 +22,7 @@ void RunSparseHeapMirrorUnwatchTests(const AgcDriver::Graphics::Context& context void RunSparseHeapMirrorBudgetTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); void RunHeapMirrorRunUnwatchTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); void RunAddressCopyCapTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access, bool remembered, int frames); +void RunHeapMirrorReferenceSizeTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); void RunAddressCopyCapMirrorTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access, int frames); void RunUnwatchedMirrorImportTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access, int frames); void RunGuestAllocationTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index dd70c1c0bd86..cf2f5a420bfc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -12934,6 +12934,27 @@ int main(int argc, char** argv) { Require(mock.live == 0, "the sparse heap mirror budget test leaked Vulkan objects"); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--heap-mirror-reference-size") { + { + mock = MockVulkan{}; + auto context = mockContext(); + context.bufferDeviceAddress = true; + context.limits.maxStorageBufferRange = 1u << 27; + RunHeapMirrorReferenceSizeTests(context, { + [](VkBuffer buffer) -> std::span { return mockBufferMemory(buffer); }, + {}, + [](VkDeviceAddress address) { + const auto offset = address - 0x100000000000ULL; + const auto buffer = reinterpret_cast(offset / 0x10000); + return mockBufferMemory(buffer).subspan(offset % 0x10000); + }, + {}, + {} + }); + } + Require(mock.live == 0, "the heap mirror reference size test leaked Vulkan objects"); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--heap-mirror-run-unwatch") { { mock = MockVulkan{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/ReferenceStorage.cpp b/core/libs/prx/libSceAgcDriver/tests/ReferenceStorage.cpp new file mode 100644 index 000000000000..d17393cc608a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ReferenceStorage.cpp @@ -0,0 +1,190 @@ +#include "prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp" +#include "AllocationFailure.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using AgcDriver::Graphics::ReferenceStorage; +using AgcDriver::Tests::SetAllocationFailureCountdown; + +namespace { + +int failures = 0; + +void Expect(bool condition, const std::string& what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what.c_str()); + ++failures; +} + +constexpr std::size_t KiB = 1024; + +std::vector Storage(std::size_t bytes) { + std::vector storage; + storage.reserve(bytes); + return storage; +} + +bool NextAllocationFails() { + try { + void* probe = ::operator new(16); + ::operator delete(probe); + return false; + } catch (const std::bad_alloc&) { + return true; + } +} + +void TakeFromEmpty() { + ReferenceStorage storage(64 * KiB); + Expect(storage.Take(1).capacity() == 0, "an empty store hands out no storage"); + Expect(storage.IdleBytes() == 0 && storage.IdleCount() == 0, "an empty store holds nothing"); +} + +void ReuseSameStorage() { + ReferenceStorage storage(64 * KiB); + auto first = Storage(16 * KiB); + const auto* data = first.data(); + storage.Return(std::move(first)); + Expect(storage.IdleBytes() == 16 * KiB && storage.IdleCount() == 1, "a returned buffer is held"); + Expect(storage.Take(32 * KiB).capacity() == 0, "a buffer smaller than the request is not handed out"); + Expect(storage.IdleCount() == 1, "a refused take keeps the held buffer"); + auto again = storage.Take(8 * KiB); + Expect(again.data() == data && again.capacity() >= 16 * KiB, "a take reuses the held allocation"); + Expect(storage.IdleBytes() == 0 && storage.IdleCount() == 0, "a taken buffer leaves the store"); +} + +void LargerStorageTakesTheReferenceSize() { + ReferenceStorage storage(256 * KiB); + std::vector larger(64 * KiB, std::byte{0x5a}); + const auto* data = larger.data(); + storage.Return(std::move(larger)); + std::vector reference(16 * KiB); + for (std::size_t index = 0; index < reference.size(); ++index) reference[index] = static_cast(index); + const std::span bytes(reference); + auto taken = storage.Take(bytes.size()); + taken.assign(bytes.begin(), bytes.end()); + Expect(taken.data() == data, "a smaller reference reuses the larger held allocation"); + Expect(taken.size() == bytes.size(), "a reference copied into larger storage keeps the reference's size"); + Expect(std::equal(taken.begin(), taken.end(), bytes.begin()), "a reference copied into larger storage holds the reference's bytes"); +} + +void SmallestFit() { + ReferenceStorage storage(256 * KiB); + auto large = Storage(64 * KiB); + auto small = Storage(16 * KiB); + const auto* smallData = small.data(); + storage.Return(std::move(large)); + storage.Return(std::move(small)); + auto taken = storage.Take(8 * KiB); + Expect(taken.data() == smallData, "the smallest buffer that fits is handed out"); + Expect(storage.IdleBytes() == 64 * KiB, "the larger buffer stays held"); +} + +void CapacityEvictsOldest() { + ReferenceStorage storage(48 * KiB); + auto older = Storage(32 * KiB); + auto newer = Storage(32 * KiB); + const auto* newerData = newer.data(); + storage.Return(std::move(older)); + storage.Return(std::move(newer)); + Expect(storage.IdleBytes() == 32 * KiB && storage.IdleCount() == 1, "the byte limit evicts the oldest buffer"); + Expect(storage.Take(1).data() == newerData, "the newest buffer survives eviction"); +} + +void OversizeDropped() { + ReferenceStorage storage(16 * KiB); + storage.Return(Storage(32 * KiB)); + Expect(storage.IdleBytes() == 0 && storage.IdleCount() == 0, "a buffer above the limit is not held"); + storage.Return(std::vector()); + Expect(storage.IdleCount() == 0, "a buffer without capacity is not held"); +} + +void SlotsEvictOldest() { + ReferenceStorage storage(1024 * KiB); + std::vector> buffers; + for (std::size_t index = 0; index < ReferenceStorage::Slots + 2; ++index) buffers.push_back(Storage(KiB + index)); + std::vector> retained; + for (std::size_t index = 2; index < buffers.size(); ++index) retained.emplace_back(buffers[index].data(), buffers[index].capacity()); + std::vector> taken; + taken.reserve(ReferenceStorage::Slots + 1); + for (auto& buffer : buffers) storage.Return(std::move(buffer)); + Expect(storage.IdleCount() == ReferenceStorage::Slots, "the store holds at most its slot count"); + std::size_t expected = 0; + for (const auto& [data, capacity] : retained) expected += capacity; + Expect(storage.IdleBytes() == expected, "evicted buffers leave the byte count"); + for (std::size_t index = 0; index <= ReferenceStorage::Slots; ++index) taken.push_back(storage.Take(1)); + Expect(taken.back().capacity() == 0, "nothing beyond the slot count was held"); + taken.pop_back(); + std::size_t matched = 0; + for (const auto& buffer : taken) { + for (auto& identity : retained) { + if (identity.first == buffer.data() && identity.second == buffer.capacity()) { + identity.first = nullptr; + ++matched; + break; + } + } + } + Expect(matched == ReferenceStorage::Slots, "the held buffers are exactly the newest ones returned"); + Expect(storage.IdleBytes() == 0 && storage.IdleCount() == 0, "every held buffer was handed out"); +} + +void NoAllocationOnReturnOrTake() { + static_assert(noexcept(std::declval().Return(std::vector()))); + static_assert(noexcept(std::declval().Take(0))); + ReferenceStorage storage(48 * KiB); + auto first = Storage(32 * KiB); + auto second = Storage(32 * KiB); + auto third = Storage(8 * KiB); + SetAllocationFailureCountdown(0); + storage.Return(std::move(first)); + storage.Return(std::move(second)); + auto taken = storage.Take(16 * KiB); + storage.Return(std::move(third)); + const bool armed = NextAllocationFails(); + SetAllocationFailureCountdown(-1); + Expect(armed, "return, eviction and take allocate nothing"); + Expect(taken.capacity() >= 32 * KiB, "a take under allocation failure still hands out held storage"); + Expect(storage.IdleBytes() == 8 * KiB && storage.IdleCount() == 1, "the byte count stays consistent under allocation failure"); +} + +} + +void IdleBuffersReleased() { + using namespace std::chrono_literals; + const auto start = ReferenceStorage::Clock::time_point{} + 1h; + ReferenceStorage storage(64 * KiB); + storage.Return(Storage(16 * KiB), start); + storage.Return(Storage(8 * KiB), start + 1s); + Expect(storage.ReleaseIdle(start + 1999ms) == 0 && storage.IdleCount() == 2, "a buffer idle for less than the limit was released"); + Expect(storage.ReleaseIdle(start + 2s) == 16 * KiB, "a buffer idle for the limit was not released"); + Expect(storage.IdleCount() == 1 && storage.IdleBytes() == 8 * KiB, "releasing one idle buffer did not keep the newer one"); + Expect(storage.Take(4 * KiB, start + 3s).capacity() == 0, "a take handed out a buffer idle for the limit"); + Expect(storage.IdleBytes() == 0 && storage.IdleCount() == 0, "a take did not release buffers idle for the limit"); + storage.Return(Storage(16 * KiB), start + 4s); + storage.Return(Storage(8 * KiB), start + 6s); + Expect(storage.IdleCount() == 1 && storage.IdleBytes() == 8 * KiB, "a return did not release buffers idle for the limit"); + Expect(storage.Take(4 * KiB, start + 7s).capacity() >= 8 * KiB, "a buffer idle for less than the limit was not reused"); +} + +int main() { + TakeFromEmpty(); + ReuseSameStorage(); + LargerStorageTakesTheReferenceSize(); + SmallestFit(); + CapacityEvictsOldest(); + OversizeDropped(); + SlotsEvictOldest(); + NoAllocationOnReturnOrTake(); + IdleBuffersReleased(); + if (failures != 0) return 1; + std::printf("reference storage tests passed\n"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake index 57daa509e3d5..b45b89f15914 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake @@ -128,6 +128,12 @@ if(BUILD_TESTING) add_test(NAME agc_sparse_heap_mirror_unwatch COMMAND agc_driver_graphics_tests --sparse-heap-mirror-unwatch) add_test(NAME agc_sparse_heap_mirror_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --sparse-heap-mirror-unwatch) add_test(NAME agc_sparse_heap_mirror_budget COMMAND agc_driver_graphics_tests --sparse-heap-mirror-budget) + add_test(NAME agc_heap_mirror_reference_size COMMAND agc_driver_graphics_tests --heap-mirror-reference-size) + set_tests_properties(agc_heap_mirror_reference_size PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_heap_mirror_reference_size PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_heap_mirror_run_unwatch COMMAND agc_driver_graphics_tests --heap-mirror-run-unwatch) add_test(NAME agc_heap_mirror_run_unwatch_no_space_cache COMMAND agc_driver_graphics_tests --heap-mirror-run-unwatch) set_tests_properties(agc_heap_mirror_unwatch_no_space_cache agc_sparse_heap_mirror_unwatch_no_space_cache agc_heap_mirror_run_unwatch_no_space_cache PROPERTIES ENVIRONMENT "APS5_NO_ADDRESS_SPACE_CACHE=1") diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_reference_storage_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_reference_storage_tests.cmake new file mode 100644 index 000000000000..3cf5dd5bd320 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_reference_storage_tests.cmake @@ -0,0 +1,14 @@ +add_test_executable(agc_reference_storage_tests EXCLUDE_FROM_ALL tests/ReferenceStorage.cpp) +target_sources(agc_reference_storage_tests PRIVATE Graphics/src/ReferenceStorage.cpp tests/AllocationFailure.cpp) +target_include_directories(agc_reference_storage_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_reference_storage_tests PRIVATE libc) +set_target_properties(agc_reference_storage_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_reference_storage_tests) +if(BUILD_TESTING) + set_target_properties(agc_reference_storage_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_reference_storage COMMAND agc_reference_storage_tests) + if(WIN32) + set_tests_properties(agc_reference_storage PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 5f18abdc08cb..e73c9585b6cb 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -304,6 +304,7 @@ - [16-bit depth with stencil](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) is stored as `D24_UNORM_S8_UINT`, or `D32_SFLOAT_S8_UINT`, on a device without `D16_UNORM_S8_UINT` as an attachment. Depth comparisons, clears and the depth image see the wider format's precision, so results at the 16-bit quantization boundary can differ from the console; the constant depth bias is scaled by 256 for D24, and a non-zero constant on the D32 float fallback throws. Not measured on console hardware. - [Depth surfaces created over storage writes](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver): a depth surface created without HTILE copies its depth plane from the storage image at the plane's address, a pending one first, when that image has the plane's extent, one level, a 2D view or a 2D array view (its layer 0 is copied) and a 16-bit format for a 16-bit plane or a 32-bit format for a 32-bit plane. A multisampled surface, a surface the device stores in another format, a surface with HTILE and a surface without such an image start from `DB_DEPTH_CLEAR`, and the stencil plane from `DB_STENCIL_CLEAR`; CPU writes to the plane's memory are not read. Not measured on the console, where the plane is that memory. - [Flip packets raise the graphics end-of-pipe event](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) (libSceAgcDriver) - a flip delivers the queue's end-of-pipe event (`sceAgcDriverAddEqEvent`) once the work recorded before it completes, as a `RELEASE_MEM` with `INT_SEL` does, but only on a queue that has never delivered an interrupt from such a release: the queue is latched the first time a release interrupt reaches the event queue, and flips on that queue deliver nothing while its own release interrupts keep arriving. The latch is cleared when the queue has flipped five times without another interrupt from its own releases: the fifth flip delivers the event and unlatched the queue, so a title that waits on the flip event after latching recovers only if it submits further flips. The bound cannot be by time: the next release interrupt can arrive seconds later than a suppressed flip across loading stalls, and delivering an unexpected flip event on a queue that pairs flips with releases can crash; no fixed window short enough to unblock a waiting title keeps such a title alive. Whether the console raises the event from the flip itself or from an interrupt its flip sequence contains is not measured; nor is how the console behaves for a title that waits on the flip event and also raises release interrupts on the same queue. +- [Heap mirror reference storage](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ReferenceStorage.cpp) (libSceAgcDriver): the 8-buffer/2 GiB idle cap and 2 s reuse window are unmeasured choices. Idle release is polled by a take, an eligible return, a heap mirror sweep or a present; without another poll, expired buffers remain until process shutdown. A title taking references less often than every 2 s reallocates them. The cap covers idle storage only, not buffers still held by active references. - [Host import windows and budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): a registered range larger than 64 MiB is imported one 64 MiB window at a time for buffer and fill accesses; an access across window edges imports the cells it touches, grown to the union of the range's imports it overlaps. An address-based build (`AcquireRegistered`) imports the range whole and replaces its windows. Live imports stay under a budget: `APS5_HOST_IMPORT_MIB`, else the larger of 6 GiB and half the physical memory, at most the largest host heap less half that heap, kept between 2 and 4 GiB (imports, type-3 mirrors and buffers and textures placed in host memory share it; on a 16 GiB machine whose host heap is half the memory, a 2 GiB reserve left too little for the rest, and on a 32 GiB machine a reserve of half its 16 GiB heap left imports 8 GiB, less than the 10466 MiB one title holds at once). Where the device enables `VK_EXT_memory_budget`, that host heap's reported budget less its other usage and an eighth, in 64 MiB steps, read at most once a second, also bounds the default budget once an import's `vkAllocateMemory` has failed; a set `APS5_HOST_IMPORT_MIB` is never lowered by the report. The report's other usage includes the host-visible copies and heap mirrors that stand in for refused imports, so it grows with the refusals; the first time the report would lower the budget, `[gpu] host import budget: host heap ...` prints the heap's reported budget and usage and the live imports. An import's `vkAllocateMemory` failure within an eighth of the budget, or the third in a row, lowers the budget to the live imports until the device or the configured budget changes; an isolated failure further below only marks its range failed. Live imports are not capped by `maxMemoryAllocationCount`: NVIDIA's Windows driver reports 4096, while Village held 27,167 live imports without a failure. A new import beyond it retires the least recently used imports not used in the current or the previous frame (a frame ends at a flip packet or a CPU flip), so a set used in every frame keeps its imports and what does not fit is copied, as below the budget; until two frames have ended imports age by lookups, an import being evictable 256 lookups after its last use; after that, without flips (a loading stretch, compute-only work) no frame ends and nothing more becomes evictable, so imports that do not fit are copied instead of evicting by recency; one that would not fit even then is refused before anything is retired, and imports made when a range is mapped never evict. An address-based build does not evict either, except for a range the budget refused (or remembers refusing) and no mirror serves whose copy would take the build's copies past `APS5_ADDRESS_COPY_MAX_MIB`; that eviction also takes imports the recency rule protects whose single range is releasable and write-watched, so a heap mirror can serve it, as long as the build has not taken them yet. A heap mirror, sparse or not, whose range has left the write watch is refreshed in full at each use, every block counted as written, unless the build can import its range first, evicting for it as the copy cap does, with the imports of earlier builds no longer held; a build plans its range again only once the import path takes the range. A request that may not evict (an address-based build, an import when a range is mapped) remembers a refused range or block, with the live bytes and the budget at the refusal, and is answered from that memory, still counted as refused, until live imports drop below those bytes, the budget changes or one of its ranges changes. A request that may evict remembers its refusal too, answering later requests for the same run from memory with the room the refusal left (a request whose range fits in that room steps down to a smaller block without scanning the imports again, and a refusal that stepped down is remembered for each block it tried), while no import was made or retired, the budget has not grown, the import window and the run are unchanged, and no import can have become evictable since: no frame has ended (or, before two frames have ended, 256 lookups have not passed), and no build holding an import two frames old has raised its floor. A join remembers its refusal the same way, apart from other requests for the same run, so a buffer bound across refused ranges does not try the join again at every upload. The shutdown line `[gpu] host imports at shutdown` gives the live, made, refused (and remembered) and failed counts per requester. The 64 MiB window, the two-frame protection, the 2 GiB reserve, the eighth, the 64 MiB steps and the three failures are choices, not measured. - [Piece-split ranges over the host import budget](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver): an address-based build imports a registered range larger than the device's largest allocation in pieces. If the budget refuses a piece, the range is mirrored or copied without the evicting retry other refused ranges get when their copy would pass `APS5_ADDRESS_COPY_MAX_MIB`, so such a copy can still exceed that cap and end the build. - [Device-local staging](../../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) - a written or atomic buffer element inside a host import, including one an address-based compute build reaches through its pointers, is copied to device memory before the dispatch and copied back whole after it, so a CPU store into that range in between is rolled back.