diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 8b5909b8e026..604162e46c7c 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -949,6 +949,12 @@ if(BUILD_TESTING) set_tests_properties(agc_storage_baseline_settled PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_readonly_fallback_reuse COMMAND agc_driver_recorder_tests --readonly-fallback-only) + set_tests_properties(agc_readonly_fallback_reuse PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_readonly_fallback_reuse PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_storage_written_buffer_order COMMAND agc_driver_recorder_tests --written-buffer-order-only) set_tests_properties(agc_storage_written_buffer_order PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) if(WIN32) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index dcc394cc75d7..4d58f9790e32 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -3164,8 +3164,33 @@ void GuestBufferMemory::UploadFinish(bool addressable) { void GuestBufferMemory::copyRegion(Region& region, bool addressable) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + static const bool noReuse = std::getenv("APS5_NO_READONLY_FALLBACK_REUSE") != nullptr; const auto bytes = region.end - region.begin; - const auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::BufferUpload); + auto* snapshots = !noReuse && GuestMemory::GpuMutex().HeldByThisThread() ? Recorder::Active() : nullptr; + if (snapshots != nullptr && (snapshots->Device() != context.device || addressable || region.sparse || + !(region.hostBacked || region.writable) || WritesOverlap(region.begin, static_cast(bytes)))) snapshots = nullptr; + std::uint64_t generation = 0, registryGeneration = 0; + if (snapshots != nullptr) { + GuestMemory::FlushGpuWrites(region.begin, static_cast(bytes)); + generation = GuestMemory::CollectWritesUncached(region.begin, static_cast(bytes)); + registryGeneration = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (generation != 0) { + if (auto cached = snapshots->ReusableDrawSnapshot(region.begin, static_cast(bytes), Recorder::SnapshotUse::Storage, nullptr, generation)) { + region.buffer = std::move(cached); + region.snapshot.clear(); + if (profile) { + static std::atomic hits{0}, savedBytes{0}; + const auto saved = savedBytes.fetch_add(bytes, std::memory_order_relaxed) + bytes; + const auto count = hits.fetch_add(1, std::memory_order_relaxed) + 1; + if (count % 1000 == 0) AgcDriver::ProfilePrint_nid_no_patch("[buffers] readonly fallback reuse: %llu copies avoided (%.0f MiB)\n", static_cast(count), saved / 1048576.0); + } + return; + } + } + } + const auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (addressable ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u) | + (snapshots != nullptr && generation != 0 ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT : 0u); std::uint64_t readBytes = 0; bool inImport = false; if (profile && context.hostImportAlignment != 0) { @@ -3187,8 +3212,6 @@ void GuestBufferMemory::copyRegion(Region& region, bool addressable) { if (region.buffer == nullptr) region.buffer = std::make_shared(context, static_cast(allocated), usage); const auto readStart = std::chrono::steady_clock::now(); if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed); - // Named for the [hooksync] attribution: the reads below go through the flush hook. - const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::BufferUpload); if (region.hostBacked || region.writable) { // Only ranges a descriptor may write need the copy the write-back diffs against; the // registered ranges of an address-based shader are read-only to it (AddressRanges). @@ -3218,6 +3241,7 @@ void GuestBufferMemory::copyRegion(Region& region, bool addressable) { readBytes += region.snapshot.size(); } region.snapshot.clear(); + if (snapshots != nullptr && generation != 0) snapshots->KeepDrawSnapshot(region.begin, static_cast(bytes), generation, registryGeneration, region.buffer); if (profile) readUs.fetch_add(microsecondsSince(readStart), std::memory_order_relaxed); if (profile) { static std::mutex reasonsMutex; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 815cf01fcbba..ae36b5d6e770 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -1931,6 +1931,245 @@ void drawDescriptorWriteTests(const Device& device, Recorder& recorder) { recorder.Activate(); } +void readonlyFallbackReuseTests(const Device& device, Recorder& recorder, std::unique_lock& gpu) { + const auto& context = device.GetContext(); + constexpr std::size_t bytes = 262144; + Require(context.hostImportAlignment != 0, "readonly fallback fixture requires host import support"); + void* block = AllocateWatched(bytes, 65536); + Require(block != nullptr, "readonly fallback fixture requires tracked memory"); + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + const auto address = reinterpret_cast(block); + Require(HostImportFor(context, address, bytes) == nullptr, "fixture must use the actual unregistered copy fallback"); + std::memset(block, 0x11, bytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + const auto read = [&] { + auto memory = std::make_shared(context); + memory->AddReadable(address, bytes); + memory->Upload(false); + return memory; + }; + const auto descriptor = [&](const std::shared_ptr& memory) { + std::uint32_t adjustment = 0; + const auto result = memory->Descriptor(address, bytes, adjustment); + Require(adjustment == 0 && result.offset == 0 && result.range == bytes, "fallback copy lost descriptor extent or alignment"); + return result; + }; + const auto first = read(); + const auto firstView = descriptor(first); + Require(descriptor(read()).buffer == firstView.buffer, "unchanged readonly fallback was copied again"); + + auto oldOutput = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, firstView.buffer, 0, oldOutput->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(first); + recorder.Keep(oldOutput); + + std::memset(block, 0x22, bytes); + const auto afterCpu = read(); + const auto cpuView = descriptor(afterCpu); + Require(cpuView.buffer != firstView.buffer, "CPU overwrite mutated a copy still held by a queued reader"); + Require(descriptor(read()).buffer == cpuView.buffer, "new readonly fallback copy was not retained"); + auto cpuOutput = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, cpuView.buffer, 0, cpuOutput->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(afterCpu); + recorder.Keep(cpuOutput); + recorder.Sync(); + Require(std::all_of(oldOutput->Bytes().begin(), oldOutput->Bytes().end(), [](std::byte value) { return value == std::byte{0x11}; }), + "queued GPU reader lost its original bytes after CPU overwrite"); + Require(std::all_of(cpuOutput->Bytes().begin(), cpuOutput->Bytes().end(), [](std::byte value) { return value == std::byte{0x22}; }), + "new GPU reader did not receive the CPU overwrite"); + + auto written = std::make_shared(context); + written->AddWritable(address, bytes); + written->Upload(false); + Require(descriptor(written).buffer != cpuView.buffer, "writable fallback bound an immutable reader's cached buffer"); + + auto gpuResult = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + commands = recorder.Commands(); + context.Function("vkCmdFillBuffer")(commands, gpuResult->Handle(), 0, bytes, 0x33333333u); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(gpuResult); + recorder.OnComplete([gpuResult, address] { AgcDriver::GuestMemory::Write(address, gpuResult->Bytes()); }); + recorder.NotePendingWrite(address, bytes); + const auto afterGpu = read(); + Require(descriptor(afterGpu).buffer != cpuView.buffer, "fallback cache reused bytes before a pending GPU result was published"); + Require(std::all_of(static_cast(block), static_cast(block) + bytes, + [](std::byte value) { return value == std::byte{0x33}; }), "pending GPU producer did not publish its bytes"); + Require(descriptor(read()).buffer == descriptor(afterGpu).buffer, "published GPU result was not retained for a second reader"); + auto gpuOutput = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, descriptor(afterGpu).buffer, 0, gpuOutput->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(afterGpu); + recorder.Keep(gpuOutput); + recorder.Sync(); + Require(std::all_of(gpuOutput->Bytes().begin(), gpuOutput->Bytes().end(), [](std::byte value) { return value == std::byte{0x33}; }), + "new GPU reader did not receive the published GPU result"); + + std::vector plain(4096, std::byte{0x44}); + const auto plainAddress = reinterpret_cast(plain.data()); + Require(AgcDriver::GuestMemory::CollectWritesUncached(plainAddress, plain.size()) == 0, + "untracked fixture unexpectedly has a generation"); + GuestBufferMemory untrackedFirst(context), untrackedSecond(context); + untrackedFirst.AddReadable(plainAddress, plain.size()); + untrackedSecond.AddReadable(plainAddress, plain.size()); + untrackedFirst.Upload(false); + untrackedSecond.Upload(false); + std::uint32_t firstAdjustment = 0, secondAdjustment = 0; + Require(untrackedFirst.Descriptor(plainAddress, plain.size(), firstAdjustment).buffer != + untrackedSecond.Descriptor(plainAddress, plain.size(), secondAdjustment).buffer, + "untracked generation-zero bytes were cached"); + auto prepareContext = context; + prepareContext.hostImportAlignment = 0; + gpu.unlock(); + std::shared_ptr unlockedFirst, unlockedSecond; + try { + unlockedFirst = std::make_shared(prepareContext); + unlockedSecond = std::make_shared(prepareContext); + unlockedFirst->AddReadable(address, bytes); + unlockedSecond->AddReadable(address, bytes); + unlockedFirst->UploadPrepare(false); + unlockedSecond->UploadPrepare(false); + } catch (...) { + gpu.lock(); + throw; + } + gpu.lock(); + unlockedFirst->UploadFinish(false); + unlockedSecond->UploadFinish(false); + Require(descriptor(unlockedFirst).buffer != descriptor(unlockedSecond).buffer && + descriptor(unlockedFirst).buffer != descriptor(afterGpu).buffer, + "unlocked upload reused a recorder's cached buffer"); + + const auto cached = recorder.ReusableDrawSnapshot(address, bytes); + Require(cached != nullptr, "fallback copy was not retained for eviction test"); + const auto registry = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + const auto generation = AgcDriver::GuestMemory::CollectWritesUncached(address, bytes); + for (std::size_t size = 1; size <= Recorder::DrawSnapshotEntries; ++size) { + recorder.KeepDrawSnapshot(address, size, generation, registry, cached); + } + Require(recorder.ReusableDrawSnapshot(address, bytes) == nullptr, "fallback copy survived entry-budget eviction"); + auto evictedOutput = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, descriptor(afterGpu).buffer, 0, evictedOutput->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(afterGpu); + recorder.Keep(evictedOutput); + recorder.Sync(); + Require(std::all_of(evictedOutput->Bytes().begin(), evictedOutput->Bytes().end(), + [](std::byte value) { return value == std::byte{0x33}; }), + "eviction invalidated a live GPU consumer's copy"); + +#ifdef _WIN32 + const auto beforeReset = read(); + const auto beforeResetView = descriptor(beforeReset); + GuestArena::GuestArenaReset_nid_postfix(block, bytes); + GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(address, bytes); + std::memset(block, 0x55, bytes); + const auto afterReset = read(); + Require(descriptor(afterReset).buffer != beforeResetView.buffer, + "private recommit reused bytes from the previous mapping at the same address"); + Require(std::all_of(cached->Bytes().begin(), cached->Bytes().end(), + [](std::byte value) { return value == std::byte{0x33}; }), + "private recommit overwrote an earlier consumer's immutable bytes"); + + struct Section { + HANDLE handle = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, 0, bytes, nullptr); + ~Section() { if (handle != nullptr) CloseHandle(handle); } + } firstSection, secondSection; + Require(firstSection.handle != nullptr && secondSection.handle != nullptr, "cannot allocate mapped fallback sections"); + GuestArena::GuestArenaReset_nid_postfix(block, bytes); + GuestArena::GuestArenaMap_nid_postfix(block, bytes, firstSection.handle, 0, PAGE_READWRITE); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(address, bytes); + MEMORY_BASIC_INFORMATION mapping{}; + Require(VirtualQuery(block, &mapping, sizeof mapping) == sizeof mapping && mapping.Type == MEM_MAPPED, + "mapped fallback fixture is not backed by a shared section"); + std::memset(block, 0x77, bytes); + const auto mappedFirst = read(); + const auto mappedFirstView = descriptor(mappedFirst); + Require(descriptor(read()).buffer == mappedFirstView.buffer, "unchanged mapped fallback was copied again"); + auto mappedOldOutput = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, mappedFirstView.buffer, 0, mappedOldOutput->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(mappedFirst); + recorder.Keep(mappedOldOutput); + std::memset(block, 0x88, bytes); + const auto mappedCpu = read(); + Require(descriptor(mappedCpu).buffer != mappedFirstView.buffer, + "mapped CPU overwrite reused a held immutable copy"); + GuestArena::GuestArenaReset_nid_postfix(block, bytes); + GuestArena::GuestArenaMap_nid_postfix(block, bytes, secondSection.handle, 0, PAGE_READWRITE); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(address, bytes); + std::memset(block, 0x99, bytes); + const auto mappedRemap = read(); + Require(descriptor(mappedRemap).buffer != descriptor(mappedCpu).buffer, + "shared-section remap reused bytes from the previous mapping at the same address"); + auto remapOutput = std::make_shared(context, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + CopyBuffer(context, commands, descriptor(mappedRemap).buffer, 0, remapOutput->Handle(), 0, bytes); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Keep(mappedRemap); + recorder.Keep(remapOutput); + recorder.Sync(); + Require(std::all_of(mappedOldOutput->Bytes().begin(), mappedOldOutput->Bytes().end(), + [](std::byte value) { return value == std::byte{0x77}; }), + "mapped overwrite or remap changed a queued consumer's original bytes"); + Require(std::all_of(remapOutput->Bytes().begin(), remapOutput->Bytes().end(), + [](std::byte value) { return value == std::byte{0x99}; }), + "new GPU reader did not receive the remapped section's bytes"); + const auto currentMapped = recorder.ReusableDrawSnapshot(address, bytes); + Require(currentMapped != nullptr && std::all_of(currentMapped->Bytes().begin(), currentMapped->Bytes().end(), + [](std::byte value) { return value == std::byte{0x99}; }), + "remapped section's current cached bytes are incorrect"); + GuestArena::GuestArenaReset_nid_postfix(block, bytes); + GuestArena::GuestArenaCommit_nid_postfix(block, bytes / 2, PAGE_READWRITE, bytes / 2); + GuestAllocations::GuestAllocationsInvalidate_nid_postfix(address, bytes); + std::memset(block, 0x66, bytes / 2); + const auto sparseFirst = read(); + const auto sparseSecond = read(); + std::uint32_t sparseAdjustment = 0; + const auto sparseFirstView = sparseFirst->Descriptor(address, bytes, sparseAdjustment); + const auto sparseSecondView = sparseSecond->Descriptor(address, bytes, sparseAdjustment); + Require(sparseFirstView.range == bytes / 2 && sparseSecondView.range == bytes / 2 && sparseAdjustment == 0, + "sparse fallback did not clip the descriptor to committed backing"); + Require(sparseFirstView.buffer != sparseSecondView.buffer, + "sparse fallback buffer was retained for reuse"); + Require(recorder.ReusableDrawSnapshot(address, bytes) == nullptr, + "sparse upload left a previous full-span snapshot reusable"); + std::cout << "Mapped CPU writes, shared-section remap, private recommit and sparse bypass tests passed\n"; +#endif + + recorder.Sync(); + std::cout << "Readonly fallback reuse, CPU overwrite, held reader, writer bypass, pending GPU publication and untracked tests passed\n"; +} + void drawSnapshotReuseTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); constexpr std::size_t bytes = 65536; @@ -6150,6 +6389,11 @@ int main(int argc, char** argv) { std::cout << "host imports " << (PrepareImportWatch(device.GetContext()) == ImportWatch::Unwatch ? "are compared" : "stay watched") << '\n'; Recorder recorder(device.GetContext()); recorder.Activate(); + if (argc == 2 && std::string_view(argv[1]) == "--readonly-fallback-only") { + if (device.GetContext().hostImportAlignment == 0 || !AgcDriver::GuestMemory::WriteWatched()) return 77; + readonlyFallbackReuseTests(device, recorder, gpu); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--copied-direct-only") { copiedDirectOrderingTests(device, recorder); return 0;