From a8d5f1218895d99fb7c27e51aa8a1ce55c8d06cc Mon Sep 17 00:00:00 2001 From: milescoviello Date: Tue, 3 Nov 2026 14:00:22 -0500 Subject: [PATCH 1/6] perf(agc): order a storage flush behind its GPU writer when the write-back is a GPU copy A label's readback store and a storage refresh flush pending storage images before going on. When every flushed image lives in a host import, its write-back is a copy the recorder records into the import, so it runs after the image's writer and before anything recorded later, including the label. The CPU wait for the writer is skipped then. A refresh whose flush skipped the wait re-uploads its layers from the import on the GPU instead of comparing guest bytes the copy has not reached yet. APS5_NO_FLUSH_GPU_ORDERED=1 and APS5_NO_REFRESH_GPU_ORDERED=1 restore the waits. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/Graphics/src/Texture.cpp | 54 ++++- .../tests/execution/LabelOrderedFlush.cpp | 201 ++++++++++++++++++ 2 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 14a15c3f38a2..9cea307d0f89 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -2247,6 +2247,13 @@ bool StorageTexture::RefreshLayers(std::uint32_t first, std::uint32_t count) { return refresh(firstLayer, lastLayer); } +namespace { + +bool& GpuOrderedFlush(); +bool& GpuOrderedSkipped(); + +} + bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) { const bool profile = LookupOutcomes::Profiled(); auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -2311,7 +2318,20 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) handedOver.clear(); handedUnits.clear(); const auto handOverKeys = alias == nullptr && !scoped ? clearHandOver() : std::nullopt; - const bool flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None); + static const bool refreshOrdered = std::getenv("APS5_NO_REFRESH_GPU_ORDERED") == nullptr; + const bool orderedEligible = refreshOrdered && !scoped && alias == nullptr && !handOverKeys.has_value() && descriptor.dccAddress == 0 && !layerPending.Any() && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr; + bool orderedSkipped = false; + bool flushed = false; + { + const bool previousOrdered = GpuOrderedFlush(); + const bool previousSkipped = GpuOrderedSkipped(); + GpuOrderedFlush() = previousOrdered || orderedEligible; + GpuOrderedSkipped() = false; + flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None); + orderedSkipped = orderedEligible && GpuOrderedSkipped(); + GpuOrderedFlush() = previousOrdered; + GpuOrderedSkipped() = previousSkipped || GpuOrderedSkipped(); + } if (profile && flushed) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start); // `original` holds the guest bytes the image was last uploaded from or written back as; while the // guest memory and the DCC keys still match, the image content is current. Pages nobody wrote @@ -2434,7 +2454,7 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (changed[layer] && layerPending[layer]) pendingChanged = true; } - if (!unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast(guestBytes))) && equalsOriginal()) { + if (!orderedSkipped && !unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast(guestBytes))) && equalsOriginal()) { unchanged = true; stamped = false; if (!scoped) outside.clear(); @@ -2463,6 +2483,12 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) } } } + if (orderedSkipped) { + changed.assign(trackedLayers, true); + unchanged = false; + anyBorrowed = false; + borrowed.clear(); + } const bool outsideChanged = std::any_of(outside.begin(), outside.end(), [](bool value) { return value; }); if (scoped) noteLayersBehind(firstLayer, lastLayer, outside); else layerBehind.clear(); @@ -4538,6 +4564,20 @@ void waitForIdlePublish(const char* reason) { } +namespace { + +bool& GpuOrderedFlush() { + thread_local bool flag = false; + return flag; +} + +bool& GpuOrderedSkipped() { + thread_local bool flag = false; + return flag; +} + +} + bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited, VkDevice device) { static_cast(FlushDepthPlanes(address, bytes)); if (published != nullptr) *published = false; @@ -4646,7 +4686,9 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons } std::lock_guard gpu(GuestMemory::GpuMutex()); if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) == nullptr; }))) ownNotes.clear(); - if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion()) { + const bool orderedSkip = GpuOrderedFlush() && Recorder::Active() != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [](const std::shared_ptr& texture) { return HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) != nullptr; }); + if (orderedSkip) GpuOrderedSkipped() = true; + if (auto* recorder = Recorder::Active(); !orderedSkip && recorder != nullptr && !Recorder::InCompletion()) { try { static const bool wholeRange = std::getenv("APS5_FLUSH_WAIT_WHOLE_RANGE") != nullptr; const bool publishes = !wholeRange && scope != PublishScope::None; @@ -5078,6 +5120,12 @@ std::size_t StorageTexture::StoreSmallPending(const char* reason) { void StorageTexture::StoreReadbackBeforeLabel(const char* reason, VkDevice device) { static const bool disabled = std::getenv("APS5_NO_LABEL_READBACK_STORE") != nullptr; if (disabled) return; + static const bool gpuOrdered = std::getenv("APS5_NO_FLUSH_GPU_ORDERED") == nullptr; + struct OrderedScope { + bool previous; + explicit OrderedScope(bool on) : previous(GpuOrderedFlush()) { GpuOrderedFlush() = previous || on; } + ~OrderedScope() { GpuOrderedFlush() = previous; } + } ordered{gpuOrdered}; std::vector> ranges; std::vector> occasional; { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp new file mode 100644 index 000000000000..1e8a86bad8e3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp @@ -0,0 +1,201 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +constexpr std::uint32_t Side = 256; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t Block = 0x10000; +constexpr std::size_t SurfaceBytes = 4 * Block; +constexpr std::uint32_t Untouched = 0xdeadbeefu; +constexpr std::uint32_t Stored = 0x12345678u; + +alignas(256) constexpr std::array StoreCode{ + 0x7e040300, 0x7e060280, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000, +}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +std::array TextureDescriptor(const std::uint32_t* texels) { + const auto address = AddressOf(texels); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +void Dispatch(std::vector& words, std::span code, std::uint32_t threads, std::uint32_t resource, std::span userData, std::uint32_t groups) { + const auto registers = [&](const std::uint32_t first, const std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{threads, 1, 1}; + registers(0x207, shape); + const auto address = AddressOf(code.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array layout{resource}; + registers(0x213, layout); + registers(0x240, userData); + words.insert(words.end(), {0xc0031500u, groups, 1, 1, 0x41u}); +} + +void Store(std::vector& words, const std::uint32_t* texels) { + std::array userData{}; + const auto texture = TextureDescriptor(texels); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + Dispatch(words, StoreCode, 8, static_cast(userData.size()) << 1u, userData, 1); +} + +class Surface { +public: + Surface() { +#ifdef _WIN32 + region = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(SurfaceBytes, Block)); + if (region != nullptr) GuestArena::GuestArenaCommit_nid_postfix(region, SurfaceBytes, PAGE_READWRITE, SurfaceBytes); +#else + void* mapped = mmap(nullptr, SurfaceBytes + Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + Block - 1) & ~std::uintptr_t{Block - 1}; + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned != begin + Block) munmap(reinterpret_cast(aligned + SurfaceBytes), begin + Block - aligned); + region = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(region, SurfaceBytes); + } +#endif + Require(region != nullptr, "label ordered flush: cannot allocate the guest surface"); + texels = reinterpret_cast(region); + std::fill(texels, texels + SurfaceBytes / 4u, Untouched); + GuestAllocations::Mutation().Add(region, SurfaceBytes, true, true, true); + } + + ~Surface() { + GuestAllocations::Mutation().Remove(region); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(region, SurfaceBytes); + GuestArena::GuestArenaRelease_nid_postfix(region, SurfaceBytes); +#else + munmap(region, SurfaceBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(region, SurfaceBytes); +#endif + } + + Surface(const Surface&) = delete; + Surface& operator=(const Surface&) = delete; + + std::uint32_t* Texels() const { return texels; } + +private: + std::uint8_t* region = nullptr; + std::uint32_t* texels = nullptr; +}; + +void LabelAfterStore(const Surface& surface, std::uint32_t* label, std::uint32_t* gate) { + std::vector words; + const auto gateAddress = AddressOf(gate); + words.insert(words.end(), {0xc0053c00u, 0x13u, static_cast(gateAddress), static_cast(gateAddress >> 32u), 1u, 0xffffffffu, 0x19u}); + Store(words, surface.Texels()); + const auto labelAddress = AddressOf(label); + words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), static_cast(labelAddress), static_cast(labelAddress >> 32u), 1u, 0, 0}); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + const auto address = AddressOf(surface.Texels()); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + for (;;) { + { + std::unique_lock hold(AgcDriver::GuestMemory::GpuMutex()); + if (StorageTexture::FindPending(address, SurfaceBytes) != nullptr) break; + } + Require(std::chrono::steady_clock::now() < deadline, "label ordered flush: the image store was never recorded"); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + Require(std::atomic_ref(*label).load() == 0, "label ordered flush: the label landed before the gate opened"); + std::atomic_ref(*gate).store(1); + while (std::atomic_ref(*label).load() == 0) { + Require(std::chrono::steady_clock::now() < deadline, "label ordered flush: the label never landed"); + std::this_thread::yield(); + } + std::vector landed(SurfaceBytes / sizeof(std::uint32_t)); + std::memcpy(landed.data(), surface.Texels(), SurfaceBytes); + const auto stored = std::count(landed.begin(), landed.end(), Stored); + Require(stored >= 8, "label ordered flush: the label was visible before the image store it follows"); + AgcDriverWaitIdle_nid_postfix(); +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + struct ShutdownGuard { + ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} } + } shutdown; + alignas(256) static std::uint32_t words[64]{}; + GuestAllocations::Mutation().Add(words, sizeof(words), true, true, true); + struct Unregister { + ~Unregister() { GuestAllocations::Mutation().Remove(words); } + } unregister; + constexpr int Runs = 8; + for (int run = 0; run < Runs; ++run) { + Surface surface; + auto* label = &words[0]; + auto* gate = &words[32]; + *label = 0; + *gate = 0; + LabelAfterStore(surface, label, gate); + } + AgcDriverShutdown_nid_postfix(); + std::puts("label ordered flush tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From 849a6694ed19863f6a810c3dd65dce2601aef89f Mon Sep 17 00:00:00 2001 From: milescoviello Date: Tue, 3 Nov 2026 14:19:24 -0500 Subject: [PATCH 2/6] test(agc): put the label right after the store in agc_driver_label_ordered_flush WAIT_REG_MEM is evaluated by the queue worker, so a gate in front of the store kept it from being recorded at all. The label now follows the store directly, while the store is still pending on the GPU, which is the case the ordered flush handles. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/LabelOrderedFlush.cpp | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp index 1e8a86bad8e3..5903283c7b7e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp @@ -134,27 +134,14 @@ class Surface { std::uint32_t* texels = nullptr; }; -void LabelAfterStore(const Surface& surface, std::uint32_t* label, std::uint32_t* gate) { +void LabelAfterStore(const Surface& surface, std::uint32_t* label) { std::vector words; - const auto gateAddress = AddressOf(gate); - words.insert(words.end(), {0xc0053c00u, 0x13u, static_cast(gateAddress), static_cast(gateAddress >> 32u), 1u, 0xffffffffu, 0x19u}); Store(words, surface.Texels()); const auto labelAddress = AddressOf(label); words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), static_cast(labelAddress), static_cast(labelAddress >> 32u), 1u, 0, 0}); Packet packet{words.data(), static_cast(words.size()), 0, {}}; AgcDriver::Submit(&packet, 0); - const auto address = AddressOf(surface.Texels()); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); - for (;;) { - { - std::unique_lock hold(AgcDriver::GuestMemory::GpuMutex()); - if (StorageTexture::FindPending(address, SurfaceBytes) != nullptr) break; - } - Require(std::chrono::steady_clock::now() < deadline, "label ordered flush: the image store was never recorded"); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - Require(std::atomic_ref(*label).load() == 0, "label ordered flush: the label landed before the gate opened"); - std::atomic_ref(*gate).store(1); while (std::atomic_ref(*label).load() == 0) { Require(std::chrono::steady_clock::now() < deadline, "label ordered flush: the label never landed"); std::this_thread::yield(); @@ -186,10 +173,8 @@ int main() { for (int run = 0; run < Runs; ++run) { Surface surface; auto* label = &words[0]; - auto* gate = &words[32]; *label = 0; - *gate = 0; - LabelAfterStore(surface, label, gate); + LabelAfterStore(surface, label); } AgcDriverShutdown_nid_postfix(); std::puts("label ordered flush tests passed"); From f4013fab1471a561c8bec11c3553986f4b15c393 Mon Sep 17 00:00:00 2001 From: milescoviello Date: Tue, 3 Nov 2026 16:41:36 -0500 Subject: [PATCH 3/6] docs: list APS5_NO_FLUSH_GPU_ORDERED and APS5_NO_REFRESH_GPU_ORDERED Co-Authored-By: Claude Opus 5.5 --- docs/dev/ENVIRONMENT.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index b26ac9c91517..e4c391adf650 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -277,6 +277,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_FAST_REVALIDATE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | always repeats the resource lookups instead of using the fast proof | | `APS5_NO_FILL_CLEAR` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` | | disables the clear-through-DCC-keys path in a fill | | `APS5_NO_FILL_HLE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` | | disables the fill-kernel HLE, so matching fills run as compute | +| `APS5_NO_FLUSH_GPU_ORDERED` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | waits on the CPU for a flushed storage image's GPU writer before a label's readback store, also when every flushed image's write-back is a GPU copy into its host import | | `APS5_NO_FLUSH_PRETEST` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | lists every pending image on FlushPending instead of testing whether a pending unit lies inside the range | | `APS5_NO_FOREIGN_WRITERS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | records no draw-written ranges as foreign writers for capture validation | | `APS5_NO_GATE_RETRY` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | retries no validation whose entry moved, leaving the dispatch or draw unvalidated | @@ -346,6 +347,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_RECORDER_SYNC_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | gives the draws the recorder waits for their own batch instead of recording them with the wait | | `APS5_NO_RECORD_COPIED_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | waits for copied-write draws at once instead of listing their write-back for later | | `APS5_NO_RECT_INDIRECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp` | | sends an indirect rectangle-list draw down the CPU path instead of the GPU-side path | +| `APS5_NO_REFRESH_GPU_ORDERED` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | waits on the CPU for the pending writers a storage refresh flushes and compares the guest bytes, instead of re-uploading the layers from the import behind the write-backs | | `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | drops the residency of linear color targets, so a target whose address has a host import, or whose import a draw would make, is copied in and out again | | `APS5_NO_RESIDENT_PRESENT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | always presents through guest memory instead of the resident image | | `APS5_NO_RESIDENT_TARGETS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | copies every render target in and out again instead of keeping it resident | From d3f7307afdf8752297fc8219277117a2d44e9dc5 Mon Sep 17 00:00:00 2001 From: milescoviello Date: Tue, 3 Nov 2026 16:44:06 -0500 Subject: [PATCH 4/6] test(agc): use a small storage image in agc_driver_label_ordered_flush A label stores pending linear and small (64 KiB or less) storage images before it; a 256 KiB image is stored there only when it was written occasionally, so the test checked an order the driver does not promise. The surface is now 128x128 R32UI, one 64 KiB block, and the test requires that size. Co-Authored-By: Claude Opus 5.5 --- .../tests/execution/LabelOrderedFlush.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp index 5903283c7b7e..ecd67a368570 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp @@ -35,12 +35,12 @@ namespace { using AgcDriver::Graphics::Require; using AgcDriver::Graphics::StorageTexture; -constexpr std::uint32_t Side = 256; +constexpr std::uint32_t Side = 128; constexpr std::uint32_t Format32UInt = 20; constexpr std::uint32_t TileR64KBX = 0x1b; constexpr std::uint32_t Type2D = 9; constexpr std::size_t Block = 0x10000; -constexpr std::size_t SurfaceBytes = 4 * Block; +constexpr std::size_t SurfaceBytes = Block; constexpr std::uint32_t Untouched = 0xdeadbeefu; constexpr std::uint32_t Stored = 0x12345678u; @@ -164,6 +164,11 @@ int main() { struct ShutdownGuard { ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} } } shutdown; + { + Surface probe; + const auto described = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(TextureDescriptor(probe.Texels()))).guestBytes; + Require(described == SurfaceBytes && described <= StorageTexture::SmallStoreBytes, "label ordered flush: the surface is not a small storage image"); + } alignas(256) static std::uint32_t words[64]{}; GuestAllocations::Mutation().Add(words, sizeof(words), true, true, true); struct Unregister { From 2da15973ce0a3fa5e4d9e574d6ee1df640544dad Mon Sep 17 00:00:00 2001 From: milescoviello Date: Tue, 3 Nov 2026 17:57:46 -0500 Subject: [PATCH 5/6] perf(agc): take the ordered flush only for images on the active recorder's device The write-backs are recorded into the active recorder, so an image of another device keeps the CPU wait. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index a00d1a77cf92..c761115716e4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -4693,7 +4693,8 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons } std::lock_guard gpu(GuestMemory::GpuMutex()); if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) == nullptr; }))) ownNotes.clear(); - const bool orderedSkip = GpuOrderedFlush() && Recorder::Active() != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [](const std::shared_ptr& texture) { return HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) != nullptr; }); + auto* const orderedRecorder = Recorder::Active(); + const bool orderedSkip = GpuOrderedFlush() && orderedRecorder != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [orderedRecorder](const std::shared_ptr& texture) { return texture->context.device == orderedRecorder->Device() && HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) != nullptr; }); if (orderedSkip) GpuOrderedSkipped() = true; if (auto* recorder = Recorder::Active(); !orderedSkip && recorder != nullptr && !Recorder::InCompletion()) { try { From b6007f69a9851871f24fd9d222220509562ef46e Mon Sep 17 00:00:00 2001 From: milescoviello Date: Tue, 3 Nov 2026 22:27:05 -0500 Subject: [PATCH 6/6] perf(agc): restore the ordered flush scope on any exit and skip only without a live shadow or lease; cover each branch Review of 2da15973ce (oneandonlydean): - The refresh's ordered scope is a destructor-restored guard, so a FlushPending that throws leaves no armed flag on its thread, and a refresh sets the flag to its own eligibility instead of inheriting an outer label scope. - FlushPending takes the ordered skip only when its publish scope is None or no live shadowed range or pending lease covers the range, since that wait also orders PublishShadow. - StorageTexture::TakeOrderedSkipCounts counts the skips taken. agc_driver_label_ordered_flush checks the label half and a storage refresh of a linear view over a pending tiled store, and its _disabled and _unimported runs check that neither skip is taken with the switches set or without a host import, with the same results. - agc_driver_ordered_refresh_failure injects a flush failure inside an armed refresh (StorageTexture::SetFlushFailureForTest) and checks that the flag is restored and that the next flush waits. Co-Authored-By: Claude Opus 5.5 --- .../Graphics/include/Texture.hpp | 8 + .../libSceAgcDriver/Graphics/src/Texture.cpp | 62 +++++- ...agc_driver_label_ordered_flush_tests.cmake | 10 + .../tests/execution/LabelOrderedFlush.cpp | 90 +++++++- .../tests/execution/OrderedRefreshFailure.cpp | 209 ++++++++++++++++++ 5 files changed, 364 insertions(+), 15 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 10737429984d..c73c428dc325 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -391,6 +391,14 @@ class StorageTexture : public std::enable_shared_from_this { std::uint64_t flushedAfterSkipSameSite; }; static HookSkipCounts TakeHookSkipCounts(); + struct OrderedSkipCounts { + std::uint64_t flushes; + std::uint64_t refreshes; + }; + static OrderedSkipCounts TakeOrderedSkipCounts(); + static void SetFlushFailureForTest(bool fail); + static int TakeFlushFailureForTest(); + static bool OrderedFlushArmedForTest(); // A consumer found the image current and used it (Refresh, the fast revalidation). void NoteProved() const; // Whether the storage cache holds this image (set and cleared by the cache under its mutex, the diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index c761115716e4..6870c4ed21e7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -2255,6 +2255,10 @@ namespace { bool& GpuOrderedFlush(); bool& GpuOrderedSkipped(); +std::atomic orderedFlushSkips{0}; +std::atomic orderedRefreshSkips{0}; +std::atomic flushFailureForTest{false}; +std::atomic flushFailureFiredForTest{0}; } @@ -2327,15 +2331,23 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) bool orderedSkipped = false; bool flushed = false; { - const bool previousOrdered = GpuOrderedFlush(); - const bool previousSkipped = GpuOrderedSkipped(); - GpuOrderedFlush() = previousOrdered || orderedEligible; - GpuOrderedSkipped() = false; + struct OrderedRefreshScope { + bool previousOrdered; + bool previousSkipped; + explicit OrderedRefreshScope(bool eligible) : previousOrdered(GpuOrderedFlush()), previousSkipped(GpuOrderedSkipped()) { + GpuOrderedFlush() = eligible; + GpuOrderedSkipped() = false; + } + ~OrderedRefreshScope() { + const bool skipped = GpuOrderedSkipped(); + GpuOrderedFlush() = previousOrdered; + GpuOrderedSkipped() = previousSkipped || skipped; + } + } ordered{orderedEligible}; flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None); orderedSkipped = orderedEligible && GpuOrderedSkipped(); - GpuOrderedFlush() = previousOrdered; - GpuOrderedSkipped() = previousSkipped || GpuOrderedSkipped(); } + if (orderedSkipped) orderedRefreshSkips.fetch_add(1, std::memory_order_relaxed); pruneAdoptedSources(); if (profile && flushed) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start); // `original` holds the guest bytes the image was last uploaded from or written back as; while the @@ -4694,8 +4706,26 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons std::lock_guard gpu(GuestMemory::GpuMutex()); if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) == nullptr; }))) ownNotes.clear(); auto* const orderedRecorder = Recorder::Active(); - const bool orderedSkip = GpuOrderedFlush() && orderedRecorder != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [orderedRecorder](const std::shared_ptr& texture) { return texture->context.device == orderedRecorder->Device() && HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) != nullptr; }); - if (orderedSkip) GpuOrderedSkipped() = true; + const auto publishesNothing = [&](const Recorder& active) { + if (scope == PublishScope::None) return true; + if (!LiveShadowedRanges(address, bytes, scope).empty()) return false; + constexpr std::uint64_t block = 65536; + constexpr auto top = std::numeric_limits::max(); + const auto first = address / block * block; + const auto end = bytes > top - address ? top : address + bytes; + const auto last = end > top - block ? end : (end + block - 1) / block * block; + return active.PendingLeaseSerial(first, static_cast(last - first)) == 0; + }; + const bool orderedSkip = GpuOrderedFlush() && orderedRecorder != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [orderedRecorder](const std::shared_ptr& texture) { return texture->context.device == orderedRecorder->Device() && HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) != nullptr; }) && publishesNothing(*orderedRecorder); + if (orderedSkip) { + GpuOrderedSkipped() = true; + orderedFlushSkips.fetch_add(1, std::memory_order_relaxed); + } + if (flushFailureForTest.exchange(false, std::memory_order_acq_rel)) { + for (const auto& texture : flush) texture->reconcilePending(); + flushFailureFiredForTest.store(GpuOrderedFlush() ? 2 : 1, std::memory_order_release); + throw std::runtime_error("StorageTexture::FlushPending: injected failure for a test"); + } if (auto* recorder = Recorder::Active(); !orderedSkip && recorder != nullptr && !Recorder::InCompletion()) { try { static const bool wholeRange = std::getenv("APS5_FLUSH_WAIT_WHOLE_RANGE") != nullptr; @@ -5575,6 +5605,22 @@ StorageTexture::HookSkipCounts StorageTexture::TakeHookSkipCounts() { return {flushedAfterSkip.exchange(0, std::memory_order_relaxed), flushedAfterSkipSameSite.exchange(0, std::memory_order_relaxed)}; } +StorageTexture::OrderedSkipCounts StorageTexture::TakeOrderedSkipCounts() { + return {orderedFlushSkips.exchange(0, std::memory_order_relaxed), orderedRefreshSkips.exchange(0, std::memory_order_relaxed)}; +} + +void StorageTexture::SetFlushFailureForTest(bool fail) { + flushFailureForTest.store(fail, std::memory_order_release); +} + +int StorageTexture::TakeFlushFailureForTest() { + return flushFailureFiredForTest.exchange(0, std::memory_order_acq_rel); +} + +bool StorageTexture::OrderedFlushArmedForTest() { + return GpuOrderedFlush(); +} + void StorageTexture::BumpPendingSerial() { pendingSerial.fetch_add(1, std::memory_order_release); } diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake new file mode 100644 index 000000000000..f74a4406cf28 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake @@ -0,0 +1,10 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_label_ordered_flush_disabled COMMAND agc_driver_label_ordered_flush_tests --disabled) + set_tests_properties(agc_driver_label_ordered_flush_disabled PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FLUSH_GPU_ORDERED=1;APS5_NO_REFRESH_GPU_ORDERED=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_label_ordered_flush_unimported COMMAND agc_driver_label_ordered_flush_tests --unimported) + set_tests_properties(agc_driver_label_ordered_flush_unimported PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_HOST_IMPORT=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_label_ordered_flush_disabled agc_driver_label_ordered_flush_unimported PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp index ecd67a368570..5640078d8be8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -48,17 +49,35 @@ alignas(256) constexpr std::array StoreCode{ 0x7e040300, 0x7e060280, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000, }; +alignas(256) constexpr std::array StoreRowsCode{ + 0x7e040300, 0x7e060210, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000, +}; + +alignas(256) constexpr std::array LoadRowCode{ + 0x7e020280, 0xf0001108, 0x00010300, 0x34080082, 0xbf8c3f70, 0xe0701000, 0x80000304, 0xbf810000, +}; + +constexpr std::uint32_t RowThreads = 32; +constexpr std::uint32_t LinearWidth = 256; +constexpr std::uint32_t LinearHeight = 64; +constexpr std::uint32_t TileLinear = 0; +constexpr std::uint32_t TgidYEnable = 1u << 8u; + +enum class Mode { Enabled, Disabled, Unimported }; + +alignas(256) std::array Output{}; + std::uint64_t AddressOf(const void* data) { return static_cast(reinterpret_cast(data)); } -std::array TextureDescriptor(const std::uint32_t* texels) { +std::array TextureDescriptor(const std::uint32_t* texels, std::uint32_t width = Side, std::uint32_t height = Side, std::uint32_t tile = TileR64KBX) { const auto address = AddressOf(texels); return { static_cast(address >> 8u), - static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u), - ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), - 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((width - 1u) & 3u) << 30u), + ((width - 1u) >> 2u) | ((height - 1u) << 14u), + 0xfacu | (tile << 20u) | (Type2D << 28u), 0u, 0u, 0u, @@ -66,7 +85,12 @@ std::array TextureDescriptor(const std::uint32_t* texels) { }; } -void Dispatch(std::vector& words, std::span code, std::uint32_t threads, std::uint32_t resource, std::span userData, std::uint32_t groups) { +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = AddressOf(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +void Dispatch(std::vector& words, std::span code, std::uint32_t threads, std::uint32_t resource, std::span userData, std::uint32_t groups, std::uint32_t rows = 1) { const auto registers = [&](const std::uint32_t first, const std::span values) { words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); words.push_back(first); @@ -80,7 +104,7 @@ void Dispatch(std::vector& words, std::span const std::array layout{resource}; registers(0x213, layout); registers(0x240, userData); - words.insert(words.end(), {0xc0031500u, groups, 1, 1, 0x41u}); + words.insert(words.end(), {0xc0031500u, groups, rows, 1, 0x41u}); } void Store(std::vector& words, const std::uint32_t* texels) { @@ -153,9 +177,44 @@ void LabelAfterStore(const Surface& surface, std::uint32_t* label) { AgcDriverWaitIdle_nid_postfix(); } +void SubmitWords(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); +} + +void LoadRow(const Surface& surface) { + Output.fill(0xdeadbeefu); + std::array userData{}; + const auto buffer = BufferDescriptor(Output.data(), static_cast(Output.size() * sizeof(std::uint32_t))); + const auto texture = TextureDescriptor(surface.Texels(), LinearWidth, LinearHeight, TileLinear); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + std::vector words; + Dispatch(words, LoadRowCode, RowThreads, static_cast(userData.size()) << 1u, userData, 1); + SubmitWords(words); +} + +void RefreshAfterStore(const Surface& surface) { + LoadRow(surface); + Require(std::all_of(Output.begin(), Output.end(), [](std::uint32_t value) { return value == Untouched; }), "label ordered flush: the linear view did not read the untouched guest bytes"); + std::array userData{}; + const auto texture = TextureDescriptor(surface.Texels()); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + std::vector words; + Dispatch(words, StoreRowsCode, Side, (static_cast(userData.size()) << 1u) | TgidYEnable, userData, 1, Side); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + LoadRow(surface); + for (std::uint32_t index = 0; index < RowThreads; ++index) { + Require(Output[index] == Stored, "label ordered flush: the refreshed linear view reads " + std::to_string(Output[index]) + " at texel " + std::to_string(index) + " instead of the tiled store"); + Require(Output[index] == surface.Texels()[index], "label ordered flush: the refreshed linear view disagrees with guest memory at texel " + std::to_string(index)); + } } -int main() { +} + +int main(int argc, char** argv) { try { { const auto device = OpenVulkanTestDevice(); @@ -174,6 +233,9 @@ int main() { struct Unregister { ~Unregister() { GuestAllocations::Mutation().Remove(words); } } unregister; + const std::string option = argc > 1 ? argv[1] : ""; + const Mode mode = option == "--disabled" ? Mode::Disabled : option == "--unimported" ? Mode::Unimported : Mode::Enabled; + static_cast(StorageTexture::TakeOrderedSkipCounts()); constexpr int Runs = 8; for (int run = 0; run < Runs; ++run) { Surface surface; @@ -181,6 +243,20 @@ int main() { *label = 0; LabelAfterStore(surface, label); } + const auto labelCounts = StorageTexture::TakeOrderedSkipCounts(); + if (mode == Mode::Enabled) Require(labelCounts.flushes != 0, "label ordered flush: no label store took the GPU-ordered flush over the host import"); + else Require(labelCounts.flushes == 0, "label ordered flush: a label store skipped its writer wait with the ordered flush off or no host import"); + GuestAllocations::Mutation().Add(Output.data(), sizeof(Output), true, true, true); + struct UnregisterOutput { + ~UnregisterOutput() { GuestAllocations::Mutation().Remove(Output.data()); } + } unregisterOutput; + for (int run = 0; run < 2; ++run) { + Surface surface; + RefreshAfterStore(surface); + } + const auto refreshCounts = StorageTexture::TakeOrderedSkipCounts(); + if (mode == Mode::Enabled) Require(refreshCounts.refreshes != 0, "label ordered flush: no storage refresh took the GPU-ordered flush over the pending image"); + else Require(refreshCounts.refreshes == 0, "label ordered flush: a storage refresh skipped its writer wait with the ordered refresh off or no host import"); AgcDriverShutdown_nid_postfix(); std::puts("label ordered flush tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp new file mode 100644 index 000000000000..70d055ba2b20 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp @@ -0,0 +1,209 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t RowThreads = 32; +constexpr std::uint32_t TiledSide = 128; +constexpr std::uint32_t LinearWidth = 256; +constexpr std::uint32_t LinearHeight = 64; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t TileLinear = 0; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Initial = 0x5a5a5a5au; +constexpr std::uint32_t Stored = 0x12345678u; + +alignas(256) constexpr std::array StoreRowsCode{ + 0x7e020210, 0x7e04020c, 0xf0201108, 0x00010200, 0xbf810000, +}; + +alignas(256) constexpr std::array LoadRowCode{ + 0x7e020280, 0xf0001108, 0x00010300, 0x34080082, 0xbf8c3f70, 0xe0701000, 0x80000304, 0x7e0202bf, 0xf0201108, 0x00010300, 0xbf810000, +}; + +constexpr std::uint32_t StoredBackRow = 63; + +alignas(256) std::array Output{}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "ordered refresh failure: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes); + GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes); +#else + munmap(block, BlockBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +std::array TextureDescriptor(const std::uint8_t* texels, std::uint32_t width, std::uint32_t height, std::uint32_t tile) { + const auto address = AddressOf(texels); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((width - 1u) & 3u) << 30u), + ((width - 1u) >> 2u) | ((height - 1u) << 14u), + 0xfacu | (tile << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = AddressOf(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +void Dispatch(AgcDriver::VulkanDevice& device, std::span code, const std::vector& userData, std::uint32_t threads, std::uint32_t rows) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{threads, 1, 1}, 0u, {false, rows > 1, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, rows, 1, {}, reinterpret_cast(code.data())); +} + +void StoreAll(AgcDriver::VulkanDevice& device, const std::uint8_t* texels) { + std::vector userData(16, 0u); + const auto texture = TextureDescriptor(texels, TiledSide, TiledSide, TileR64KBX); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + userData[12] = Stored; + Dispatch(device, StoreRowsCode, userData, TiledSide, TiledSide); +} + +void LoadRow(AgcDriver::VulkanDevice& device, const std::uint8_t* texels) { + Output.fill(0xdeadbeefu); + std::vector userData(16, 0u); + const auto buffer = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + const auto texture = TextureDescriptor(texels, LinearWidth, LinearHeight, TileLinear); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + Dispatch(device, LoadRowCode, userData, RowThreads, 1); +} + +} + +int main() { + try { + GuestBlock block; + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestAllocations::Mutation().Add(Output.data(), sizeof(Output), true, true, true); + auto* texels = block.Data(); + const auto surface = AddressOf(texels); + std::fill_n(reinterpret_cast(texels), BlockBytes / 4u, Initial); + AgcDriver::GuestMemory::CollectWrites(surface, BlockBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + LoadRow(*device, texels); + StorageTexture::FlushPending(surface, BlockBytes, nullptr, "test"); + device->WaitIdle(); + Require(std::all_of(Output.begin(), Output.end(), [](std::uint32_t value) { return value == Initial; }), "ordered refresh failure: the linear view did not read the initial guest bytes"); + StoreAll(*device, texels); + Require(StorageTexture::FindPending(surface, BlockBytes) != nullptr, "ordered refresh failure: the tiled store left no pending image"); + static_cast(StorageTexture::TakeOrderedSkipCounts()); + static_cast(StorageTexture::TakeFlushFailureForTest()); + StorageTexture::SetFlushFailureForTest(true); + try { + LoadRow(*device, texels); + } catch (const std::exception& error) { + Require(std::strstr(error.what(), "injected failure") != nullptr, std::string("ordered refresh failure: the load failed for another reason: ") + error.what()); + } + StorageTexture::SetFlushFailureForTest(false); + const auto fired = StorageTexture::TakeFlushFailureForTest(); + Require(fired != 0, "ordered refresh failure: the load over the pending image never reached the injected flush failure"); + Require(fired == 2, "ordered refresh failure: the injected flush failure fired outside an armed ordered scope"); + Require(!StorageTexture::OrderedFlushArmedForTest(), "ordered refresh failure: the failed refresh left the ordered flush armed on its thread"); + Require(StorageTexture::FindPending(surface, BlockBytes) != nullptr, "ordered refresh failure: the failed flush dropped the pending image"); + static_cast(StorageTexture::TakeOrderedSkipCounts()); + StorageTexture::FlushPending(surface, BlockBytes, nullptr, "test"); + Require(StorageTexture::TakeOrderedSkipCounts().flushes == 0, "ordered refresh failure: a flush after the failed refresh skipped its writer wait"); + device->WaitIdle(); + } + const auto* words = reinterpret_cast(texels); + for (std::size_t index = 0; index < BlockBytes / 4u; ++index) { + if (index / LinearWidth == StoredBackRow && index % LinearWidth < RowThreads) continue; + Require(words[index] == Stored, "ordered refresh failure: guest memory does not hold the tiled store after the flush at dword " + std::to_string(index)); + } + GuestAllocations::Mutation().Remove(Output.data()); + AgcDriver::Graphics::ClearCachedTextures(device->Device()); + std::puts("ordered refresh failure tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +}