perf(agc): order a storage flush behind its GPU writer when the write-back is a GPU copy

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -2251,6 +2251,13 @@ bool StorageTexture::RefreshLayers(std::uint32_t first, std::uint32_t count) {
     return refresh(firstLayer, lastLayer);
 }
 
+namespace {
+
+bool& GpuOrderedFlush();
+bool& GpuOrderedSkipped();
+
+}
+
 bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) {
     const bool profile = LookupOutcomes::Profiled();
     auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
@@ -2315,7 +2322,20 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer)
     handedOver.clear();
     handedUnits.clear();
     const auto handOverKeys = alias == nullptr && !scoped ? clearHandOver() : std::nullopt;
-    const bool flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None);
+    static const bool refreshOrdered = std::getenv("APS5_NO_REFRESH_GPU_ORDERED") == nullptr;
+    const bool orderedEligible = refreshOrdered && !scoped && alias == nullptr && !handOverKeys.has_value() && descriptor.dccAddress == 0 && !layerPending.Any() && HostImportFor(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes)) != nullptr;
+    bool orderedSkipped = false;
+    bool flushed = false;
+    {
+        const bool previousOrdered = GpuOrderedFlush();
+        const bool previousSkipped = GpuOrderedSkipped();
+        GpuOrderedFlush() = previousOrdered || orderedEligible;
+        GpuOrderedSkipped() = false;
+        flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None);
+        orderedSkipped = orderedEligible && GpuOrderedSkipped();
+        GpuOrderedFlush() = previousOrdered;
+        GpuOrderedSkipped() = previousSkipped || GpuOrderedSkipped();
+    }
     pruneAdoptedSources();
     if (profile && flushed) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start);
     // `original` holds the guest bytes the image was last uploaded from or written back as; while the
@@ -2439,7 +2459,7 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer)
         for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) {
             if (changed[layer] && layerPending[layer]) pendingChanged = true;
         }
-        if (!unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) && equalsOriginal()) {
+        if (!orderedSkipped && !unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) && equalsOriginal()) {
             unchanged = true;
             stamped = false;
             if (!scoped) outside.clear();
@@ -2468,6 +2488,12 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer)
             }
         }
     }
+    if (orderedSkipped) {
+        changed.assign(trackedLayers, true);
+        unchanged = false;
+        anyBorrowed = false;
+        borrowed.clear();
+    }
     const bool outsideChanged = std::any_of(outside.begin(), outside.end(), [](bool value) { return value; });
     if (scoped) noteLayersBehind(firstLayer, lastLayer, outside);
     else layerBehind.clear();
@@ -4545,6 +4571,20 @@ void waitForIdlePublish(const char* reason) {
 
 }
 
+namespace {
+
+bool& GpuOrderedFlush() {
+    thread_local bool flag = false;
+    return flag;
+}
+
+bool& GpuOrderedSkipped() {
+    thread_local bool flag = false;
+    return flag;
+}
+
+}
+
 bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited, VkDevice device) {
     static_cast<void>(FlushDepthPlanes(address, bytes));
     if (published != nullptr) *published = false;
@@ -4653,7 +4693,10 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons
     }
     std::lock_guard gpu(GuestMemory::GpuMutex());
     if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr<StorageTexture>& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast<std::size_t>(texture->guestBytes)) == nullptr; }))) ownNotes.clear();
-    if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion()) {
+    auto* const orderedRecorder = Recorder::Active();
+    const bool orderedSkip = GpuOrderedFlush() && orderedRecorder != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [orderedRecorder](const std::shared_ptr<StorageTexture>& texture) { return texture->context.device == orderedRecorder->Device() && HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast<std::size_t>(texture->guestBytes)) != nullptr; });
+    if (orderedSkip) GpuOrderedSkipped() = true;
+    if (auto* recorder = Recorder::Active(); !orderedSkip && recorder != nullptr && !Recorder::InCompletion()) {
         try {
             static const bool wholeRange = std::getenv("APS5_FLUSH_WAIT_WHOLE_RANGE") != nullptr;
             const bool publishes = !wholeRange && scope != PublishScope::None;
@@ -5085,6 +5128,12 @@ std::size_t StorageTexture::StoreSmallPending(const char* reason) {
 void StorageTexture::StoreReadbackBeforeLabel(const char* reason, VkDevice device) {
     static const bool disabled = std::getenv("APS5_NO_LABEL_READBACK_STORE") != nullptr;
     if (disabled) return;
+    static const bool gpuOrdered = std::getenv("APS5_NO_FLUSH_GPU_ORDERED") == nullptr;
+    struct OrderedScope {
+        bool previous;
+        explicit OrderedScope(bool on) : previous(GpuOrderedFlush()) { GpuOrderedFlush() = previous || on; }
+        ~OrderedScope() { GpuOrderedFlush() = previous; }
+    } ordered{gpuOrdered};
     std::vector<std::pair<std::uint64_t, std::size_t>> ranges;
     std::vector<std::pair<std::uint64_t, std::size_t>> occasional;
     {

core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp

@@ -0,0 +1,191 @@
+#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
+#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
+#include "prx/libc/include/GuestAllocations.hpp"
+#include "prx/libc/include/GuestArena.hpp"
+#include "prx/libc/include/GuestWriteWatch.hpp"
+#include "VulkanTestDevice.hpp"
+#ifdef _WIN32
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <span>
+#include <stdexcept>
+#include <thread>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using AgcDriver::Graphics::StorageTexture;
+constexpr std::uint32_t Side = 128;
+constexpr std::uint32_t Format32UInt = 20;
+constexpr std::uint32_t TileR64KBX = 0x1b;
+constexpr std::uint32_t Type2D = 9;
+constexpr std::size_t Block = 0x10000;
+constexpr std::size_t SurfaceBytes = Block;
+constexpr std::uint32_t Untouched = 0xdeadbeefu;
+constexpr std::uint32_t Stored = 0x12345678u;
+
+alignas(256) constexpr std::array<std::uint32_t, 7> StoreCode{
+    0x7e040300, 0x7e060280, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000,
+};
+
+std::uint64_t AddressOf(const void* data) {
+    return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(data));
+}
+
+std::array<std::uint32_t, 8> TextureDescriptor(const std::uint32_t* texels) {
+    const auto address = AddressOf(texels);
+    return {
+        static_cast<std::uint32_t>(address >> 8u),
+        static_cast<std::uint32_t>((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u),
+        ((Side - 1u) >> 2u) | ((Side - 1u) << 14u),
+        0xfacu | (TileR64KBX << 20u) | (Type2D << 28u),
+        0u,
+        0u,
+        0u,
+        0u,
+    };
+}
+
+void Dispatch(std::vector<std::uint32_t>& words, std::span<const std::uint32_t> code, std::uint32_t threads, std::uint32_t resource, std::span<const std::uint32_t> userData, std::uint32_t groups) {
+    const auto registers = [&](const std::uint32_t first, const std::span<const std::uint32_t> values) {
+        words.push_back(0xc0007600u | (static_cast<std::uint32_t>(values.size()) << 16u));
+        words.push_back(first);
+        words.insert(words.end(), values.begin(), values.end());
+    };
+    const std::array<std::uint32_t, 3> shape{threads, 1, 1};
+    registers(0x207, shape);
+    const auto address = AddressOf(code.data());
+    const std::array<std::uint32_t, 2> program{static_cast<std::uint32_t>(address >> 8u), static_cast<std::uint32_t>(address >> 40u)};
+    registers(0x20c, program);
+    const std::array<std::uint32_t, 1> layout{resource};
+    registers(0x213, layout);
+    registers(0x240, userData);
+    words.insert(words.end(), {0xc0031500u, groups, 1, 1, 0x41u});
+}
+
+void Store(std::vector<std::uint32_t>& words, const std::uint32_t* texels) {
+    std::array<std::uint32_t, 16> userData{};
+    const auto texture = TextureDescriptor(texels);
+    std::copy(texture.begin(), texture.end(), userData.begin() + 8);
+    Dispatch(words, StoreCode, 8, static_cast<std::uint32_t>(userData.size()) << 1u, userData, 1);
+}
+
+class Surface {
+public:
+    Surface() {
+#ifdef _WIN32
+        region = static_cast<std::uint8_t*>(GuestArena::GuestArenaAllocate_nid_postfix(SurfaceBytes, Block));
+        if (region != nullptr) GuestArena::GuestArenaCommit_nid_postfix(region, SurfaceBytes, PAGE_READWRITE, SurfaceBytes);
+#else
+        void* mapped = mmap(nullptr, SurfaceBytes + Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (mapped != MAP_FAILED) {
+            const auto begin = reinterpret_cast<std::uintptr_t>(mapped);
+            const auto aligned = (begin + Block - 1) & ~std::uintptr_t{Block - 1};
+            if (aligned != begin) munmap(mapped, aligned - begin);
+            if (aligned != begin + Block) munmap(reinterpret_cast<void*>(aligned + SurfaceBytes), begin + Block - aligned);
+            region = reinterpret_cast<std::uint8_t*>(aligned);
+            GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(region, SurfaceBytes);
+        }
+#endif
+        Require(region != nullptr, "label ordered flush: cannot allocate the guest surface");
+        texels = reinterpret_cast<std::uint32_t*>(region);
+        std::fill(texels, texels + SurfaceBytes / 4u, Untouched);
+        GuestAllocations::Mutation().Add(region, SurfaceBytes, true, true, true);
+    }
+
+    ~Surface() {
+        GuestAllocations::Mutation().Remove(region);
+#ifdef _WIN32
+        GuestArena::GuestArenaReset_nid_postfix(region, SurfaceBytes);
+        GuestArena::GuestArenaRelease_nid_postfix(region, SurfaceBytes);
+#else
+        munmap(region, SurfaceBytes);
+        GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(region, SurfaceBytes);
+#endif
+    }
+
+    Surface(const Surface&) = delete;
+    Surface& operator=(const Surface&) = delete;
+
+    std::uint32_t* Texels() const { return texels; }
+
+private:
+    std::uint8_t* region = nullptr;
+    std::uint32_t* texels = nullptr;
+};
+
+void LabelAfterStore(const Surface& surface, std::uint32_t* label) {
+    std::vector<std::uint32_t> words;
+    Store(words, surface.Texels());
+    const auto labelAddress = AddressOf(label);
+    words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), static_cast<std::uint32_t>(labelAddress), static_cast<std::uint32_t>(labelAddress >> 32u), 1u, 0, 0});
+    Packet packet{words.data(), static_cast<std::uint32_t>(words.size()), 0, {}};
+    AgcDriver::Submit(&packet, 0);
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
+    while (std::atomic_ref<std::uint32_t>(*label).load() == 0) {
+        Require(std::chrono::steady_clock::now() < deadline, "label ordered flush: the label never landed");
+        std::this_thread::yield();
+    }
+    std::vector<std::uint32_t> landed(SurfaceBytes / sizeof(std::uint32_t));
+    std::memcpy(landed.data(), surface.Texels(), SurfaceBytes);
+    const auto stored = std::count(landed.begin(), landed.end(), Stored);
+    Require(stored >= 8, "label ordered flush: the label was visible before the image store it follows");
+    AgcDriverWaitIdle_nid_postfix();
+}
+
+}
+
+int main() {
+    try {
+        {
+            const auto device = OpenVulkanTestDevice();
+            if (!device) return VulkanTestSkipped;
+        }
+        struct ShutdownGuard {
+            ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} }
+        } shutdown;
+        {
+            Surface probe;
+            const auto described = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(TextureDescriptor(probe.Texels()))).guestBytes;
+            Require(described == SurfaceBytes && described <= StorageTexture::SmallStoreBytes, "label ordered flush: the surface is not a small storage image");
+        }
+        alignas(256) static std::uint32_t words[64]{};
+        GuestAllocations::Mutation().Add(words, sizeof(words), true, true, true);
+        struct Unregister {
+            ~Unregister() { GuestAllocations::Mutation().Remove(words); }
+        } unregister;
+        constexpr int Runs = 8;
+        for (int run = 0; run < Runs; ++run) {
+            Surface surface;
+            auto* label = &words[0];
+            *label = 0;
+            LabelAfterStore(surface, label);
+        }
+        AgcDriverShutdown_nid_postfix();
+        std::puts("label ordered flush tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

docs/dev/ENVIRONMENT.md

@@ -278,6 +278,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_NO_FILL_CLEAR` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` |  | disables the clear-through-DCC-keys path in a fill |
 | `APS5_NO_FILL_ELISION` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | records every buffer fill, also a repeat of the last fill of the same range with the same pattern over bytes unchanged since |
 | `APS5_NO_FILL_HLE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` |  | disables the fill-kernel HLE, so matching fills run as compute |
+| `APS5_NO_FLUSH_GPU_ORDERED` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` |  | waits on the CPU for a flushed storage image's GPU writer before a label's readback store, also when every flushed image's write-back is a GPU copy into its host import |
 | `APS5_NO_FLUSH_PRETEST` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` |  | lists every pending image on FlushPending instead of testing whether a pending unit lies inside the range |
 | `APS5_NO_FOREIGN_WRITERS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` |  | records no draw-written ranges as foreign writers for capture validation |
 | `APS5_NO_GATE_RETRY` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` |  | retries no validation whose entry moved, leaving the dispatch or draw unvalidated |
@@ -347,6 +348,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_NO_RECORDER_SYNC_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | gives the draws the recorder waits for their own batch instead of recording them with the wait |
 | `APS5_NO_RECORD_COPIED_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | waits for copied-write draws at once instead of listing their write-back for later |
 | `APS5_NO_RECT_INDIRECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp` |  | sends an indirect rectangle-list draw down the CPU path instead of the GPU-side path |
+| `APS5_NO_REFRESH_GPU_ORDERED` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` |  | waits on the CPU for the pending writers a storage refresh flushes and compares the guest bytes, instead of re-uploading the layers from the import behind the write-backs |
 | `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | drops the residency of linear color targets, so a target whose address has a host import, or whose import a draw would make, is copied in and out again |
 | `APS5_NO_RESIDENT_PRESENT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` |  | always presents through guest memory instead of the resident image |
 | `APS5_NO_RESIDENT_TARGETS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | copies every render target in and out again instead of keeping it resident |