diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 10737429984d..c73c428dc325 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -391,6 +391,14 @@ class StorageTexture : public std::enable_shared_from_this { std::uint64_t flushedAfterSkipSameSite; }; static HookSkipCounts TakeHookSkipCounts(); + struct OrderedSkipCounts { + std::uint64_t flushes; + std::uint64_t refreshes; + }; + static OrderedSkipCounts TakeOrderedSkipCounts(); + static void SetFlushFailureForTest(bool fail); + static int TakeFlushFailureForTest(); + static bool OrderedFlushArmedForTest(); // A consumer found the image current and used it (Refresh, the fast revalidation). void NoteProved() const; // Whether the storage cache holds this image (set and cleared by the cache under its mutex, the diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 9b80f2bf042d..6870c4ed21e7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -2251,6 +2251,17 @@ bool StorageTexture::RefreshLayers(std::uint32_t first, std::uint32_t count) { return refresh(firstLayer, lastLayer); } +namespace { + +bool& GpuOrderedFlush(); +bool& GpuOrderedSkipped(); +std::atomic orderedFlushSkips{0}; +std::atomic orderedRefreshSkips{0}; +std::atomic flushFailureForTest{false}; +std::atomic flushFailureFiredForTest{0}; + +} + bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) { const bool profile = LookupOutcomes::Profiled(); auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -2315,7 +2326,28 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) handedOver.clear(); handedUnits.clear(); const auto handOverKeys = alias == nullptr && !scoped ? clearHandOver() : std::nullopt; - const bool flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None); + static const bool refreshOrdered = std::getenv("APS5_NO_REFRESH_GPU_ORDERED") == nullptr; + const bool orderedEligible = refreshOrdered && !scoped && alias == nullptr && !handOverKeys.has_value() && descriptor.dccAddress == 0 && !layerPending.Any() && HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) != nullptr; + bool orderedSkipped = false; + bool flushed = false; + { + struct OrderedRefreshScope { + bool previousOrdered; + bool previousSkipped; + explicit OrderedRefreshScope(bool eligible) : previousOrdered(GpuOrderedFlush()), previousSkipped(GpuOrderedSkipped()) { + GpuOrderedFlush() = eligible; + GpuOrderedSkipped() = false; + } + ~OrderedRefreshScope() { + const bool skipped = GpuOrderedSkipped(); + GpuOrderedFlush() = previousOrdered; + GpuOrderedSkipped() = previousSkipped || skipped; + } + } ordered{orderedEligible}; + flushed = !handOverKeys.has_value() && FlushPending(scopeBegin, scopeBytes, alias.get(), "storage refresh", PublishScope::None); + orderedSkipped = orderedEligible && GpuOrderedSkipped(); + } + if (orderedSkipped) orderedRefreshSkips.fetch_add(1, std::memory_order_relaxed); pruneAdoptedSources(); if (profile && flushed) start = LookupOutcomes::Add(LookupOutcomes::PendingFlush, start); // `original` holds the guest bytes the image was last uploaded from or written back as; while the @@ -2439,7 +2471,7 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (changed[layer] && layerPending[layer]) pendingChanged = true; } - if (!unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast(guestBytes))) && equalsOriginal()) { + if (!orderedSkipped && !unchanged && !keysChanged && !pendingChanged && (scoped ? originalHolds(scopeBegin, scopeBytes) : originalHolds(descriptor.baseAddress, static_cast(guestBytes))) && equalsOriginal()) { unchanged = true; stamped = false; if (!scoped) outside.clear(); @@ -2468,6 +2500,12 @@ bool StorageTexture::refresh(std::uint32_t firstLayer, std::uint32_t lastLayer) } } } + if (orderedSkipped) { + changed.assign(trackedLayers, true); + unchanged = false; + anyBorrowed = false; + borrowed.clear(); + } const bool outsideChanged = std::any_of(outside.begin(), outside.end(), [](bool value) { return value; }); if (scoped) noteLayersBehind(firstLayer, lastLayer, outside); else layerBehind.clear(); @@ -4545,6 +4583,20 @@ void waitForIdlePublish(const char* reason) { } +namespace { + +bool& GpuOrderedFlush() { + thread_local bool flag = false; + return flag; +} + +bool& GpuOrderedSkipped() { + thread_local bool flag = false; + return flag; +} + +} + bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except, const char* reason, PublishScope scope, bool* published, bool waited, VkDevice device) { static_cast(FlushDepthPlanes(address, bytes)); if (published != nullptr) *published = false; @@ -4653,7 +4705,28 @@ bool StorageTexture::FlushPending(std::uint64_t address, std::size_t bytes, cons } std::lock_guard gpu(GuestMemory::GpuMutex()); if (!ownNotes.empty() && (flush.size() != 1 || std::any_of(flush.begin(), flush.end(), [](const std::shared_ptr& texture) { return ExistingHostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) == nullptr; }))) ownNotes.clear(); - if (auto* recorder = Recorder::Active(); recorder != nullptr && !Recorder::InCompletion()) { + auto* const orderedRecorder = Recorder::Active(); + const auto publishesNothing = [&](const Recorder& active) { + if (scope == PublishScope::None) return true; + if (!LiveShadowedRanges(address, bytes, scope).empty()) return false; + constexpr std::uint64_t block = 65536; + constexpr auto top = std::numeric_limits::max(); + const auto first = address / block * block; + const auto end = bytes > top - address ? top : address + bytes; + const auto last = end > top - block ? end : (end + block - 1) / block * block; + return active.PendingLeaseSerial(first, static_cast(last - first)) == 0; + }; + const bool orderedSkip = GpuOrderedFlush() && orderedRecorder != nullptr && !Recorder::InCompletion() && !flush.empty() && std::all_of(flush.begin(), flush.end(), [orderedRecorder](const std::shared_ptr& texture) { return texture->context.device == orderedRecorder->Device() && HostImportFor(texture->context, texture->descriptor.baseAddress, static_cast(texture->guestBytes)) != nullptr; }) && publishesNothing(*orderedRecorder); + if (orderedSkip) { + GpuOrderedSkipped() = true; + orderedFlushSkips.fetch_add(1, std::memory_order_relaxed); + } + if (flushFailureForTest.exchange(false, std::memory_order_acq_rel)) { + for (const auto& texture : flush) texture->reconcilePending(); + flushFailureFiredForTest.store(GpuOrderedFlush() ? 2 : 1, std::memory_order_release); + throw std::runtime_error("StorageTexture::FlushPending: injected failure for a test"); + } + if (auto* recorder = Recorder::Active(); !orderedSkip && recorder != nullptr && !Recorder::InCompletion()) { try { static const bool wholeRange = std::getenv("APS5_FLUSH_WAIT_WHOLE_RANGE") != nullptr; const bool publishes = !wholeRange && scope != PublishScope::None; @@ -5085,6 +5158,12 @@ std::size_t StorageTexture::StoreSmallPending(const char* reason) { void StorageTexture::StoreReadbackBeforeLabel(const char* reason, VkDevice device) { static const bool disabled = std::getenv("APS5_NO_LABEL_READBACK_STORE") != nullptr; if (disabled) return; + static const bool gpuOrdered = std::getenv("APS5_NO_FLUSH_GPU_ORDERED") == nullptr; + struct OrderedScope { + bool previous; + explicit OrderedScope(bool on) : previous(GpuOrderedFlush()) { GpuOrderedFlush() = previous || on; } + ~OrderedScope() { GpuOrderedFlush() = previous; } + } ordered{gpuOrdered}; std::vector> ranges; std::vector> occasional; { @@ -5526,6 +5605,22 @@ StorageTexture::HookSkipCounts StorageTexture::TakeHookSkipCounts() { return {flushedAfterSkip.exchange(0, std::memory_order_relaxed), flushedAfterSkipSameSite.exchange(0, std::memory_order_relaxed)}; } +StorageTexture::OrderedSkipCounts StorageTexture::TakeOrderedSkipCounts() { + return {orderedFlushSkips.exchange(0, std::memory_order_relaxed), orderedRefreshSkips.exchange(0, std::memory_order_relaxed)}; +} + +void StorageTexture::SetFlushFailureForTest(bool fail) { + flushFailureForTest.store(fail, std::memory_order_release); +} + +int StorageTexture::TakeFlushFailureForTest() { + return flushFailureFiredForTest.exchange(0, std::memory_order_acq_rel); +} + +bool StorageTexture::OrderedFlushArmedForTest() { + return GpuOrderedFlush(); +} + void StorageTexture::BumpPendingSerial() { pendingSerial.fetch_add(1, std::memory_order_release); } diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake new file mode 100644 index 000000000000..f74a4406cf28 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_label_ordered_flush_tests.cmake @@ -0,0 +1,10 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_label_ordered_flush_disabled COMMAND agc_driver_label_ordered_flush_tests --disabled) + set_tests_properties(agc_driver_label_ordered_flush_disabled PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_FLUSH_GPU_ORDERED=1;APS5_NO_REFRESH_GPU_ORDERED=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_label_ordered_flush_unimported COMMAND agc_driver_label_ordered_flush_tests --unimported) + set_tests_properties(agc_driver_label_ordered_flush_unimported PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_HOST_IMPORT=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_label_ordered_flush_disabled agc_driver_label_ordered_flush_unimported PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp new file mode 100644 index 000000000000..5640078d8be8 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LabelOrderedFlush.cpp @@ -0,0 +1,267 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +constexpr std::uint32_t Side = 128; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t Block = 0x10000; +constexpr std::size_t SurfaceBytes = Block; +constexpr std::uint32_t Untouched = 0xdeadbeefu; +constexpr std::uint32_t Stored = 0x12345678u; + +alignas(256) constexpr std::array StoreCode{ + 0x7e040300, 0x7e060280, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000, +}; + +alignas(256) constexpr std::array StoreRowsCode{ + 0x7e040300, 0x7e060210, 0x7e0202ff, Stored, 0xf0200108, 0x00020102, 0xbf810000, +}; + +alignas(256) constexpr std::array LoadRowCode{ + 0x7e020280, 0xf0001108, 0x00010300, 0x34080082, 0xbf8c3f70, 0xe0701000, 0x80000304, 0xbf810000, +}; + +constexpr std::uint32_t RowThreads = 32; +constexpr std::uint32_t LinearWidth = 256; +constexpr std::uint32_t LinearHeight = 64; +constexpr std::uint32_t TileLinear = 0; +constexpr std::uint32_t TgidYEnable = 1u << 8u; + +enum class Mode { Enabled, Disabled, Unimported }; + +alignas(256) std::array Output{}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +std::array TextureDescriptor(const std::uint32_t* texels, std::uint32_t width = Side, std::uint32_t height = Side, std::uint32_t tile = TileR64KBX) { + const auto address = AddressOf(texels); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((width - 1u) & 3u) << 30u), + ((width - 1u) >> 2u) | ((height - 1u) << 14u), + 0xfacu | (tile << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = AddressOf(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +void Dispatch(std::vector& words, std::span code, std::uint32_t threads, std::uint32_t resource, std::span userData, std::uint32_t groups, std::uint32_t rows = 1) { + const auto registers = [&](const std::uint32_t first, const std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{threads, 1, 1}; + registers(0x207, shape); + const auto address = AddressOf(code.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array layout{resource}; + registers(0x213, layout); + registers(0x240, userData); + words.insert(words.end(), {0xc0031500u, groups, rows, 1, 0x41u}); +} + +void Store(std::vector& words, const std::uint32_t* texels) { + std::array userData{}; + const auto texture = TextureDescriptor(texels); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + Dispatch(words, StoreCode, 8, static_cast(userData.size()) << 1u, userData, 1); +} + +class Surface { +public: + Surface() { +#ifdef _WIN32 + region = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(SurfaceBytes, Block)); + if (region != nullptr) GuestArena::GuestArenaCommit_nid_postfix(region, SurfaceBytes, PAGE_READWRITE, SurfaceBytes); +#else + void* mapped = mmap(nullptr, SurfaceBytes + Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + Block - 1) & ~std::uintptr_t{Block - 1}; + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned != begin + Block) munmap(reinterpret_cast(aligned + SurfaceBytes), begin + Block - aligned); + region = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(region, SurfaceBytes); + } +#endif + Require(region != nullptr, "label ordered flush: cannot allocate the guest surface"); + texels = reinterpret_cast(region); + std::fill(texels, texels + SurfaceBytes / 4u, Untouched); + GuestAllocations::Mutation().Add(region, SurfaceBytes, true, true, true); + } + + ~Surface() { + GuestAllocations::Mutation().Remove(region); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(region, SurfaceBytes); + GuestArena::GuestArenaRelease_nid_postfix(region, SurfaceBytes); +#else + munmap(region, SurfaceBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(region, SurfaceBytes); +#endif + } + + Surface(const Surface&) = delete; + Surface& operator=(const Surface&) = delete; + + std::uint32_t* Texels() const { return texels; } + +private: + std::uint8_t* region = nullptr; + std::uint32_t* texels = nullptr; +}; + +void LabelAfterStore(const Surface& surface, std::uint32_t* label) { + std::vector words; + Store(words, surface.Texels()); + const auto labelAddress = AddressOf(label); + words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), static_cast(labelAddress), static_cast(labelAddress >> 32u), 1u, 0, 0}); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (std::atomic_ref(*label).load() == 0) { + Require(std::chrono::steady_clock::now() < deadline, "label ordered flush: the label never landed"); + std::this_thread::yield(); + } + std::vector landed(SurfaceBytes / sizeof(std::uint32_t)); + std::memcpy(landed.data(), surface.Texels(), SurfaceBytes); + const auto stored = std::count(landed.begin(), landed.end(), Stored); + Require(stored >= 8, "label ordered flush: the label was visible before the image store it follows"); + AgcDriverWaitIdle_nid_postfix(); +} + +void SubmitWords(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); +} + +void LoadRow(const Surface& surface) { + Output.fill(0xdeadbeefu); + std::array userData{}; + const auto buffer = BufferDescriptor(Output.data(), static_cast(Output.size() * sizeof(std::uint32_t))); + const auto texture = TextureDescriptor(surface.Texels(), LinearWidth, LinearHeight, TileLinear); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + std::vector words; + Dispatch(words, LoadRowCode, RowThreads, static_cast(userData.size()) << 1u, userData, 1); + SubmitWords(words); +} + +void RefreshAfterStore(const Surface& surface) { + LoadRow(surface); + Require(std::all_of(Output.begin(), Output.end(), [](std::uint32_t value) { return value == Untouched; }), "label ordered flush: the linear view did not read the untouched guest bytes"); + std::array userData{}; + const auto texture = TextureDescriptor(surface.Texels()); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + std::vector words; + Dispatch(words, StoreRowsCode, Side, (static_cast(userData.size()) << 1u) | TgidYEnable, userData, 1, Side); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + LoadRow(surface); + for (std::uint32_t index = 0; index < RowThreads; ++index) { + Require(Output[index] == Stored, "label ordered flush: the refreshed linear view reads " + std::to_string(Output[index]) + " at texel " + std::to_string(index) + " instead of the tiled store"); + Require(Output[index] == surface.Texels()[index], "label ordered flush: the refreshed linear view disagrees with guest memory at texel " + std::to_string(index)); + } +} + +} + +int main(int argc, char** argv) { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + struct ShutdownGuard { + ~ShutdownGuard() { try { AgcDriverShutdown_nid_postfix(); } catch (...) {} } + } shutdown; + { + Surface probe; + const auto described = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(TextureDescriptor(probe.Texels()))).guestBytes; + Require(described == SurfaceBytes && described <= StorageTexture::SmallStoreBytes, "label ordered flush: the surface is not a small storage image"); + } + alignas(256) static std::uint32_t words[64]{}; + GuestAllocations::Mutation().Add(words, sizeof(words), true, true, true); + struct Unregister { + ~Unregister() { GuestAllocations::Mutation().Remove(words); } + } unregister; + const std::string option = argc > 1 ? argv[1] : ""; + const Mode mode = option == "--disabled" ? Mode::Disabled : option == "--unimported" ? Mode::Unimported : Mode::Enabled; + static_cast(StorageTexture::TakeOrderedSkipCounts()); + constexpr int Runs = 8; + for (int run = 0; run < Runs; ++run) { + Surface surface; + auto* label = &words[0]; + *label = 0; + LabelAfterStore(surface, label); + } + const auto labelCounts = StorageTexture::TakeOrderedSkipCounts(); + if (mode == Mode::Enabled) Require(labelCounts.flushes != 0, "label ordered flush: no label store took the GPU-ordered flush over the host import"); + else Require(labelCounts.flushes == 0, "label ordered flush: a label store skipped its writer wait with the ordered flush off or no host import"); + GuestAllocations::Mutation().Add(Output.data(), sizeof(Output), true, true, true); + struct UnregisterOutput { + ~UnregisterOutput() { GuestAllocations::Mutation().Remove(Output.data()); } + } unregisterOutput; + for (int run = 0; run < 2; ++run) { + Surface surface; + RefreshAfterStore(surface); + } + const auto refreshCounts = StorageTexture::TakeOrderedSkipCounts(); + if (mode == Mode::Enabled) Require(refreshCounts.refreshes != 0, "label ordered flush: no storage refresh took the GPU-ordered flush over the pending image"); + else Require(refreshCounts.refreshes == 0, "label ordered flush: a storage refresh skipped its writer wait with the ordered refresh off or no host import"); + AgcDriverShutdown_nid_postfix(); + std::puts("label ordered flush tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp new file mode 100644 index 000000000000..70d055ba2b20 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/OrderedRefreshFailure.cpp @@ -0,0 +1,209 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t RowThreads = 32; +constexpr std::uint32_t TiledSide = 128; +constexpr std::uint32_t LinearWidth = 256; +constexpr std::uint32_t LinearHeight = 64; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t TileLinear = 0; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Initial = 0x5a5a5a5au; +constexpr std::uint32_t Stored = 0x12345678u; + +alignas(256) constexpr std::array StoreRowsCode{ + 0x7e020210, 0x7e04020c, 0xf0201108, 0x00010200, 0xbf810000, +}; + +alignas(256) constexpr std::array LoadRowCode{ + 0x7e020280, 0xf0001108, 0x00010300, 0x34080082, 0xbf8c3f70, 0xe0701000, 0x80000304, 0x7e0202bf, 0xf0201108, 0x00010300, 0xbf810000, +}; + +constexpr std::uint32_t StoredBackRow = 63; + +alignas(256) std::array Output{}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "ordered refresh failure: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes); + GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes); +#else + munmap(block, BlockBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +std::array TextureDescriptor(const std::uint8_t* texels, std::uint32_t width, std::uint32_t height, std::uint32_t tile) { + const auto address = AddressOf(texels); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((width - 1u) & 3u) << 30u), + ((width - 1u) >> 2u) | ((height - 1u) << 14u), + 0xfacu | (tile << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = AddressOf(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +void Dispatch(AgcDriver::VulkanDevice& device, std::span code, const std::vector& userData, std::uint32_t threads, std::uint32_t rows) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{threads, 1, 1}, 0u, {false, rows > 1, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, rows, 1, {}, reinterpret_cast(code.data())); +} + +void StoreAll(AgcDriver::VulkanDevice& device, const std::uint8_t* texels) { + std::vector userData(16, 0u); + const auto texture = TextureDescriptor(texels, TiledSide, TiledSide, TileR64KBX); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + userData[12] = Stored; + Dispatch(device, StoreRowsCode, userData, TiledSide, TiledSide); +} + +void LoadRow(AgcDriver::VulkanDevice& device, const std::uint8_t* texels) { + Output.fill(0xdeadbeefu); + std::vector userData(16, 0u); + const auto buffer = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + const auto texture = TextureDescriptor(texels, LinearWidth, LinearHeight, TileLinear); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + Dispatch(device, LoadRowCode, userData, RowThreads, 1); +} + +} + +int main() { + try { + GuestBlock block; + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestAllocations::Mutation().Add(Output.data(), sizeof(Output), true, true, true); + auto* texels = block.Data(); + const auto surface = AddressOf(texels); + std::fill_n(reinterpret_cast(texels), BlockBytes / 4u, Initial); + AgcDriver::GuestMemory::CollectWrites(surface, BlockBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + LoadRow(*device, texels); + StorageTexture::FlushPending(surface, BlockBytes, nullptr, "test"); + device->WaitIdle(); + Require(std::all_of(Output.begin(), Output.end(), [](std::uint32_t value) { return value == Initial; }), "ordered refresh failure: the linear view did not read the initial guest bytes"); + StoreAll(*device, texels); + Require(StorageTexture::FindPending(surface, BlockBytes) != nullptr, "ordered refresh failure: the tiled store left no pending image"); + static_cast(StorageTexture::TakeOrderedSkipCounts()); + static_cast(StorageTexture::TakeFlushFailureForTest()); + StorageTexture::SetFlushFailureForTest(true); + try { + LoadRow(*device, texels); + } catch (const std::exception& error) { + Require(std::strstr(error.what(), "injected failure") != nullptr, std::string("ordered refresh failure: the load failed for another reason: ") + error.what()); + } + StorageTexture::SetFlushFailureForTest(false); + const auto fired = StorageTexture::TakeFlushFailureForTest(); + Require(fired != 0, "ordered refresh failure: the load over the pending image never reached the injected flush failure"); + Require(fired == 2, "ordered refresh failure: the injected flush failure fired outside an armed ordered scope"); + Require(!StorageTexture::OrderedFlushArmedForTest(), "ordered refresh failure: the failed refresh left the ordered flush armed on its thread"); + Require(StorageTexture::FindPending(surface, BlockBytes) != nullptr, "ordered refresh failure: the failed flush dropped the pending image"); + static_cast(StorageTexture::TakeOrderedSkipCounts()); + StorageTexture::FlushPending(surface, BlockBytes, nullptr, "test"); + Require(StorageTexture::TakeOrderedSkipCounts().flushes == 0, "ordered refresh failure: a flush after the failed refresh skipped its writer wait"); + device->WaitIdle(); + } + const auto* words = reinterpret_cast(texels); + for (std::size_t index = 0; index < BlockBytes / 4u; ++index) { + if (index / LinearWidth == StoredBackRow && index % LinearWidth < RowThreads) continue; + Require(words[index] == Stored, "ordered refresh failure: guest memory does not hold the tiled store after the flush at dword " + std::to_string(index)); + } + GuestAllocations::Mutation().Remove(Output.data()); + AgcDriver::Graphics::ClearCachedTextures(device->Device()); + std::puts("ordered refresh failure tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 6215ac4e85dd..be31ceb35312 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -278,6 +278,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_FILL_CLEAR` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` | | disables the clear-through-DCC-keys path in a fill | | `APS5_NO_FILL_ELISION` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | records every buffer fill, also a repeat of the last fill of the same range with the same pattern over bytes unchanged since | | `APS5_NO_FILL_HLE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp` | | disables the fill-kernel HLE, so matching fills run as compute | +| `APS5_NO_FLUSH_GPU_ORDERED` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | waits on the CPU for a flushed storage image's GPU writer before a label's readback store, also when every flushed image's write-back is a GPU copy into its host import | | `APS5_NO_FLUSH_PRETEST` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | lists every pending image on FlushPending instead of testing whether a pending unit lies inside the range | | `APS5_NO_FOREIGN_WRITERS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | records no draw-written ranges as foreign writers for capture validation | | `APS5_NO_GATE_RETRY` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | retries no validation whose entry moved, leaving the dispatch or draw unvalidated | @@ -347,6 +348,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_RECORDER_SYNC_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | gives the draws the recorder waits for their own batch instead of recording them with the wait | | `APS5_NO_RECORD_COPIED_DRAWS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | waits for copied-write draws at once instead of listing their write-back for later | | `APS5_NO_RECT_INDIRECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp` | | sends an indirect rectangle-list draw down the CPU path instead of the GPU-side path | +| `APS5_NO_REFRESH_GPU_ORDERED` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp` | | waits on the CPU for the pending writers a storage refresh flushes and compares the guest bytes, instead of re-uploading the layers from the import behind the write-backs | | `APS5_NO_RESIDENT_LINEAR` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | drops the residency of linear color targets, so a target whose address has a host import, or whose import a draw would make, is copied in and out again | | `APS5_NO_RESIDENT_PRESENT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | always presents through guest memory instead of the resident image | | `APS5_NO_RESIDENT_TARGETS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | copies every render target in and out again instead of keeping it resident |