From 3b34d1fdb6763cd53182c2cd4b37ffe42c1f5b84 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Thu, 15 Oct 2026 04:48:47 +0200 Subject: [PATCH 01/18] perf(agc): pipeline queue-0 draw commits behind the packet worker (APS5_PIPELINED_DRAWS) With APS5_PIPELINED_DRAWS=1, queue 0's worker decodes, captures and recompiles a draw and hands it to a committer thread that records it (Graphics::Draw under the GPU lock), so the two run in parallel instead of one after the other. Work the committer has not reached yet is tracked by the guest ranges it will write; a later packet that reads or writes them, or one the pipeline cannot order, drains it. Ordered as commit items instead of drains: - labels (WRITE_DATA, RELEASE_MEM) and same-queue waits on them; APS5_PIPELINE_DRAIN_LABELS=1 drains; - the end of a queue-0 submission and a released REWIND tail; APS5_PIPELINE_DRAIN_SUBMISSIONS=1 drains; - the title's buffer fill kernels; APS5_PIPELINE_DRAIN_FILLS=1 drains; - DMA_DATA immediates and copies (APS5_PIPELINE_DMA=1). A draw's write ranges include the storage images it writes and its exact color and depth target ranges (APS5_EXACT_DRAW_WRITES=1), not estimates. Draw-path costs the pipeline exposed: - ShaderMemory moves, so a commit owns its capture's pages; - a submission made before the worker's current collect epoch began keeps the epoch, so a worker behind a queue of submissions walks a surface once per catch-up (APS5_SUBMISSION_EPOCH_EACH=1 bumps at every submission); a thread remembers the ranges it walked for the whole epoch; - the pipeline signals only threads that wait; the pending-image and unit-shadow registries take a spinning slim lock instead of std::mutex (winpthreads contention was kernel events); ScanPending skips the registry walk when no pending image can overlap; - fewer heap allocations per recorded draw (reused descriptor sets and scratch vectors, buffer pool size classes kept). GTA V Performance mode, prologue bank interior, RTX 5080: 11.7 -> about 20 fps together with the cheaper captures below it. Off by default here; the follow-up turns it on. Co-Authored-By: Claude Opus 5.5 (1M context) Co-Authored-By: Gjon Gojcevic <7807870+gjonn@users.noreply.github.com> --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 1 + .../include/Driver/Draw/DrawPipeline.hpp | 74 +++++++ .../Execution/include/Driver/Driver.hpp | 16 ++ .../include/Driver/Memory/DrawWriteRanges.hpp | 18 ++ .../src/Driver/Dispatch/Dispatch.cpp | 2 + .../Execution/src/Driver/Draw/Draw.cpp | 44 +++++ .../src/Driver/Draw/DrawPipeline.cpp | 185 ++++++++++++++++++ .../src/Driver/Memory/BufferFill.cpp | 32 ++- .../src/Driver/Memory/WriteEvidence.cpp | 5 + .../src/Driver/Memory/WrittenBuffers.cpp | 82 +++++++- .../src/Driver/Packets/PacketExecution.cpp | 81 +++++++- .../src/Driver/Packets/PacketFlush.cpp | 6 + .../src/Driver/Packets/PacketMemory.cpp | 138 +++++++++++++ .../src/Driver/Queues/QueueWorker.cpp | 57 +++++- .../src/Driver/Queues/Submission.cpp | 48 ++++- .../Driver/Synchronization/WaitStatistics.cpp | 8 + .../Execution/src/GuestMemory.cpp | 35 +++- .../Graphics/include/ShaderResources.hpp | 5 + .../Graphics/include/SlimMutex.hpp | 42 ++++ .../Graphics/src/BufferPool.cpp | 4 +- .../Graphics/src/ShaderResources.cpp | 48 ++++- .../libSceAgcDriver/Graphics/src/Texture.cpp | 11 +- .../Graphics/src/UnitShadow.cpp | 3 +- .../tests/execution/DrawWriteRanges.cpp | 116 +++++++++++ .../tests/execution/PipelinedDma.cpp | 68 +++++++ 25 files changed, 1098 insertions(+), 31 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp create mode 100644 core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 444a1eeaec2a..ada0118bfc50 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -151,6 +151,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCapture.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawDecode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawLookup.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPipeline.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPrecheck.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawRegisterKey.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawTiming.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp new file mode 100644 index 000000000000..8f290e4caae3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp @@ -0,0 +1,74 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_DRAWPIPELINE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_DRAWPIPELINE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +class DrawPipeline { +public: + using Commit = std::function; + using Range = std::pair; + enum class DrainReason : std::uint8_t { Packet, Flush, Labels, Capture, Indirect, Submission, Count }; + + static DrawPipeline& Queue0(); + static std::size_t Depth(); + static bool& Active(); + static std::atomic& EpochToken(); + static void FollowEpoch(std::uint64_t token); + + void Enqueue(Commit commit, std::vector writes, std::uint64_t labelAddress = 0, std::vector labelBytes = {}); + void Drain(DrainReason reason, std::uint32_t opcode = 0x100); + bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; } + bool Overlaps(std::uint64_t address, std::size_t bytes); + std::optional PendingLabel(std::uint64_t address, std::size_t bytes); + +private: + struct Item { + Commit commit; + std::vector writes; + std::uint64_t labelAddress = 0; + std::vector labelBytes; + }; + DrawPipeline() = default; + void run(); + void rethrowFailure(); + void report(std::chrono::steady_clock::time_point now); + + std::mutex mutex; + std::condition_variable wake; + std::condition_variable idle; + std::deque items; + std::exception_ptr failure; + std::atomic outstanding{0}; + std::uint32_t idleWaiters = 0; + bool committerWaiting = false; + std::thread thread; + std::uint64_t commits = 0; + std::uint64_t commitNs = 0; + std::uint64_t commitErrors = 0; + std::uint64_t enqueued = 0; + std::uint64_t fullWaitNs = 0; + std::array(DrainReason::Count)> drains{}; + std::array(DrainReason::Count)> drainWaitNs{}; + std::array drainOpcodes{}; + std::uint64_t depthSum = 0; + std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 7da8e8022b37..f01ee64f4310 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -97,6 +97,11 @@ class Driver { void enqueue(Submission submission); void noteHeldAtSubmit(Submission& submission, std::size_t cursor); static void forgetUnfinishedWrites(QueueWorker& worker, const Submission& submission); + static void forgetUnfinishedWrites(QueueWorker& worker, std::span labelWrites); + static bool& completionDeferred(); + void deferCompletion(const Submission& submission); + void submitOpenWork(); + void completeSubmission(std::uint64_t serial, std::uint64_t received, std::span labelWrites); static bool waitFree(const Submission& submission); bool queue0Before(std::uint64_t received) const; bool orderReleased(std::uint32_t queue, std::uint64_t received) const; @@ -207,6 +212,13 @@ class Driver { static void appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector>& ranges); void noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue); void noteDrawWriters(std::span stages, std::uint32_t queue); + static std::vector> drawWriteRanges(const Graphics::State& graphics, std::span stages); + void commitDraw(std::shared_ptr localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span stages, std::span snapshots, const std::shared_ptr& recipe, const std::vector>& recipeStages, std::uint64_t drawKey); + bool enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue); + bool enqueueDmaPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state); + void commitDmaStore(std::uint32_t queue, std::uint64_t address, std::span bytes); + void commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, bool endOfPipeInterrupt); + void submitDueAfterCommit(VulkanDevice* localDevice); std::optional newestWriterLocked(std::uint64_t begin, std::uint64_t end) const; std::optional newestWriter(std::uint64_t begin, std::uint64_t end); std::string describeWriters(std::uint64_t begin, std::uint64_t end); @@ -264,6 +276,9 @@ class Driver { static Graphics::Recorder::LateStatistics& lateCountsSeen(); static EpochBumps& epochBumps(); static void bumpEpoch(std::uint64_t EpochBumps::*counter); + static std::chrono::steady_clock::time_point& lastEpochBump(); + static std::deque& releasedTails(); + void noteReleasedTails(const Submission& submission); static bool packetEpoch(); static bool labelTryEachPacket(); static std::chrono::microseconds labelFlushDeadline(); @@ -372,6 +387,7 @@ class Driver { std::uint32_t idleWaiters = 0; std::uint64_t queue0Executing = 0; + std::deque queue0Uncommitted; std::atomic orderHolders{0}; std::atomic runningWorkers{0}; std::atomic idleEpoch{0}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp new file mode 100644 index 000000000000..b5e1b0331032 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp @@ -0,0 +1,18 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWWRITERANGES_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAWWRITERANGES_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +std::vector> DrawWriteRanges(const Graphics::State& graphics, std::span stages, bool exact); +bool ExactDrawWrites(); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index c00249bc0c0a..7002f98de94c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -87,6 +88,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, pendingDispatchPhases().outcome = DispatchOutcome::FillHle; return; } + if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; return; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 23e4f3418cc5..037ff80df8c4 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -1,6 +1,8 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawProgramTrace.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" @@ -32,6 +34,8 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe return DrawVerdict::Drawn; }; auto drawParameters = Pm4::ResolveDraw(packet, queue); + const bool pipelined = DrawPipeline::Active(); + if (pipelined && drawParameters.indirect) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Indirect); bool traceIndirect = false; if (const auto verdict = precheckDraw(queue, submission, packet, drawParameters, rejected, traceIndirect)) return *verdict; phaseTiming.Phase(DrawRowPrecheck); @@ -367,6 +371,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe for (const auto& read : reads) memory.push_back({read.address, std::as_bytes(std::span(read.bytes))}); } + if (pipelined && !deferredLabels().labels.empty()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Labels); if (recordQueuedLabelsAfterCapture(submission.queue, memory)) return draw(queue, packet, submission, rejected); bool rectListBuilt = false; @@ -515,6 +520,23 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe return DrawVerdict::Rejected; } } + if (pipelined && !drawParameters.indirect && deferredLabels().labels.empty()) { + phaseTiming.Phase(DrawRowVectors); + auto writes = drawWriteRanges(graphics, stages); + const auto commitQueue = submission.queue; + const auto commitPacket = (packet[0] >> 8u) & 0xffu; + const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed); + auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), drawParameters, stages = std::move(stages), snapshots = std::move(snapshots), recipe = std::move(recipe), recipeStages = std::move(recipeStages), drawKey, commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry)] { + DrawPipeline::FollowEpoch(epoch); + GuestMemory::SetCurrentPacket(commitPacket, commitQueue); + commitDraw(localDevice, commitQueue, decode->state, drawParameters, stages, snapshots, recipe, recipeStages, drawKey); + }; + auto held = std::make_shared(std::move(commit)); + DrawPipeline::Queue0().Enqueue([held] { (*held)(); }, std::move(writes)); + phaseTiming.Phase(DrawRowLockWait); + timing.Mark("draw_enqueued"); + return drawn(); + } lockForDraw(); noteDrawWriters(stages, submission.queue); phaseTiming.Phase(DrawRowVectors); @@ -535,4 +557,26 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe return drawn(); } +void Driver::commitDraw(std::shared_ptr localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span stages, std::span snapshots, const std::shared_ptr& recipe, const std::vector>& recipeStages, std::uint64_t drawKey) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (auto current = device.Load(); current != nullptr && current != localDevice) localDevice = std::move(current); + struct SubmitDue { + Driver& driver; + const std::shared_ptr& device; + ~SubmitDue() noexcept(false) { + if (std::uncaught_exceptions() == 0) driver.submitDueAfterCommit(device.get()); + } + } submitDue{*this, localDevice}; + recordLabelsForPacket(localDevice.get(), queue); + noteDrawWriters(stages, queue); + if (recipe != nullptr) { + if (localDevice->DrawFromRecipe(graphics, drawParameters, stages, snapshots, recipe) == RecipeOutcome::Recorded) return; + VulkanDevice::NoteRecipe(VulkanDevice::RecipeEvent::Restart, VulkanDevice::RecipeKind::Draw); + } + std::shared_ptr built; + localDevice->Draw(graphics, drawParameters, stages, snapshots, recipeStages.empty() ? nullptr : &built); + if (built != nullptr) attachDrawRecipe(drawKey, recipeStages, std::move(built)); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp new file mode 100644 index 000000000000..049b061f9d9b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -0,0 +1,185 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +namespace { + +constexpr std::array(DrawPipeline::DrainReason::Count)> DrainNames{"packet", "flush", "labels", "capture", "indirect", "submission"}; + +std::uint64_t elapsedNs(std::chrono::steady_clock::time_point since) { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - since).count()); +} + +} + +DrawPipeline& DrawPipeline::Queue0() { + static DrawPipeline* pipeline = new DrawPipeline(); + return *pipeline; +} + +std::size_t DrawPipeline::Depth() { + static const std::size_t depth = [] { + const char* text = std::getenv("APS5_PIPELINED_DRAWS"); + if (text == nullptr || std::getenv("APS5_LOCKED_DRAW_PREPARE") != nullptr || std::getenv("APS5_DRAW_DRAIN") != nullptr || std::getenv("APS5_DRAIN_ALL") != nullptr || std::getenv("APS5_NO_WORDWISE_CAPTURE") != nullptr) return std::size_t{0}; + const auto value = std::strtoull(text, nullptr, 10); + return value == 0 ? std::size_t{0} : value == 1 ? std::size_t{8} : static_cast(std::min(value, 256)); + }(); + return depth; +} + +bool& DrawPipeline::Active() { + static thread_local bool active = false; + return active; +} + +std::atomic& DrawPipeline::EpochToken() { + static std::atomic token{1}; + return token; +} + +void DrawPipeline::FollowEpoch(std::uint64_t token) { + static thread_local std::uint64_t followed = 0; + if (followed == token) return; + followed = token; + GuestMemory::BumpCollectEpoch(); +} + +void DrawPipeline::rethrowFailure() { + if (failure == nullptr) return; + auto error = std::exchange(failure, nullptr); + std::rethrow_exception(error); +} + +void DrawPipeline::Enqueue(Commit commit, std::vector writes, std::uint64_t labelAddress, std::vector labelBytes) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + std::unique_lock lock(mutex); + rethrowFailure(); + if (items.size() >= Depth()) { + const auto start = std::chrono::steady_clock::now(); + ++idleWaiters; + idle.wait(lock, [&] { return items.size() < Depth(); }); + --idleWaiters; + fullWaitNs += elapsedNs(start); + rethrowFailure(); + } + depthSum += items.size(); + ++enqueued; + items.push_back({std::move(commit), std::move(writes), labelAddress, std::move(labelBytes)}); + outstanding.fetch_add(1, std::memory_order_release); + if (!thread.joinable()) thread = std::thread([this] { run(); }); + if (committerWaiting) wake.notify_one(); + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + if (now - lastReport >= std::chrono::seconds(10)) report(now); +} + +void DrawPipeline::report(std::chrono::steady_clock::time_point now) { + lastReport = now; + std::string drainText; + for (std::size_t i = 0; i < drains.size(); ++i) { + char item[64]; + std::snprintf(item, sizeof(item), " %s %llu (%.1f ms)", DrainNames[i], static_cast(drains[i]), static_cast(drainWaitNs[i]) / 1e6); + drainText += item; + } + drainText += "; packet drains by opcode:"; + for (int shown = 0; shown < 5; ++shown) { + const auto top = std::max_element(drainOpcodes.begin(), drainOpcodes.begin() + 0x100); + if (*top == 0) break; + char item[32]; + std::snprintf(item, sizeof(item), " 0x%02x x%llu", static_cast(top - drainOpcodes.begin()), static_cast(*top)); + drainText += item; + *top = 0; + } + drainOpcodes = {}; + const auto perCommit = commits == 0 ? 0.0 : static_cast(commitNs) / 1e3 / static_cast(commits); + const auto perDraw = enqueued == 0 ? 0.0 : static_cast(fullWaitNs) / 1e3 / static_cast(enqueued); + const auto queued = enqueued == 0 ? 0.0 : static_cast(depthSum) / static_cast(enqueued); + std::fprintf(stderr, "[draw] pipelined (10 s, depth %zu): %llu commits at %.1f us each on the committer, %llu failed; %llu enqueued behind %.2f on average, waiting %.1f us per draw for room; drains:%s\n", Depth(), static_cast(commits), perCommit, static_cast(commitErrors), static_cast(enqueued), queued, perDraw, drainText.c_str()); + commits = commitNs = commitErrors = enqueued = fullWaitNs = depthSum = 0; + drains = {}; + drainWaitNs = {}; +} + +void DrawPipeline::run() { + for (;;) { + Commit* commit = nullptr; + { + std::unique_lock lock(mutex); + committerWaiting = true; + wake.wait(lock, [&] { return !items.empty(); }); + committerWaiting = false; + commit = &items.front().commit; + } + const auto start = std::chrono::steady_clock::now(); + std::exception_ptr fatal; + bool failed = false; + try { + (*commit)(); + } catch (const std::exception& error) { + failed = true; + static std::atomic reported{0}; + if (reported.fetch_add(1, std::memory_order_relaxed) < 20) std::fprintf(stderr, "[draw] pipelined commit failed: %.200s\n", error.what()); + } catch (...) { + fatal = std::current_exception(); + } + const auto spent = elapsedNs(start); + std::lock_guard lock(mutex); + items.pop_front(); + ++commits; + commitNs += spent; + if (failed) ++commitErrors; + if (fatal != nullptr && failure == nullptr) failure = fatal; + outstanding.fetch_sub(1, std::memory_order_release); + if (idleWaiters != 0) idle.notify_all(); + } +} + +void DrawPipeline::Drain(DrainReason reason, std::uint32_t opcode) { + if (!Busy()) return; + if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline drain under the GPU lock (reason " + std::to_string(static_cast(reason)) + ", opcode " + std::to_string(opcode) + ")"); + const auto start = std::chrono::steady_clock::now(); + std::unique_lock lock(mutex); + ++idleWaiters; + idle.wait(lock, [&] { return items.empty(); }); + --idleWaiters; + const auto index = static_cast(reason); + ++drains[index]; + ++drainOpcodes[std::min(opcode, 0x100)]; + drainWaitNs[index] += elapsedNs(start); + rethrowFailure(); +} + +std::optional DrawPipeline::PendingLabel(std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > 8) return std::nullopt; + const auto end = address + bytes; + std::lock_guard lock(mutex); + for (auto item = items.rbegin(); item != items.rend(); ++item) { + const bool overlaps = std::any_of(item->writes.begin(), item->writes.end(), [&](const Range& range) { return address < range.second && range.first < end; }); + if (!overlaps) continue; + if (item->labelBytes.empty() || address < item->labelAddress || end > item->labelAddress + item->labelBytes.size()) return std::nullopt; + std::uint64_t value = 0; + std::memcpy(&value, item->labelBytes.data() + (address - item->labelAddress), bytes); + return value; + } + return std::nullopt; +} + +bool DrawPipeline::Overlaps(std::uint64_t address, std::size_t bytes) { + const auto end = address + bytes; + std::lock_guard lock(mutex); + for (const auto& item : items) { + for (const auto& [begin, limit] : item.writes) { + if (address < limit && begin < end) return true; + } + } + return false; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp index 283bdbb2e712..a8a9f85ef431 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp @@ -3,6 +3,7 @@ #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -273,16 +274,17 @@ void Driver::storeBufferFill(std::uint32_t queueId, std::uint64_t base, std::siz ++fills; filledBytes += bytes; if (bytes == 0) return; - GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); - std::lock_guard gpuLock(GuestMemory::GpuMutex()); - const auto holdStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - auto phaseStart = holdStart; - const auto phase = [&](FillPhase which) { - if (!profile) return; - const auto now = std::chrono::steady_clock::now(); - fillPhaseMs[which] += std::chrono::duration(now - phaseStart).count(); - phaseStart = now; - }; + const auto store = [this, queueId, base, bytes, pattern, localDevice, memoryValueFill, noteMetadata] { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto holdStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + auto phaseStart = holdStart; + const auto phase = [&](FillPhase which) { + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + fillPhaseMs[which] += std::chrono::duration(now - phaseStart).count(); + phaseStart = now; + }; recordLabelsForPacket(localDevice.get(), queueId); phase(FillLabels); @@ -380,5 +382,15 @@ void Driver::storeBufferFill(std::uint32_t queueId, std::uint64_t base, std::siz std::fprintf(stderr, "[fill] %llu buffer fills (%.0f MiB), %llu stored by the CPU; hold phases ms (cumulative):%s; pre-store flushes %llu (%llu stored an image); stored on the GPU: uniform %llu (%.0f MiB), pattern %llu (%.0f MiB), chain copies %llu\n", static_cast(fills.load()), filledBytes.load() / 1048576.0, static_cast(cpuFills.load()), line.c_str(), static_cast(flushes), static_cast(flushed), static_cast(uniformFills), uniformBytes / 1048576.0, static_cast(patternFills), patternBytes / 1048576.0, static_cast(chainCopies)); } } + }; + static const bool orderedFills = std::getenv("APS5_PIPELINE_DRAIN_FILLS") == nullptr; + if (orderedFills && queueId == 0 && DrawPipeline::Active()) { + DrawPipeline::Queue0().Enqueue([store, queueId] { + GuestMemory::SetCurrentPacket(0x15, queueId); + store(); + }, {{base, base + bytes}}); + return; + } + store(); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp index bf0f1183818c..fa2ca447d58f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/WriteEvidence.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include @@ -204,6 +205,10 @@ ShaderMemory::PendingWrite Driver::classifyPendingWrite(std::uint64_t address, s } ShaderMemory::PendingWrite Driver::queryPendingWrite(std::uint64_t address, std::size_t bytes, std::span known) { + if (DrawPipeline::Active()) { + auto& pipeline = DrawPipeline::Queue0(); + if (pipeline.Busy() && pipeline.Overlaps(address, bytes)) pipeline.Drain(DrawPipeline::DrainReason::Capture); + } std::uint64_t ValidateCounters::*reason = nullptr; PendingView pending; pending.Load(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 682b28dca221..d7c56a64462c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -1,10 +1,19 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "Optimization/ShaderStageInputInfo.hpp" +#include +#include namespace AgcDriver::DriverDetail { +namespace { + template -void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { +void visitWrittenBuffers(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { for (const auto& binding : compiled.bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; for (std::uint32_t element = 0; element < binding.count; ++element) { @@ -20,6 +29,13 @@ void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compi } } +} + +template +void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { + visitWrittenBuffers(compiled, std::forward(visit)); +} + void Driver::appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector>& ranges) { forEachWrittenBuffer(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); } @@ -46,6 +62,70 @@ void Driver::noteDrawWriters(std::span stages, s } } +bool ExactDrawWrites() { + static const bool exact = std::getenv("APS5_EXACT_DRAW_WRITES") != nullptr; + return exact; +} + +std::vector> DrawWriteRanges(const Graphics::State& graphics, std::span stages, bool exact) { + constexpr std::uint64_t Slack = 64 * 1024; + constexpr std::uint64_t UnknownBytes = 64ull * 1024 * 1024; + std::vector> ranges; + const auto add = [&](std::uint64_t begin, std::uint64_t bytes) { + if (begin != 0) ranges.emplace_back(begin, begin + bytes); + }; + for (const auto& stage : stages) { + if (stage.program == nullptr) continue; + visitWrittenBuffers(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); + for (const auto& binding : stage.program->bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; + for (std::uint32_t element = 0; element < binding.count; ++element) { + const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; + if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 8) continue; + const auto resource = Graphics::DecodeTextureResource(std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8)); + if (resource.baseAddress == 0) continue; + const auto surfaceBytes = Graphics::DescribeSurface(resource).guestBytes; + add(resource.baseAddress, surfaceBytes != 0 ? surfaceBytes : UnknownBytes); + add(resource.dccAddress, surfaceBytes != 0 ? surfaceBytes / 256 + 1 : UnknownBytes); + } + } + } + const auto addColor = [&](const Graphics::ColorTarget& color) { + if (exact && color.bytes != 0) { + const auto slices = static_cast(std::max(color.depth, 1)); + const auto bytes = static_cast(color.bytes) * slices; + add(color.address, bytes); + add(color.dccAddress, (bytes + 255) / 256); + return; + } + const std::uint64_t bytes = color.bytes != 0 ? static_cast(color.bytes) : UnknownBytes; + const auto base = color.surfaceAddress != 0 ? std::min(color.surfaceAddress, color.address) : color.address; + if (base != 0) ranges.emplace_back(base, color.address + 2 * bytes + Slack); + add(color.dccAddress, bytes / 16 + Slack); + }; + if (graphics.hasColorTarget) addColor(graphics.color); + for (const auto& color : graphics.colors) addColor(color); + if (graphics.depth) { + const auto& depth = *graphics.depth; + if (exact && depth.extent.width != 0 && depth.extent.height != 0) { + const bool d16 = depth.format == VK_FORMAT_D16_UNORM || depth.format == VK_FORMAT_D16_UNORM_S8_UINT; + add(depth.address, Graphics::DepthSliceBytes(depth.extent, d16 ? 2u : 4u)); + add(depth.stencilAddress, Graphics::DepthSliceBytes(depth.extent, 1u)); + } else { + const auto align = [](std::uint64_t value) { return (value + 255) & ~255ull; }; + const auto pixels = align(depth.extent.width) * align(depth.extent.height); + const auto bytes = pixels != 0 ? pixels * 8 + Slack : UnknownBytes; + add(depth.address, bytes); + add(depth.stencilAddress, bytes); + } + } + return ranges; +} + +std::vector> Driver::drawWriteRanges(const Graphics::State& graphics, std::span stages) { + return DrawWriteRanges(graphics, stages, ExactDrawWrites()); +} + std::optional Driver::newestWriterLocked(std::uint64_t begin, std::uint64_t end) const { for (auto it = writtenBuffers.rbegin(); it != writtenBuffers.rend(); ++it) { if (begin < it->end && it->begin < end) return *it; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 8daba37b06ab..e304f94d4ccf 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" @@ -15,6 +16,32 @@ namespace AgcDriver::DriverDetail { +namespace { + +bool orderedFills() { + static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_FILLS") == nullptr; + return ordered; +} + +bool keepsDrawPipeline(std::span packet, std::uint32_t header, std::uint32_t opcode) { + if (header == FlipPacketHeader || header == RenderingWaitPacketHeader) return false; + if (Pm4::DrawOpcode(opcode)) return true; + switch (opcode) { + case 0x11: case 0x12: case 0x13: case 0x26: case 0x2a: case 0x2f: case 0x42: case 0x58: case 0x59: case 0x69: case 0x76: case 0x79: case 0x7a: case 0x81: return true; + case 0x46: return packet.size() >= 2 && (packet[1] & 0x3fu) != 0x39u; + case 0x15: return orderedFills(); + case 0x63: case 0x64: case 0x9f: + return packet.size() >= 5 && !DrawPipeline::Queue0().Overlaps(packet[1] | (static_cast(packet[2]) << 32u), static_cast(packet[4]) * 8); + case 0x10: { + const auto operation = (header >> 2u) & 0x3fu; + return operation == 0 || operation == 0x09 || operation == 0x0b || operation == 0x0c || operation == 0x1a; + } + default: return false; + } +} + +} + template void Driver::timed(double WorkerProfile::*bucket, TWork&& work) { static thread_local WorkerProfile profile; @@ -41,6 +68,21 @@ void Driver::execute(const Submission& submission) { } ~FlipHold() { Release(); } } flipHold{*this, submission.holdsFlip}; + const bool pipelined = submission.queue == 0 && DrawPipeline::Depth() != 0; + static const bool orderedWaits = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; + DrawPipeline::Active() = pipelined; + struct PipelineDrain { + bool pipelined; + ~PipelineDrain() { + if (!pipelined || std::uncaught_exceptions() == 0) return; + try { + DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); + } catch (...) { + } + } + } pipelineDrain{pipelined}; + static const bool orderedCompletion = std::getenv("APS5_PIPELINE_DRAIN_SUBMISSIONS") == nullptr && orderedWaits; + if (pipelined && (submission.suspend || !orderedCompletion)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); if (submission.suspend) { PerformanceContext timingContext(submissionTiming); PerformanceTimer timing("Driver.Suspend"); @@ -108,7 +150,11 @@ void Driver::execute(const Submission& submission) { auto& packetProfile = ShaderRecompiler::ThreadOwned(packetProfileSlot); ++packetProfile.submissions; - bumpEpoch(&EpochBumps::submissions); + static const bool epochEach = std::getenv("APS5_SUBMISSION_EPOCH_EACH") != nullptr; + if (epochEach || submission.enqueuedAt == std::chrono::steady_clock::time_point{} || submission.enqueuedAt > lastEpochBump()) { + if (!epochEach) noteReleasedTails(submission); + bumpEpoch(&EpochBumps::submissions); + } clearResolvedAhead(); gateOpened() = true; struct ClearAhead { @@ -126,13 +172,31 @@ void Driver::execute(const Submission& submission) { if (APS5_ENABLE_TIMING_LOG && submission.queue == 0 && pendingFrameTiming == nullptr) includeTimingSubmission(submission, false); PerformanceContext timingContext(submission.queue == 0 ? pendingFrameTiming.get() : FrameTiming::Async()); PerformanceTimer timing("Driver.Packet"); + const bool predicated = Pm4::Predicated(header) && queue.predication.operation != 0; + if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && (enqueueLabelPacket(packet, opcode, submission.queue) || enqueueDmaPacket(packet, opcode, submission.queue, queue))) { + traceLabel(packet, submission.queue); + recent.Record(cursor); + cursor = nextCursor; + continue; + } + if (pipelined && !predicated && (opcode == 0x3c || opcode == 0x93) && header != RenderingWaitPacketHeader && orderedWaits && DrawPipeline::Queue0().Busy()) { + const std::uint64_t awaited = packet[2] | (static_cast(packet[3]) << 32u); + if (const auto value = DrawPipeline::Queue0().PendingLabel(awaited, Pm4::WaitAwaitedBytes(packet)); value.has_value() && Pm4::WaitComparesValue(packet, *value)) { + ++waitOutcomes().fromRecorderSameQueue; + gateOpened() = true; + recent.Record(cursor); + cursor = nextCursor; + continue; + } + } + if (pipelined && DrawPipeline::Queue0().Busy() && !keepsDrawPipeline(packet, header, opcode)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, header == FlipPacketHeader ? 0xffu : opcode); std::shared_lock deviceUse(deviceReplacement, std::defer_lock); if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock(); timing.Mark("device_lock"); GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue); CaptureTrace::Log("packet submission=%llu queue=%x offset=%zu header=%08x words=%zu", static_cast(submission.serial), submission.queue, cursor, header, packet.size()); - if (Pm4::Predicated(header) && queue.predication.operation != 0) { + if (predicated) { landPendingWaits(submission.queue); recordQueuedLabelsBeforeRead(submission.queue); if (!Pm4::PredicationPasses(queue)) { @@ -249,6 +313,7 @@ void Driver::execute(const Submission& submission) { currentPacketOffset() = cursor; if (gateOpened()) { gateOpened() = false; + if (pipelined && DrawPipeline::Queue0().Busy()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode); timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); }); } if (opcode == 0x15) timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); }); @@ -337,6 +402,18 @@ void Driver::execute(const Submission& submission) { PerformanceContext endContext(submission.queue == 0 ? frameTiming() : FrameTiming::Async()); PerformanceTimer endTiming("Driver.EndSubmission"); static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr; + if (pipelined && orderedCompletion && deferredLabels().labels.empty()) { + if (submission.rewindTail == nullptr) { + deferCompletion(submission); + return; + } + if ((std::atomic_ref(*const_cast(submission.rewindTail - 1)).load(std::memory_order_acquire) & 0x80000000u) != 0) { + DrawPipeline::Queue0().Enqueue([this] { submitOpenWork(); }, {}); + executeRewindTail(submission); + return; + } + } + if (pipelined) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Submission); if (!deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { auto& costs = submissionCosts(submission.queue); if (!submitAtEnd && submission.rewindTail == nullptr && submission.queue == 0 && workerQueued() != nullptr && workerQueued()->load(std::memory_order_acquire) != 0) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp index 33863d1b2cac..410a98b3c8cc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketFlush.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -50,6 +51,11 @@ void Driver::flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool return; } if (reap) lastReapTry = now; + if (DrawPipeline::Active() && (submit || record || capped || recordAtLock)) { + static const bool committerSubmits = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; + if (committerSubmits && !record && !recordAtLock && DrawPipeline::Queue0().Busy()) return; + DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Flush); + } std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); if (record) GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 3609990f909f..8a5d4da464df 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -9,6 +10,7 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include #include #include @@ -265,4 +267,140 @@ void Driver::dumpSampleCounters(std::uint64_t address) { } } +bool Driver::enqueueDmaPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state) { + static const bool enabled = std::getenv("APS5_PIPELINE_DMA") != nullptr; + if (!enabled || opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false; + const auto source = ((packet[1] >> 29u) & 3u) | ((packet[6] >> 24u) & 4u) | ((packet[6] >> 25u) & 8u); + constexpr std::size_t immediateLimit = std::size_t{16} << 20u; + if (source == 2) { + const auto store = Pm4::ResolveStore(packet, state, immediateLimit); + if (!store.has_value() || store->Bytes().empty() || !GuestMemory::Accessible(reinterpret_cast(store->address), store->Bytes().size(), true)) return false; + const auto address = store->address; + std::vector bytes(store->Bytes().begin(), store->Bytes().end()); + const auto size = bytes.size(); + DrawPipeline::Queue0().Enqueue([this, queue, address, bytes = std::move(bytes)] { + GuestMemory::SetCurrentPacket(0x50, queue); + commitDmaStore(queue, address, bytes); + }, {{address, address + size}}); + return true; + } + const auto copy = Pm4::DecodeMemoryCopy(packet); + if (!copy.has_value() || copy->destination % 4 != 0 || copy->bytes % 4 != 0 || !GuestMemory::Accessible(reinterpret_cast(copy->source), copy->bytes) || !GuestMemory::Accessible(reinterpret_cast(copy->destination), copy->bytes, true)) return false; + DrawPipeline::Queue0().Enqueue([this, queue, copy = *copy] { + GuestMemory::SetCurrentPacket(0x50, queue); + constexpr std::size_t gpuStoreLimit = 65536; + if (copy.bytes > gpuStoreLimit) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Copy); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (const auto localDevice = device.Load()) { + recordDeferredLabels(localDevice.get(), queue); + const auto outcome = localDevice->CopyBuffer(copy.destination, copy.source, copy.bytes, 0, std::numeric_limits::max(), 0, 0, queue, [](std::span, std::uint64_t) {}); + if (outcome.path == 1 || outcome.path == 3) return; + } + } + std::vector bytes(copy.bytes); + GuestMemory::Read(copy.source, bytes, 1); + commitDmaStore(queue, copy.destination, bytes); + }, {{copy->destination, copy->destination + copy->bytes}}); + return true; +} + +void Driver::commitDmaStore(std::uint32_t queue, std::uint64_t address, std::span bytes) { + constexpr std::size_t gpuStoreLimit = 65536; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + Graphics::StorageTexture::FlushPending(address, bytes.size(), nullptr, "packet store", Graphics::PublishScope::PartialUnits); + recordDeferredLabels(localDevice.get(), queue); + int reason = 4; + std::uint64_t stamp = 0; + if (bytes.size() <= gpuStoreLimit && localDevice != nullptr) { + stamp = ++eventSerial; + reason = localDevice->WriteLabelOnGpu(address, bytes, stamp, queue); + } + if (reason == 0 || reason == 5 || reason == 6) { + noteLabelStore(address, bytes, stamp, queue); + if (reason == 0) ++storesOnGpu; + else ++storesBehindCompletions; + } else { + if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); + GuestMemory::Write(address, bytes, 1); + noteLabelStore(address, bytes, stamp != 0 ? stamp : ++eventSerial, queue); + ++storesOnCpu; + } + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); +} + +bool Driver::enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue) { + static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; + if (!ordered || !DeferLabels() || (opcode != 0x49 && opcode != 0x37)) return false; + const bool endOfPipeInterrupt = opcode == 0x49 && ((packet[2] >> 24u) & 7u) != 0; + const auto label = Pm4::DecodeLabelWrite(packet); + if (!label.has_value()) { + if (opcode != 0x49) return false; + const bool storesNothing = (packet[2] >> 29u) == 0 || (packet[3] | (static_cast(packet[4]) << 32u)) == 0; + if (!storesNothing) return false; + if (!endOfPipeInterrupt) { + ++noOpLabels; + return true; + } + } + std::uint64_t address = 0; + std::vector bytes; + std::vector writes; + if (label.has_value()) { + const auto stored = label->Bytes(); + address = label->address; + bytes.assign(stored.begin(), stored.end()); + writes.emplace_back(address, address + bytes.size()); + } + if (endOfPipeInterrupt) bumpEpoch(&EpochBumps::drains); + auto stored = bytes; + DrawPipeline::Queue0().Enqueue([this, queue, opcode, address, bytes = std::move(bytes), endOfPipeInterrupt] { + GuestMemory::SetCurrentPacket(opcode, queue); + commitLabel(queue, address, bytes, endOfPipeInterrupt); + }, std::move(writes), address, std::move(stored)); + return true; +} + +void Driver::commitLabel(std::uint32_t queue, std::uint64_t address, std::span bytes, bool endOfPipeInterrupt) { + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Label); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), queue); + const bool workOpen = Graphics::Recorder::RecordedWorkSinceSubmit() != 0; + int reason = localDevice != nullptr ? 0 : 1; + if (!bytes.empty()) { + const auto stamp = ++eventSerial; + reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(address, bytes, stamp, queue) : 4; + if (reason != 0 && reason != 5 && reason != 6) { + if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); + GuestMemory::Write(address, bytes, 4); + } + noteLabelStore(address, bytes, stamp, queue); + Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); + countLabelOutcome(reason); + ++immediateLabels; + } else { + ++noOpLabels; + } + if (endOfPipeInterrupt) { + bool deferred = false; + if (localDevice != nullptr && (reason == 0 || reason == 5 || reason == 6)) { + deferred = localDevice->AfterRecordedWork([queue] { AgcDriverDeliverEopInterrupt(queue); }, queue == 0); + if (deferred && workOpen) localDevice->SubmitRecorded(queue == 0); + } + if (!deferred) AgcDriverDeliverEopInterrupt(queue); + } + submitDueAfterCommit(localDevice.get()); +} + +void Driver::submitDueAfterCommit(VulkanDevice* localDevice) { + if (localDevice == nullptr) return; + const auto pending = Graphics::Recorder::PendingLabelSince(); + const bool due = pending.has_value() && std::chrono::steady_clock::now() - *pending >= labelFlushDeadline(); + const bool capped = batchCap() != 0 && Graphics::Recorder::RecordedWorkSinceSubmit() >= batchCap(); + if (due || capped) localDevice->SubmitRecorded(true); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index 56db0c1f6851..309b16849eba 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Queues/WorkerAffinity.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp" @@ -54,6 +56,55 @@ void Driver::markCompleted(std::uint64_t serial) { } } +bool& Driver::completionDeferred() { + static thread_local bool deferred = false; + return deferred; +} + +void Driver::deferCompletion(const Submission& submission) { + std::uint64_t received = submission.received; + { + std::lock_guard lock(mutex); + if (queue0Executing != 0) received = std::min(received, queue0Executing); + queue0Uncommitted.push_back(received); + } + completionDeferred() = true; + DrawPipeline::Queue0().Enqueue([this, serial = submission.serial, received, labelWrites = submission.labelWrites] { + completeSubmission(serial, received, labelWrites); + }, {}); +} + +void Driver::submitOpenWork() { + if (deferredLabels().labels.empty() && !Graphics::Recorder::PendingLabelSince().has_value() && Graphics::Recorder::RecordedWorkSinceSubmit() == 0) return; + ++submissionCosts(0).endSubmits; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::End); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), 0); + if (localDevice != nullptr) localDevice->SubmitRecorded(true); +} + +void Driver::completeSubmission(std::uint64_t serial, std::uint64_t received, std::span labelWrites) { + struct Complete { + Driver& driver; + std::uint64_t serial, received; + std::span labelWrites; + ~Complete() { + bool notify = true; + { + std::lock_guard lock(driver.mutex); + driver.markCompleted(serial); + driver.forgetUnfinishedWrites(driver.workers.at(0), labelWrites); + const auto found = std::find(driver.queue0Uncommitted.begin(), driver.queue0Uncommitted.end(), received); + if (found != driver.queue0Uncommitted.end()) driver.queue0Uncommitted.erase(found); + notify = driver.idleWaiters != 0 || driver.orderHolders.load(std::memory_order_acquire) != 0; + } + if (notify) driver.changed.notify_all(); + } + } complete{*this, serial, received, labelWrites}; + submitOpenWork(); +} + const std::atomic*& Driver::workerQueued() { static thread_local const std::atomic* queued = nullptr; return queued; @@ -203,8 +254,10 @@ void Driver::run(std::uint32_t id) noexcept { { std::lock_guard lock(mutex); rethrowFailure(); - markCompleted(submission.serial); - forgetUnfinishedWrites(workers.at(id), submission); + if (!std::exchange(completionDeferred(), false)) { + markCompleted(submission.serial); + forgetUnfinishedWrites(workers.at(id), submission); + } if (id == 0) queue0Executing = 0; notify = idleWaiters != 0 || orderHolders.load(std::memory_order_acquire) != 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index 3d428973f27b..b7cfef3d625f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -7,6 +7,8 @@ #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "prx/libSceAgcDriver/Execution/include/SubmissionCapture.hpp" #include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp" +#include "ThreadOwned.hpp" +#include "prx/libc/include/HostThreadLocal.hpp" #include #include #include @@ -36,6 +38,33 @@ bool EopRelease(std::span words) { } +std::deque& Driver::releasedTails() { + struct ReleasedTailsStorage {}; + return HostThreadLocal, ReleasedTailsStorage>(); +} + +void Driver::noteReleasedTails(const Submission& submission) { + auto& released = releasedTails(); + released.clear(); + thread_local Submission* scratchSlot = nullptr; + auto& scratch = ShaderRecompiler::ThreadOwned(scratchSlot); + const std::uint32_t* tail = submission.rewindTail; + std::size_t words = submission.rewindWords; + for (std::size_t chunk = 0; chunk < 256 && tail != nullptr; ++chunk) { + if ((std::atomic_ref(*const_cast(tail - 1)).load(std::memory_order_acquire) & 0x80000000u) == 0) break; + released.push_back(tail - 1); + scratch.rewindTail = nullptr; + scratch.rewindWords = 0; + try { + copyCommands(scratch, tail, words); + } catch (const std::exception&) { + break; + } + tail = scratch.rewindTail; + words = scratch.rewindWords; + } +} + void Driver::copyCommands(Submission& submission, const std::uint32_t* guest, std::size_t words) { submission.commands.clear(); submission.conditionalEnds.clear(); @@ -248,6 +277,14 @@ void Driver::executeRewindTail(const Submission& stalled) { Submission tail{}; tail.queue = stalled.queue; if (APS5_ENABLE_TIMING_LOG) tail.receivedAt = std::chrono::steady_clock::now(); + auto& released = releasedTails(); + if (!released.empty() && released.front() == stalled.rewindTail - 1) { + released.pop_front(); + tail.enqueuedAt = lastEpochBump(); + } else { + released.clear(); + tail.enqueuedAt = std::chrono::steady_clock::now(); + } copyCommands(tail, stalled.rewindTail, stalled.rewindWords); if (APS5_ENABLE_TIMING_LOG) tail.copiedAt = std::chrono::steady_clock::now(); validate(tail, stalled.rewindTail); @@ -315,10 +352,10 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { submission.serial = accepted + 1; submission.received = ++eventSerial; + const auto now = std::chrono::steady_clock::now(); + submission.enqueuedAt = now; if (capture != nullptr) capture->Take(captured, submission.serial, captureTime(), submission.heldAtSubmit); if (profile) { - const auto now = std::chrono::steady_clock::now(); - submission.enqueuedAt = now; ++costs.submissions; costs.validateNs += static_cast(std::chrono::duration_cast(validated - copied).count()); costs.copyNs += static_cast(std::chrono::duration_cast((copied - start) + (now - validated)).count()); @@ -373,6 +410,7 @@ bool Driver::waitFree(const Submission& submission) { bool Driver::queue0Before(std::uint64_t received) const { if (queue0Executing != 0 && queue0Executing < received) return true; + if (!queue0Uncommitted.empty() && queue0Uncommitted.front() < received) return true; const auto worker = workers.find(0); if (worker == workers.end()) return false; for (const auto& pending : worker->second.pending) { @@ -455,7 +493,11 @@ void Driver::noteHeldAtSubmit(Submission& submission, std::size_t cursor) { } void Driver::forgetUnfinishedWrites(QueueWorker& worker, const Submission& submission) { - for (const auto dword : submission.labelWrites) { + forgetUnfinishedWrites(worker, submission.labelWrites); +} + +void Driver::forgetUnfinishedWrites(QueueWorker& worker, std::span labelWrites) { + for (const auto dword : labelWrites) { const auto found = worker.unfinishedWrites.find(dword); if (found == worker.unfinishedWrites.end()) continue; if (--found->second == 0) worker.unfinishedWrites.erase(found); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp index 99466faac398..28dfef04d264 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitStatistics.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include @@ -24,8 +25,15 @@ EpochBumps& Driver::epochBumps() { return bumps; } +std::chrono::steady_clock::time_point& Driver::lastEpochBump() { + static thread_local std::chrono::steady_clock::time_point at{}; + return at; +} + void Driver::bumpEpoch(std::uint64_t EpochBumps::*counter) { + lastEpochBump() = std::chrono::steady_clock::now(); GuestMemory::BumpCollectEpoch(); + if (DrawPipeline::Active()) DrawPipeline::EpochToken().fetch_add(1, std::memory_order_relaxed); ++(epochBumps().*counter); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index cd907d4a3dfe..8850453b340d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1070,6 +1070,26 @@ struct ThreadCollectMemo { }; thread_local ThreadCollectMemo threadCollectMemo; +struct EpochCollectMap { + std::uint64_t epoch = 0; + std::unordered_map ranges; +}; + +EpochCollectMap& epochCollectMap(std::uint64_t epoch) { + thread_local EpochCollectMap* slot = nullptr; + auto& map = ShaderRecompiler::ThreadOwned(slot); + if (map.epoch != epoch) { + map.ranges.clear(); + map.epoch = epoch; + } + return map; +} + +bool epochCollectMapEnabled() { + static const bool enabled = std::getenv("APS5_NO_EPOCH_COLLECT_MAP") == nullptr; + return enabled; +} + bool sharedCollectMemo() { static const bool shared = std::getenv("APS5_SHARED_COLLECT_MEMO") != nullptr; return shared; @@ -1155,6 +1175,13 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi if (useMemo && !sharedCollectMemo()) { // An entry exists only for a completed walk of an in-arena range, so the tracker is // initialized and watched; nothing below the lock needs asking. + if (epochCollectMapEnabled()) { + const auto& map = epochCollectMap(epoch); + if (const auto found = map.ranges.find(first); found != map.ranges.end() && found->second.unwatched == unwatched && stop <= found->second.end) { + collectMemoHits.fetch_add(1, std::memory_order_relaxed); + return tracker.generation.load(std::memory_order_relaxed); + } + } for (const auto& entry : threadCollectMemo.entries) { if (entry.epoch == epoch && entry.unwatched == unwatched && entry.begin <= first && stop <= entry.end) { // The current generation, not the memoized one: blocks stamped since (MarkWritten, an @@ -1193,7 +1220,13 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi if (collectMemoEnabled()) { const auto serial = unwatchSerial.load(std::memory_order_relaxed); if (sharedCollectMemo()) tracker.memo[tracker.nextMemo++ % tracker.memo.size()] = {first, stop, epoch, serial}; - else threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial}; + else { + threadCollectMemo.entries[threadCollectMemo.next++ % threadCollectMemo.entries.size()] = {first, stop, epoch, serial}; + if (epochCollectMapEnabled()) { + auto& entry = epochCollectMap(epoch).ranges[first]; + if (entry.unwatched != serial || entry.end < stop) entry = {first, stop, epoch, serial}; + } + } } return tracker.generation; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 77058182fef9..efefda3a7f18 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -81,6 +81,8 @@ class DescriptorCache { }; SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false, std::optional variableCount = std::nullopt); void Free(const SetAllocation& allocation) noexcept; + SetAllocation AllocateRewritten(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false, std::optional variableCount = std::nullopt); + void Recycle(VkDescriptorSetLayout layout, const SetAllocation& allocation) noexcept; // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. struct Stats { std::uint64_t layoutHits = 0; @@ -106,6 +108,7 @@ class DescriptorCache { std::vector pools; std::vector updateAfterBindPools; std::vector dedicated; + std::unordered_map> recycled; Stats stats; }; @@ -152,6 +155,8 @@ class ShaderResources { std::shared_ptr buffer; }; DescriptorCache* cache = nullptr; + VkDescriptorSetLayout layout = VK_NULL_HANDLE; + VkDescriptorSetLayout fragmentLayout = VK_NULL_HANDLE; DescriptorCache::SetAllocation fragmentAllocation; DescriptorCache::SetAllocation allocation; std::vector snapshots; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp new file mode 100644 index 000000000000..32de92284c23 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp @@ -0,0 +1,42 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_SLIMMUTEX_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_SLIMMUTEX_HPP + +#include +#include +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif + +namespace AgcDriver::Graphics { + +class SlimMutex { +public: + SlimMutex() = default; + SlimMutex(const SlimMutex&) = delete; + SlimMutex& operator=(const SlimMutex&) = delete; + + void lock() { + for (unsigned spins = 0;;) { + if (!locked.exchange(true, std::memory_order_acquire)) return; + while (locked.load(std::memory_order_relaxed)) { + if (spins < 128) { + ++spins; +#if defined(__x86_64__) || defined(_M_X64) + _mm_pause(); +#endif + } else { + std::this_thread::yield(); + } + } + } + } + bool try_lock() { return !locked.load(std::memory_order_relaxed) && !locked.exchange(true, std::memory_order_acquire); } + void unlock() { locked.store(false, std::memory_order_release); } + +private: + std::atomic locked{false}; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 6ec77770be96..578f5f5ba348 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include #include #include #include @@ -250,7 +252,6 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag } auto result = found->second.back().allocation; found->second.pop_back(); - if (found->second.empty()) tier.free.erase(found); --tier.slots; tier.retainedBytes -= result.allocationBytes; ++tier.hits; @@ -263,7 +264,6 @@ void BufferPool::evictOldest(Tier& tier, std::vector& evicted) evicted.push_back(oldest->second.front().allocation); tier.retainedBytes -= oldest->second.front().allocation.allocationBytes; oldest->second.pop_front(); - if (oldest->second.empty()) tier.free.erase(oldest); --tier.slots; ++tier.evictions; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 554709280db3..7a1fd91704d1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -3476,6 +3476,34 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l return {set, pool, chain.size() - 1, updateAfterBind}; } +DescriptorCache::SetAllocation DescriptorCache::AllocateRewritten(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind, std::optional variableCount) { + { + std::lock_guard lock(mutex); + if (const auto found = recycled.find(layout); found != recycled.end() && !found->second.empty()) { + const auto allocation = found->second.back(); + found->second.pop_back(); + return allocation; + } + } + return Allocate(layout, sizes, updateAfterBind, variableCount); +} + +void DescriptorCache::Recycle(VkDescriptorSetLayout layout, const SetAllocation& allocation) noexcept { + if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; + { + std::lock_guard lock(mutex); + try { + auto& list = recycled[layout]; + if (list.size() < 4096) { + list.push_back(allocation); + return; + } + } catch (...) { + } + } + Free(allocation); +} + void DescriptorCache::Free(const SetAllocation& allocation) noexcept { if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; std::lock_guard lock(mutex); @@ -4394,8 +4422,14 @@ bool ShaderResources::LayoutUpdateAfterBind() const { } ShaderResources::DrawBindings::~DrawBindings() { - if (cache != nullptr && allocation.set != VK_NULL_HANDLE) cache->Free(allocation); - if (cache != nullptr && fragmentAllocation.set != VK_NULL_HANDLE) cache->Free(fragmentAllocation); + if (cache == nullptr) return; + const auto release = [&](VkDescriptorSetLayout setLayout, const DescriptorCache::SetAllocation& set) { + if (set.set == VK_NULL_HANDLE) return; + if (setLayout != VK_NULL_HANDLE) cache->Recycle(setLayout, set); + else cache->Free(set); + }; + release(layout, allocation); + release(fragmentLayout, fragmentAllocation); } std::optional> ShaderResources::MovedReadOnlyBuffers(std::span shaders, Recorder& recorder) const { @@ -4458,7 +4492,9 @@ std::shared_ptr ShaderResources::PrepareDrawBindi const auto maxSnapshotBytes = maxDrawSnapshotBytes; std::size_t totalSnapshotBytes = 0; auto result = std::make_shared(); - std::vector selected; + thread_local std::vector* selectedSlot = nullptr; + auto& selected = ShaderRecompiler::ThreadOwned(selectedSlot); + selected.clear(); std::array stamp{}; bool stamped = false; for (std::size_t index = 0; index < allocations.size(); ++index) { @@ -4546,11 +4582,13 @@ std::shared_ptr ShaderResources::PrepareDrawBindi const auto variableOf = [&](std::size_t set) { return tableBindings[set] != std::numeric_limits::max() ? std::optional(std::max(1u, static_cast(bindings[tableBindings[set]].imageAllocations.size()))) : std::nullopt; }; result->cache = context.descriptorCache; if (!descriptions[0].empty() || splitSets) { - result->allocation = result->cache->Allocate(_layout, sizesOf(0), updateAfterBind, variableOf(0)); + result->allocation = result->cache->AllocateRewritten(_layout, sizesOf(0), updateAfterBind, variableOf(0)); + result->layout = _layout; Require(result->allocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed"); } if (!descriptions[1].empty() || splitSets) { - result->fragmentAllocation = result->cache->Allocate(fragment.layout, sizesOf(1), updateAfterBind, variableOf(1)); + result->fragmentAllocation = result->cache->AllocateRewritten(fragment.layout, sizesOf(1), updateAfterBind, variableOf(1)); + result->fragmentLayout = fragment.layout; Require(result->fragmentAllocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed"); } auto buffers = descriptorBuffers; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index ae43856dd348..e71366c10b81 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp" #include "prx/libc/include/General.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" @@ -1190,7 +1191,7 @@ class PendingList { // Storage images whose results have not reached guest memory yet. struct PendingWrites { - std::mutex mutex; + SlimMutex mutex; PendingList textures; // Images taken out of `textures` by a FlushPending still storing them (see adjacentPendingUnchanged). std::vector flushing; @@ -3759,9 +3760,11 @@ bool StorageTexture::ScanPending(std::span queries) { query.found = nullptr; if (query.end <= query.begin) continue; const auto bytes = static_cast(query.end - query.begin); - for (const auto* texture : pending.textures) { - if (query.found == nullptr && texture->descriptor.baseAddress == query.begin && texture->guestBytes >= bytes) query.found = texture; - if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; + if (pending.textures.MayOverlap(query.begin, bytes)) { + for (const auto* texture : pending.textures) { + if (query.found == nullptr && texture->descriptor.baseAddress == query.begin && texture->guestBytes >= bytes) query.found = texture; + if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; + } } for (const auto* texture : pending.flushing) { if (texture != query.except && texture->overlaps(query.begin, bytes)) query.overlaps = true; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp index a6a8c6af1de1..45d57d64da3c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" @@ -97,7 +98,7 @@ struct UnitShadow { }; struct Shadows { - std::mutex mutex; + SlimMutex mutex; std::map> byBase; std::uint64_t liveBytes = 0; std::uint64_t peakBytes = 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp new file mode 100644 index 000000000000..9b47b48cccee --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp @@ -0,0 +1,116 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Range = std::pair; +using Ranges = std::vector; + +void Require(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error("draw write ranges: " + what); +} + +bool Contains(const Ranges& ranges, std::uint64_t begin, std::uint64_t end) { + return std::any_of(ranges.begin(), ranges.end(), [&](const Range& range) { return range.first <= begin && end <= range.second; }); +} + +bool Touches(const Ranges& ranges, std::uint64_t begin, std::uint64_t end) { + return std::any_of(ranges.begin(), ranges.end(), [&](const Range& range) { return range.first < end && begin < range.second; }); +} + +std::array ImageWords(std::uint64_t base, std::uint32_t width, std::uint32_t height) { + std::array words{}; + const auto base40 = base >> 8u; + words[0] = static_cast(base40); + words[1] = static_cast((base40 >> 32u) & 0xffu) | (56u << 20u) | (((width - 1u) & 0x3u) << 30u); + words[2] = ((width - 1u) >> 2u) | ((height - 1u) << 14u); + words[3] = 4u | (5u << 3u) | (6u << 6u) | (7u << 9u) | (9u << 28u); + return words; +} + +} + +int main() { + try { + using AgcDriver::DriverDetail::DrawWriteRanges; + constexpr std::uint64_t written = 0x100000000ull; + constexpr std::uint64_t readOnly = 0x200000000ull; + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding images; + images.kind = ShaderRecompiler::DescriptorKind::StorageImage; + images.role = ShaderRecompiler::DescriptorRole::GuestImages; + images.count = 2; + for (const auto base : {written, readOnly}) { + const auto words = ImageWords(base, 256, 128); + images.guestDescriptor.insert(images.guestDescriptor.end(), words.begin(), words.end()); + } + images.imageWritten = {true, false}; + program.bindings.push_back(images); + const std::array stages{{{ShaderRecompiler::ShaderStage::Fragment, &program, 0}}}; + AgcDriver::Graphics::State state{}; + const auto resource = AgcDriver::Graphics::DecodeTextureResource(std::span(images.guestDescriptor).subspan(0, 8)); + const auto surfaceBytes = AgcDriver::Graphics::DescribeSurface(resource).guestBytes; + Require(resource.baseAddress == written && surfaceBytes >= 256u * 128u * 4u, "the test T# does not decode as meant"); + for (const bool exact : {false, true}) { + const auto ranges = DrawWriteRanges(state, stages, exact); + Require(Contains(ranges, written, written + surfaceBytes), "a storage image the draw stores to is not a write range"); + Require(!Touches(ranges, readOnly, readOnly + surfaceBytes), "a storage image the draw only reads is a write range"); + } + program.bindings[0].imageWritten.clear(); + Require(Contains(DrawWriteRanges(state, stages, true), readOnly, readOnly + surfaceBytes), "an image without written flags is not a write range"); + AgcDriver::Graphics::State targets{}; + targets.hasColorTarget = true; + auto& color = targets.color; + color.address = 0x300000000ull; + color.surfaceAddress = color.address; + color.extent = {1920, 1080}; + color.tileMode = AgcDriver::Graphics::ColorTileMode::RenderTarget; + color.elementBytes = 4; + const AgcDriver::Graphics::ColorTargetLayout layout(1920, 1080, color.tileMode, 4); + color.bytes = layout.Bytes(); + color.dccAddress = 0x380000000ull; + AgcDriver::Graphics::DepthTarget depth{}; + depth.address = 0x400000000ull; + depth.stencilAddress = 0x480000000ull; + depth.extent = {1920, 1080}; + depth.format = VK_FORMAT_D32_SFLOAT_S8_UINT; + targets.depth = depth; + const std::span none; + const auto exact = DrawWriteRanges(targets, none, true); + const auto estimated = DrawWriteRanges(targets, none, false); + std::size_t last = 0; + for (std::uint32_t y = 0; y < 1080; y += 7) { + for (std::uint32_t x = 0; x < 1920; x += 5) last = std::max(last, layout.Offset(x, y)); + } + last = std::max(last, layout.Offset(1919, 1079)); + Require(last + 4 <= color.bytes, "the color layout addresses past its bytes"); + Require(Contains(exact, color.address, color.address + color.bytes), "the exact color range misses the layout"); + Require(!Touches(exact, color.address + color.bytes, color.address + color.bytes + 1), "the exact color range runs past the layout"); + Require(Contains(exact, color.dccAddress, color.dccAddress + color.bytes / 256) && !Touches(exact, color.dccAddress + (color.bytes + 255) / 256, color.dccAddress + (color.bytes + 255) / 256 + 1), "the exact DCC range is not one key byte per 256 surface bytes"); + const auto depthBytes = AgcDriver::Graphics::DepthSliceBytes(depth.extent, 4); + const auto stencilBytes = AgcDriver::Graphics::DepthSliceBytes(depth.extent, 1); + Require(Contains(exact, depth.address, depth.address + depthBytes) && !Touches(exact, depth.address + depthBytes, depth.address + depthBytes + 1), "the exact depth range is not DepthSliceBytes"); + Require(Contains(exact, depth.stencilAddress, depth.stencilAddress + stencilBytes) && !Touches(exact, depth.stencilAddress + stencilBytes, depth.stencilAddress + stencilBytes + 1), "the exact stencil range is not DepthSliceBytes"); + for (const auto& [begin, end] : exact) Require(Contains(estimated, begin, end), "an exact range is not inside the estimates"); + std::printf("draw write ranges: color %zu bytes (estimate %llu), depth %llu, stencil %llu\n", color.bytes, static_cast(2 * color.bytes + 65536), static_cast(depthBytes), static_cast(stencilBytes)); + std::puts("draw write ranges tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp new file mode 100644 index 000000000000..c470b4fdc0a7 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedDma.cpp @@ -0,0 +1,68 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void check(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error("pipelined DMA: " + what); +} + +std::array DmaData(bool immediate, std::uint64_t source, std::uint32_t data, std::uint64_t destination, std::uint32_t bytes) { + return {0xc0055000u, immediate ? (2u << 29u) : 0u, immediate ? data : static_cast(source), immediate ? 0u : static_cast(source >> 32u), static_cast(destination), static_cast(destination >> 32u), bytes}; +} + +void Submit(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "submission failed"); +} + +} + +int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIPELINED_DRAWS", "1"); + _putenv_s("APS5_PIPELINE_DMA", "1"); +#else + setenv("APS5_PIPELINED_DRAWS", "1", 1); + setenv("APS5_PIPELINE_DMA", "1", 1); +#endif + try { + alignas(256) static std::array first{}; + alignas(256) static std::array second{}; + first.fill(0); + second.fill(0); + using namespace AgcDriver::DriverDetail; + const auto drainedBefore = storesDrained.load(); + const auto cpuBefore = storesOnCpu.load() + storesOnGpu.load() + storesBehindCompletions.load(); + std::vector words; + const auto fill = DmaData(true, 0, 0x5eed1234u, reinterpret_cast(first.data()), static_cast(sizeof(first))); + const auto copy = DmaData(false, reinterpret_cast(first.data()), 0, reinterpret_cast(second.data()), 512); + words.insert(words.end(), fill.begin(), fill.end()); + words.insert(words.end(), copy.begin(), copy.end()); + Submit(words); + AgcDriverWaitIdle_nid_postfix(); + for (const auto word : first) check(word == 0x5eed1234u, "the immediate fill did not land"); + for (std::size_t i = 0; i < second.size(); ++i) check(second[i] == (i < 128 ? 0x5eed1234u : 0u), "the copy did not see the fill before it, or wrote past its bytes"); + const auto stores = storesOnCpu.load() + storesOnGpu.load() + storesBehindCompletions.load() - cpuBefore; + check(stores == 2, "the DMA_DATA packets were not committed (" + std::to_string(stores) + " committed stores)"); + check(storesDrained.load() == drainedBefore, "a DMA_DATA packet still drained"); + LibcRunShutdown_nid_postfix(); + std::puts("pipelined DMA tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { LibcRunShutdown_nid_postfix(); } catch (...) {} + return 1; + } +} From a18ab8c0094c35625ed901fd000609cdb39a5299 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Tue, 13 Oct 2026 17:36:03 +0200 Subject: [PATCH 02/18] fix(agc): a null storage image in a draw's written set writes nothing DrawWriteRanges decoded every written storage image element, and DecodeTextureResource refuses a zero base address, so a draw with a null storage image bound failed (PPSA01317) instead of skipping it. The element is now skipped by the binder's own null rule (base address words zero) before it is decoded. agc_driver_draw_write_ranges binds a null image next to a written one. Co-Authored-By: Gjon Gojcevic <7807870+gjonn@users.noreply.github.com> Co-Authored-By: SleglJan <90033145+SleglJan@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Execution/src/Driver/Memory/WrittenBuffers.cpp | 4 +++- .../tests/execution/DrawWriteRanges.cpp | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index d7c56a64462c..5da9b3dac887 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -82,7 +82,9 @@ std::vector> DrawWriteRanges(const Graph for (std::uint32_t element = 0; element < binding.count; ++element) { const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 8) continue; - const auto resource = Graphics::DecodeTextureResource(std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8)); + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; + const auto resource = Graphics::DecodeTextureResource(words); if (resource.baseAddress == 0) continue; const auto surfaceBytes = Graphics::DescribeSurface(resource).guestBytes; add(resource.baseAddress, surfaceBytes != 0 ? surfaceBytes : UnknownBytes); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp index 9b47b48cccee..56a46eeb1204 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp @@ -72,6 +72,16 @@ int main() { } program.bindings[0].imageWritten.clear(); Require(Contains(DrawWriteRanges(state, stages, true), readOnly, readOnly + surfaceBytes), "an image without written flags is not a write range"); + auto nullImage = ImageWords(0, 256, 128); + nullImage[0] = 0; + nullImage[1] &= ~0xffu; + std::copy(nullImage.begin(), nullImage.end(), program.bindings[0].guestDescriptor.begin() + 8); + program.bindings[0].imageWritten = {true, true}; + for (const bool exact : {false, true}) { + const auto ranges = DrawWriteRanges(state, stages, exact); + Require(Contains(ranges, written, written + surfaceBytes), "a written image next to a null one is not a write range"); + Require(!Touches(ranges, 0, surfaceBytes), "a null storage image is a write range"); + } AgcDriver::Graphics::State targets{}; targets.hasColorTarget = true; auto& color = targets.color; From 2cd5c75dc8f8238d72c688cc032b032b2067b346 Mon Sep 17 00:00:00 2001 From: Michon <231849@buas.nl> Date: Tue, 13 Oct 2026 17:43:13 +0200 Subject: [PATCH 03/18] fix(agc): a pipelined release with INT_SEL 3 raises no interrupt The pipelined label path treated every nonzero RELEASE_MEM INT_SEL as an end-of-pipe interrupt; main's PacketMemory excludes 3 (send data after write confirm). agc_driver failed on it with APS5_PIPELINED_DRAWS=1 ("release with INT_SEL send data after write confirm raised an interrupt"). Co-Authored-By: TheAggie <67454+TheAggie@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Execution/src/Driver/Packets/PacketMemory.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 8a5d4da464df..f8704cd7554b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -334,7 +334,8 @@ void Driver::commitDmaStore(std::uint32_t queue, std::uint64_t address, std::spa bool Driver::enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue) { static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; if (!ordered || !DeferLabels() || (opcode != 0x49 && opcode != 0x37)) return false; - const bool endOfPipeInterrupt = opcode == 0x49 && ((packet[2] >> 24u) & 7u) != 0; + const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u; + const bool endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3; const auto label = Pm4::DecodeLabelWrite(packet); if (!label.has_value()) { if (opcode != 0x49) return false; From a39261b75c40a913967df2bb40d4805145c9fe31 Mon Sep 17 00:00:00 2001 From: raghav314 Date: Wed, 14 Oct 2026 18:51:29 +0530 Subject: [PATCH 04/18] fix(agc): a CB metadata pass drains the draw pipeline before it runs The resolve, CMASK and DCC passes read the color target on the worker while queued draws could still be writing it. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c50e23120c5f139d3e11391b78b9ab228acee2c0) --- .../libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp index 267b6cf7040d..ebee83d0d4c6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" @@ -15,6 +16,7 @@ std::optional Driver::precheckDraw(const QueueState& queue, const S if (!drawParameters.indirect && !drawParameters.indexed && (drawParameters.indexCount == 0 || drawParameters.instanceCount == 0)) return DrawVerdict::Nothing; if (const auto pass = Graphics::DecodeColorMetadataPass(queue)) { require(!drawParameters.indirect, "indirect CB metadata passes are unsupported"); + if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, (packet[0] >> 8u) & 0xffu); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Draw); std::lock_guard gpuLock(GuestMemory::GpuMutex()); if (device == nullptr) device = std::make_shared(); From 7b40c05a6093f2ab5726dd1540b7fdce0a0d4f51 Mon Sep 17 00:00:00 2001 From: raghav314 Date: Wed, 14 Oct 2026 18:51:29 +0530 Subject: [PATCH 05/18] fix(agc): entry validation drains queued draws that write its runs A capture already drains first; validation compared the runs with guest memory while a queued draw could still write them. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a9c15be3e1c27e48b2e6a04e44c47dd5203a3057) --- .../Execution/src/Driver/Dispatch/VariantValidation.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp index 64d3fc89fe6c..ebefe68205c5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp @@ -1,6 +1,8 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include #include #include @@ -78,6 +80,10 @@ bool Driver::syncPendingRuns(std::uint64_t program, std::uint32_t queue, const S EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional& sampling, std::vector>* live) { if (live != nullptr) live->clear(); + if (DrawPipeline::Active()) { + auto& pipeline = DrawPipeline::Queue0(); + if (pipeline.Busy() && std::any_of(variant.runs.begin(), variant.runs.end(), [&](const auto& run) { return pipeline.Overlaps(run.first, static_cast(run.second - run.first)); })) pipeline.Drain(DrawPipeline::DrainReason::Capture); + } const bool masked = live != nullptr && !variant.dataPositions.empty(); const DataMask mask{variant.dataPositions, live}; PendingView pending; From dc027cc004811bf3fc7866ad82e11ebabf6d8db3 Mon Sep 17 00:00:00 2001 From: raghav314 Date: Wed, 14 Oct 2026 18:51:29 +0530 Subject: [PATCH 06/18] fix(agc): note an HTILE depth clear after the pipeline drain The note landed ahead of the draws still queued over that depth surface. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 41f385acbf8da99fdf3f8a8c0bd9a8802383d05f) --- .../Execution/src/Driver/Dispatch/Dispatch.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 7002f98de94c..de522dba8d9f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -77,10 +77,6 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return; if (!partialLastGroup && indirectArguments == 0 && (matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute) || MatchesStreamCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute))) return; } else { - if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) { - const auto threads = (packet[4] & 0x20u) != 0 ? static_cast(packet[1]) : static_cast(packet[1]) * compute.numThreads[0]; - Graphics::NoteHtileDepthClear(htile, static_cast(threads * 16u)); - } if (indirectArguments == 0 && (packet[4] & 0x20u) == 0) { if (const auto fill = Graphics::MatchMemoryValueFill(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads, {packet[1], packet[2], packet[3]})) Graphics::NoteMemoryValueFill(*fill); } @@ -89,6 +85,10 @@ void Driver::dispatch(QueueState& queue, std::span packet, return; } if (DrawPipeline::Active()) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, 0x15); + if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) { + const auto threads = (packet[4] & 0x20u) != 0 ? static_cast(packet[1]) : static_cast(packet[1]) * compute.numThreads[0]; + Graphics::NoteHtileDepthClear(htile, static_cast(threads * 16u)); + } if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; return; From 6ad05318c9d340959be2af15990c4ee0ab7fcfbd Mon Sep 17 00:00:00 2001 From: Eli Yarson Date: Wed, 14 Oct 2026 16:26:10 -0300 Subject: [PATCH 07/18] fix(agc): a predicated packet drains queued writes to its predicate before reading it A predicated draw, dispatch or register packet stays on the pipelined keep list, so it did not drain before the worker read its predicate. A WRITE_DATA or label to the predicate address, queued behind earlier commit items, was not stored yet: SET_PREDICATION on P, WRITE_DATA P = 1, then a predicated dispatch skipped it as if P were still 0 (and ran it when P was cleared). The worker now drains the draw pipeline first when a queued item writes the predicate's bytes. (cherry picked from commit 56b94480b3de04dfb5c4f4a1bfc5731df59eae29) --- .../src/Driver/Packets/PacketExecution.cpp | 1 + .../execution/PipelinedPredicationRead.cpp | 106 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicationRead.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index e304f94d4ccf..4bd279a45f30 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -190,6 +190,7 @@ void Driver::execute(const Submission& submission) { } } if (pipelined && DrawPipeline::Queue0().Busy() && !keepsDrawPipeline(packet, header, opcode)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, header == FlipPacketHeader ? 0xffu : opcode); + if (pipelined && predicated && DrawPipeline::Queue0().Busy() && DrawPipeline::Queue0().Overlaps(queue.predication.address, queue.predication.operation == 3 ? 8 : 4)) DrawPipeline::Queue0().Drain(DrawPipeline::DrainReason::Packet, opcode); std::shared_lock deviceUse(deviceReplacement, std::defer_lock); if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicationRead.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicationRead.cpp new file mode 100644 index 000000000000..5b400d503642 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicationRead.cpp @@ -0,0 +1,106 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +alignas(256) constexpr std::array Code{0xd7460000u, 0x04010c04u, 0x7e020280u, 0xe0102000u, 0x80000100u, 0xbf810000u}; +constexpr std::uint32_t Untouched = 0xcafebabeu; +alignas(256) std::array Output{}; +alignas(256) std::array Predicate{}; + +void Registers(std::vector& words, std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); +} + +void SetPredication(std::vector& words, const void* address, bool executeWhenSet) { + const auto pointer = reinterpret_cast(address); + words.insert(words.end(), {0xc0022000u, (4u << 16u) | (executeWhenSet ? 0x100u : 0u), static_cast(pointer), static_cast(pointer >> 32u)}); +} + +void ClearPredication(std::vector& words) { + words.insert(words.end(), {0xc0022000u, 0, 0, 0}); +} + +void WritePredicate(std::vector& words, std::uint32_t value) { + const auto pointer = reinterpret_cast(Predicate.data()); + words.insert(words.end(), {0xc0033700u, 0x00100200u, static_cast(pointer), static_cast(pointer >> 32u), value}); +} + +void PredicatedFill(std::vector& words) { + const std::array shape{64, 1, 1}; + Registers(words, 0x207, shape); + const auto address = reinterpret_cast(Code.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + Registers(words, 0x20c, program); + const std::array resource{(4u << 1u) | (1u << 7u)}; + Registers(words, 0x213, resource); + const auto output = reinterpret_cast(Output.data()); + const std::array descriptor{static_cast(output), static_cast((output >> 32u) & 0xffffu) | (4u << 16u), static_cast(Output.size()), 0x00014204u}; + Registers(words, 0x240, descriptor); + words.insert(words.end(), {0xc0031501u, 1, 1, 1, 0x41u}); +} + +void SubmitBehindCommits(std::vector& words) { + std::unique_lock gpuLock(AgcDriver::GuestMemory::GpuMutex()); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); +} + +void Run(std::uint32_t before, std::uint32_t written, bool expectFill) { + Output.fill(Untouched); + Predicate = {before, 0, 0, 0}; + std::vector words; + SetPredication(words, Predicate.data(), true); + WritePredicate(words, written); + PredicatedFill(words); + ClearPredication(words); + SubmitBehindCommits(words); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); + Require(Predicate[0] == written, "the WRITE_DATA to the predicate did not land"); + for (std::size_t i = 0; i < Output.size(); ++i) { + const auto expected = expectFill ? 0u : Untouched; + Require(Output[i] == expected, std::string("a predicated dispatch read a predicate that a queued WRITE_DATA had not yet stored (predicate ") + std::to_string(before) + " -> " + std::to_string(written) + ", word " + std::to_string(i) + ")"); + } +} + +} + +int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIPELINED_DRAWS", "1"); +#else + setenv("APS5_PIPELINED_DRAWS", "1", 1); +#endif + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + Run(0, 1, true); + Run(1, 0, false); + AgcDriverShutdown_nid_postfix(); + std::puts("pipelined predication read tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} From 084d1fe0ae24c68da03c7280c334f6d4bdac1fa4 Mon Sep 17 00:00:00 2001 From: Eli Yarson Date: Wed, 14 Oct 2026 16:26:10 -0300 Subject: [PATCH 08/18] fix(agc): a label packet behind a deferred label is not enqueued ahead of it enqueueLabelPacket lacked the empty-deferred-labels guard that enqueueDmaPacket and the draw path have. A predicated WRITE_DATA is never enqueued, so it is deferred on the worker; an unpredicated WRITE_DATA to the same address that follows it was enqueued and committed first, and the deferred store was recorded after it, leaving the older value (an EOP interrupt could fire out of order the same way). With the guard the later label takes the packet path, which drains and orders it behind the earlier one. (cherry picked from commit 1f894c9b641da3460b684a8356655fbf1aeee635) --- .../src/Driver/Packets/PacketMemory.cpp | 2 +- .../PipelinedPredicatedLabelOrder.cpp | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicatedLabelOrder.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index f8704cd7554b..3ca776207a73 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -333,7 +333,7 @@ void Driver::commitDmaStore(std::uint32_t queue, std::uint64_t address, std::spa bool Driver::enqueueLabelPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue) { static const bool ordered = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr; - if (!ordered || !DeferLabels() || (opcode != 0x49 && opcode != 0x37)) return false; + if (!ordered || !DeferLabels() || (opcode != 0x49 && opcode != 0x37) || !deferredLabels().labels.empty()) return false; const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u; const bool endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3; const auto label = Pm4::DecodeLabelWrite(packet); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicatedLabelOrder.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicatedLabelOrder.cpp new file mode 100644 index 000000000000..3f29450f0e56 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedPredicatedLabelOrder.cpp @@ -0,0 +1,55 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void check(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error("pipelined predicated label order: " + what); +} + +alignas(256) std::array Predicate{}; +alignas(256) std::array Target{}; + +void WriteData(std::vector& words, const void* destination, std::uint32_t value, bool predicated) { + const auto pointer = reinterpret_cast(destination); + words.insert(words.end(), {0xc0033700u | (predicated ? 1u : 0u), 0x00100200u, static_cast(pointer), static_cast(pointer >> 32u), value}); +} + +} + +int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIPELINED_DRAWS", "1"); +#else + setenv("APS5_PIPELINED_DRAWS", "1", 1); +#endif + try { + Predicate = {1, 0, 0, 0}; + Target = {0, 0, 0, 0}; + const auto predicate = reinterpret_cast(Predicate.data()); + std::vector words; + words.insert(words.end(), {0xc0022000u, (4u << 16u) | 0x100u, static_cast(predicate), static_cast(predicate >> 32u)}); + WriteData(words, Target.data(), 1, true); + WriteData(words, Target.data(), 2, false); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "submission failed"); + AgcDriverWaitIdle_nid_postfix(); + check(Target[0] == 2, "a predicated WRITE_DATA landed after the unpredicated WRITE_DATA that follows it (target " + std::to_string(Target[0]) + ", expected 2)"); + LibcRunShutdown_nid_postfix(); + std::puts("pipelined predicated label order test passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { LibcRunShutdown_nid_postfix(); } catch (...) {} + return 1; + } +} From a719fdca60508069696683b367b5f8053b93025e Mon Sep 17 00:00:00 2001 From: Dean Galvin Date: Wed, 14 Oct 2026 21:09:33 +1300 Subject: [PATCH 09/18] fix(agc): a pipelined fill reads its inputs after queued writes and never waits for room under the GPU lock A pattern fill reads its count and pattern from guest memory on the queue worker. With pipelined draws it read them while earlier commit items that write them (labels, DMA stores, draws) were still queued, so it stored the stale value (agc_driver_pattern_buffer_fill under load). It now drains the draw pipeline first when a queued item writes one of its input ranges. The same fill held the GPU lock from the input read through DrawPipeline::Enqueue. With the pipeline full, Enqueue waited for the committer while the committer's next item waited for the GPU lock: a deadlock. The input lock is released before the fill is queued, and Enqueue throws when it would wait for room while the calling thread holds the GPU lock. (cherry picked from commit c653e9ea1d1030ebf691015b30ea04f1cdd59ef8) --- .../include/Driver/Draw/DrawPipeline.hpp | 1 + .../src/Driver/Draw/DrawPipeline.cpp | 1 + .../src/Driver/Memory/BufferFill.cpp | 17 ++ .../tests/execution/PipelinedFill.cpp | 183 ++++++++++++++++++ 4 files changed, 202 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/PipelinedFill.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp index 8f290e4caae3..78f21db92af0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp @@ -33,6 +33,7 @@ class DrawPipeline { void Enqueue(Commit commit, std::vector writes, std::uint64_t labelAddress = 0, std::vector labelBytes = {}); void Drain(DrainReason reason, std::uint32_t opcode = 0x100); bool Busy() const { return outstanding.load(std::memory_order_acquire) != 0; } + std::size_t Outstanding() const { return outstanding.load(std::memory_order_acquire); } bool Overlaps(std::uint64_t address, std::size_t bytes); std::optional PendingLabel(std::uint64_t address, std::size_t bytes); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp index 049b061f9d9b..e7417aa3f109 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -62,6 +62,7 @@ void DrawPipeline::Enqueue(Commit commit, std::vector writes, std::uint64 std::unique_lock lock(mutex); rethrowFailure(); if (items.size() >= Depth()) { + if (GuestMemory::GpuMutex().HeldByThisThread()) throw std::runtime_error("draw pipeline full while the enqueuing thread holds the GPU lock"); const auto start = std::chrono::steady_clock::now(); ++idleWaiters; idle.wait(lock, [&] { return items.size() < Depth(); }); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp index a8a9f85ef431..f4747886fb7c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp @@ -10,6 +10,7 @@ #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include #include +#include #include #include @@ -188,6 +189,20 @@ void Driver::fillClearCount(const Graphics::StorageTexture::FillCoverage& covera std::fprintf(stderr, "%s\n", line.c_str()); } +namespace { + +void drainPendingInputWrites(std::uint32_t queueId, std::initializer_list> inputs) { + if (queueId != 0 || !DrawPipeline::Active()) return; + auto& pipeline = DrawPipeline::Queue0(); + for (const auto& [address, bytes] : inputs) { + if (!pipeline.Overlaps(address, bytes)) continue; + pipeline.Drain(DrawPipeline::DrainReason::Capture, 0x15); + return; + } +} + +} + bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr& localDevice) { auto dwordFill = MatchDwordPatternFill(packet, code, userData, compute); if (!dwordFill) dwordFill = MatchConstantDwordFill(packet, code, userData, compute); @@ -215,6 +230,7 @@ bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::spaninvocations == 0) return true; + drainPendingInputWrites(queueId, {{range->control, 8}, {range->source, 4}}); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); inputLock.lock(); recordLabelsForPacket(localDevice.get(), queueId); @@ -385,6 +401,7 @@ void Driver::storeBufferFill(std::uint32_t queueId, std::uint64_t base, std::siz }; static const bool orderedFills = std::getenv("APS5_PIPELINE_DRAIN_FILLS") == nullptr; if (orderedFills && queueId == 0 && DrawPipeline::Active()) { + if (inputLock.owns_lock()) inputLock.unlock(); DrawPipeline::Queue0().Enqueue([store, queueId] { GuestMemory::SetCurrentPacket(0x15, queueId); store(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedFill.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedFill.cpp new file mode 100644 index 000000000000..ff6892f765cc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelinedFill.cpp @@ -0,0 +1,183 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using namespace AgcDriver::DriverDetail; +alignas(256) constexpr auto Code = PatternFillKernel; +alignas(256) std::array Pattern{}; +alignas(256) std::array Control{}; +alignas(256) std::array Output{}; +alignas(256) std::array Labels{}; +alignas(256) std::array ScratchPattern{0x0f0f0f0fu, 0, 0, 0}; +alignas(256) std::array ScratchControl{256, 0, 0, 0}; +alignas(256) std::array Scratch{}; + +std::array UserData(const void* pattern, const void* control, void* output) { + std::array data{}; + const auto descriptor = [&](std::size_t first, const void* pointer, std::uint32_t stride, std::uint32_t count, std::uint32_t format) { + const auto address = reinterpret_cast(pointer); + data[first] = static_cast(address); + data[first + 1] = static_cast(address >> 32u) | (stride << 16u); + data[first + 2] = count; + data[first + 3] = format; + }; + descriptor(0, pattern, 4, 4, 0x14004u); + descriptor(4, output, 4, 256, 0x14004u); + descriptor(8, control, 16, 1, 0x4dfacu); + return data; +} + +void AppendFill(std::vector& words, const void* pattern, const void* control, void* output) { + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{64, 1, 1}; + registers(0x207, shape); + const auto address = reinterpret_cast(Code.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array resource{(12u << 1u) | (1u << 7u)}; + registers(0x213, resource); + registers(0x240, UserData(pattern, control, output)); + words.insert(words.end(), {0xc0031500u, 4, 1, 1, 0x41u | 0x8000u}); +} + +void AppendScratchFill(std::vector& words) { + AppendFill(words, ScratchPattern.data(), ScratchControl.data(), Scratch.data()); +} + +void AppendOutputFill(std::vector& words) { + AppendFill(words, Pattern.data(), Control.data(), Output.data()); +} + +void AppendLabel(std::vector& words, std::uint32_t* target, std::uint32_t value) { + const auto address = reinterpret_cast(target); + words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), static_cast(address), static_cast(address >> 32u), value, 0, 0}); +} + +struct Blocker { + std::atomic release{false}; + void Enqueue() { + DrawPipeline::Queue0().Enqueue([this] { + while (!release.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::lock_guard gpuLock(AgcDriver::GuestMemory::GpuMutex()); + }, {}); + } +}; + +void SubmitBehind(Blocker& blocker, std::vector& words) { + std::unique_lock gpuLock(AgcDriver::GuestMemory::GpuMutex()); + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + blocker.Enqueue(); +} + +void ReleaseWhen(Blocker& blocker, std::size_t reached, std::size_t passed) { + const auto start = std::chrono::steady_clock::now(); + auto reachedAt = std::chrono::steady_clock::time_point{}; + for (;;) { + const auto outstanding = DrawPipeline::Queue0().Outstanding(); + const auto now = std::chrono::steady_clock::now(); + if (outstanding >= passed) break; + if (outstanding >= reached && reachedAt == std::chrono::steady_clock::time_point{}) reachedAt = now; + if (reachedAt != std::chrono::steady_clock::time_point{} && now - reachedAt > std::chrono::seconds(2)) break; + Require(now - start < std::chrono::seconds(20), "the queue worker never reached the fill"); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + blocker.release = true; +} + +void Finish() { + auto idle = std::async(std::launch::async, [] { AgcDriverWaitIdle_nid_postfix(); }); + if (idle.wait_for(std::chrono::seconds(20)) != std::future_status::ready) { + std::fputs("pipelined fill: the queue worker and the draw pipeline deadlocked\n", stderr); + std::fflush(stderr); + std::_Exit(1); + } + idle.get(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Scratch.data()), sizeof(Scratch)); +} + +void InputsAfterQueuedLabels() { + Output.fill(0xcafebabeu); + Pattern = {0x40404040u, 0, 0, 0}; + Control = {256, 0, 0, 0}; + std::vector words; + AppendScratchFill(words); + AppendLabel(words, &Control[0], 128); + AppendLabel(words, &Pattern[0], 0xababababu); + AppendOutputFill(words); + Blocker blocker; + SubmitBehind(blocker, words); + ReleaseWhen(blocker, 4, 5); + Finish(); + for (std::size_t i = 0; i < Output.size(); ++i) { + const auto expected = i < 128 ? 0xababababu : 0xcafebabeu; + Require(Output[i] == expected, "a fill read its control or pattern before the queued labels that write them, at " + std::to_string(i)); + } +} + +void FillIntoFullPipeline() { + Output.fill(0xcafebabeu); + Pattern = {0x5a5a5a5au, 0, 0, 0}; + Control = {256, 0, 0, 0}; + Labels.fill(0); + const auto depth = DrawPipeline::Depth(); + Require(depth >= 3 && depth - 2 <= Labels.size(), "unexpected draw pipeline depth " + std::to_string(depth)); + std::vector words; + AppendScratchFill(words); + for (std::size_t i = 0; i + 2 < depth; ++i) AppendLabel(words, &Labels[i], static_cast(i + 1)); + AppendOutputFill(words); + Blocker blocker; + SubmitBehind(blocker, words); + ReleaseWhen(blocker, depth, depth + 1); + Finish(); + for (std::size_t i = 0; i + 2 < depth; ++i) Require(Labels[i] == i + 1, "a queued label was not stored"); + for (std::size_t i = 0; i < Output.size(); ++i) Require(Output[i] == (i < 256 ? 0x5a5a5a5au : 0xcafebabeu), "a fill behind a full draw pipeline did not land at " + std::to_string(i)); +} + +} + +int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIPELINED_DRAWS", "1"); +#else + setenv("APS5_PIPELINED_DRAWS", "1", 1); +#endif + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + InputsAfterQueuedLabels(); + FillIntoFullPipeline(); + AgcDriverShutdown_nid_postfix(); + std::puts("pipelined fill tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} From 3826c131461f611c64a34ca3583cde09b9db4d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:16:56 +0200 Subject: [PATCH 10/18] fix(agc): a pipelined memory-value or buffer-pattern fill reads its source after queued writes Co-Authored-By: Claude --- .../Execution/src/Driver/Memory/BufferFill.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp index f4747886fb7c..a70d23d10019 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp @@ -218,6 +218,7 @@ bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::spanbytes; pattern = dwordFill->pattern; } else if (memoryFill) { + drainPendingInputWrites(queueId, {{memoryFill->valueAddress, 4}}); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); inputLock.lock(); recordLabelsForPacket(localDevice.get(), queueId); @@ -260,6 +261,7 @@ bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span(records * 16u); if (matchesBufferPatternFillKernel(code)) { const auto source = userData[4] | (static_cast(userData[5] & 0xffffu) << 32u); + drainPendingInputWrites(queueId, {{source, sizeof(pattern)}}); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); inputLock.lock(); recordLabelsForPacket(localDevice.get(), queueId); @@ -269,6 +271,7 @@ bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span Date: Thu, 15 Oct 2026 11:16:56 +0200 Subject: [PATCH 11/18] fix(agc): a failed pipelined commit is rethrown like an inline draw error Co-Authored-By: Claude --- .../libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp index e7417aa3f109..19837ee8fea8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -127,6 +127,7 @@ void DrawPipeline::run() { failed = true; static std::atomic reported{0}; if (reported.fetch_add(1, std::memory_order_relaxed) < 20) std::fprintf(stderr, "[draw] pipelined commit failed: %.200s\n", error.what()); + fatal = std::current_exception(); } catch (...) { fatal = std::current_exception(); } From 5d5aeab1fbc71d2d775f66e611db5bc4fa22ec05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:16:56 +0200 Subject: [PATCH 12/18] fix(agc): a queued draw's writes include its color targets' CMASK Co-Authored-By: Claude --- .../Execution/src/Driver/Memory/WrittenBuffers.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 5da9b3dac887..7115bcabf5dd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -98,12 +98,14 @@ std::vector> DrawWriteRanges(const Graph const auto bytes = static_cast(color.bytes) * slices; add(color.address, bytes); add(color.dccAddress, (bytes + 255) / 256); + add(color.cmaskAddress, static_cast(color.cmaskBytes) * slices); return; } const std::uint64_t bytes = color.bytes != 0 ? static_cast(color.bytes) : UnknownBytes; const auto base = color.surfaceAddress != 0 ? std::min(color.surfaceAddress, color.address) : color.address; if (base != 0) ranges.emplace_back(base, color.address + 2 * bytes + Slack); add(color.dccAddress, bytes / 16 + Slack); + add(color.cmaskAddress, static_cast(color.cmaskBytes) * std::max(color.depth, 1)); }; if (graphics.hasColorTarget) addColor(graphics.color); for (const auto& color : graphics.colors) addColor(color); From 4051ee19c5c25db82bc3a8f759c097d328ba11c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:19:22 +0200 Subject: [PATCH 13/18] docs(architecture): describe the queue-0 draw pipeline Co-Authored-By: Claude --- docs/dev/ARCHITECTURE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index d6186fb9a116..da58a5d0f0c5 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -97,6 +97,7 @@ flowchart LR - `ShaderRecompiler::Recompile` keeps compiled variants in memory, and `ShaderDiskCache` stores them on disk so later runs reuse them. Before the writable cache, a miss is also looked up in read-only prebuilt packs and loose entry folders under `shader_cache/prebuilt`; pack entries are ordinary encoded cache entries behind a sorted key-hash index, validated by magic, compiler source version and index checksum when mapped, and a miss or rejected entry falls through to the writable cache and the compiler. Cache format 35 also stores the finite stencil-reference set, its replay-safety proof and reserved push-data word in the artifact and binding layouts. Entries from formats 31 to 34 are rejected. - A compute shader whose LDS plus its lock dword exceeds the host's `maxComputeSharedMemorySize` is not given a `Workgroup` array. `Recompile` sets `ShaderInfo::sharedMemoryBytes` for it and the SPIR-V backend addresses the LDS in a coherent storage buffer at descriptor set 1 (`WorkgroupMemoryDescriptorSet`), with a slice of `RecompileResult::workgroupMemoryDwords` per workgroup, indexed by `WorkgroupID` and `NumWorkGroups`. `VulkanDevice` keeps one such buffer, sized from the dispatch dimensions, and binds it after the resource set; programs that fit keep using shared memory. - `Driver::RegisterShader` splits shader preparation in two. `PlanRegistered` runs on the registering guest thread: it validates the registers and entry point, decodes the code and builds the recompile request, so invalid registrations still fail there. The RDNA to IR translation and resource plan (`PrepareShader`) run on a background pool of `hardware_concurrency() / 2` detached threads (at least 2, 64 MiB stacks). +- With `APS5_PIPELINED_DRAWS=1`, queue 0's packet worker decodes, captures and prepares a draw and hands its recording (`Graphics::Draw` under the GPU lock) to a committer thread in [`DrawPipeline.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp), so the next packet is decoded while the previous draw is recorded. Each queued item carries the guest ranges it will write (`DrawWriteRanges`: written buffers and storage images, color targets with their DCC and CMASK, depth and stencil). A later packet that reads or writes one of those ranges, or that the pipeline can't order, drains it first; labels, submission ends, buffer fills and (with `APS5_PIPELINE_DMA=1`) DMA packets are queued as items instead. An exception thrown by a commit is rethrown on the worker at its next enqueue or drain, as an inline draw error would be. Off by default. - The shader registry maps a code address to every header registered for that code, in registration order. Static, graphics and rectangle ABI resolution pick the header at the bound address (or one equal to it, `SameHeader`, which also takes a hull header fused by `sceAgcFuseShaderHalves` (type 3) as the registered back half (type 7) it copies, the fused registers aside; graphics ABI resolution then uses the fused header's shader registers); draws and dispatches pick the program by the stage's binary type (`RegisteredProgram`), and a front entry matching both type 2 and type 4 throws. Within a type the last registration wins; re-registering an identical earlier header makes its snapshot current again without preparing it again, and changed code drops the older headers. - The snapshot is published as pending. `SourceHandleFor`, `InvocationFor` and `ShaderPreparationTransaction::Edit` / `Read` wait for the pending job before reading `PreparedShaders`, and rethrow its failure. `APS5_SYNC_SHADER_PREPARE=1` prepares on the registering thread instead. - On devices with graphics pipeline libraries (`SpirvTarget::registrationModules`), a vertex or pixel artifact is emitted in registration form: the buffer V# words, image modes, swizzles and mip ranges, depth-compare state, export mappings and vertex component selectors it used to specialize on per draw are read at run time (image `ResourceMetadata` and a few state dwords in the shader data), descriptor heaps keep every element in place (an unused one gets a null or placeholder descriptor), and the pixel stage's bindings live in descriptor set 1 so each stage's library depends only on its own set layout (`INDEPENDENT_SETS`). An artifact above `registrationWordLimit` SPIR-V words is re-emitted specialized, and `CompiledShaderArtifact::registrationModule` records which form it has. For artifacts in registration form, `RegistrationResult` materializes the module once, and `PrebuildStageLibrary` builds its pre-rasterization or fragment-shader library on a separate one-thread pool (`APS5_PREBUILD_THREADS`) with a fixed default state, pairing it with a stub stage. A later draw with that state finds both libraries in the store and only links. From ee0a1889b5cf57e64be3272df90d353d79757b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:21:01 +0200 Subject: [PATCH 14/18] fix(agc): an emptied buffer pool size class is erased again Keeping it made evictOldest read the front of an empty deque (agc_buffer_pool aborted with a double free) and stopped Take at an empty class instead of trying the next size. Co-Authored-By: Claude --- core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index 578f5f5ba348..6ec77770be96 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -1,7 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" -#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" -#include #include #include #include @@ -252,6 +250,7 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag } auto result = found->second.back().allocation; found->second.pop_back(); + if (found->second.empty()) tier.free.erase(found); --tier.slots; tier.retainedBytes -= result.allocationBytes; ++tier.hits; @@ -264,6 +263,7 @@ void BufferPool::evictOldest(Tier& tier, std::vector& evicted) evicted.push_back(oldest->second.front().allocation); tier.retainedBytes -= oldest->second.front().allocation.allocationBytes; oldest->second.pop_front(); + if (oldest->second.empty()) tier.free.erase(oldest); --tier.slots; ++tier.evictions; } From 5c09de1510f6a9c9e9d2a23ac835994d61a435ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:23:30 +0200 Subject: [PATCH 15/18] fix(agc): a failed pipelined commit stops queue 0 like a failed worker Co-Authored-By: Claude --- .../include/Driver/Draw/DrawPipeline.hpp | 1 + .../Execution/src/Driver/Draw/DrawPipeline.cpp | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp index 78f21db92af0..d89415137c83 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp @@ -57,6 +57,7 @@ class DrawPipeline { std::atomic outstanding{0}; std::uint32_t idleWaiters = 0; bool committerWaiting = false; + bool stopped = false; std::thread thread; std::uint64_t commits = 0; std::uint64_t commitNs = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp index 19837ee8fea8..7b481921c8a1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include #include @@ -116,22 +117,28 @@ void DrawPipeline::run() { committerWaiting = true; wake.wait(lock, [&] { return !items.empty(); }); committerWaiting = false; - commit = &items.front().commit; + commit = stopped ? nullptr : &items.front().commit; } const auto start = std::chrono::steady_clock::now(); std::exception_ptr fatal; bool failed = false; + bool shutdown = false; try { - (*commit)(); - } catch (const std::exception& error) { + if (commit != nullptr) (*commit)(); + } catch (const std::exception&) { failed = true; - static std::atomic reported{0}; - if (reported.fetch_add(1, std::memory_order_relaxed) < 20) std::fprintf(stderr, "[draw] pipelined commit failed: %.200s\n", error.what()); + fatal = std::current_exception(); + } catch (const ProcessShutdown&) { + shutdown = true; fatal = std::current_exception(); } catch (...) { fatal = std::current_exception(); } const auto spent = elapsedNs(start); + if (fatal != nullptr && !stopped) { + stopped = true; + if (!shutdown) Driver::Get().ReportFailure(fatal); + } std::lock_guard lock(mutex); items.pop_front(); ++commits; From 62e5bc28d0a86f0c56b29818dea72a9c6cd45256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:24:19 +0200 Subject: [PATCH 16/18] fix(agc): a pipelined draw keeps its patched and freshly materialized stages alive The commit lambda held raw pointers into the draw's stage-data hits and compiled stages, which were freed when the worker returned (agc_driver_draw_cache_reuse with APS5_PIPELINED_DRAWS=1). Co-Authored-By: Claude --- .../libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 037ff80df8c4..6a6ae3111d04 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -526,7 +526,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe const auto commitQueue = submission.queue; const auto commitPacket = (packet[0] >> 8u) & 0xffu; const auto epoch = DrawPipeline::EpochToken().load(std::memory_order_relaxed); - auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), drawParameters, stages = std::move(stages), snapshots = std::move(snapshots), recipe = std::move(recipe), recipeStages = std::move(recipeStages), drawKey, commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry)] { + auto commit = [this, localDevice = std::move(localDevice), decode = std::move(decode), drawParameters, stages = std::move(stages), snapshots = std::move(snapshots), recipe = std::move(recipe), recipeStages = std::move(recipeStages), drawKey, commitQueue, commitPacket, epoch, programs = std::move(programs), results = std::move(results), memory = std::move(memory), linked = std::move(linked), stageCaptures = std::move(stageCaptures), matched = std::move(matched), matchedRegions = std::move(matchedRegions), fresh = std::move(fresh), decodeReads = std::move(decodeReads), shaderMemory = std::move(shaderMemory), entry = std::move(entry), dataEntry = std::move(dataEntry), compiledStages = std::move(compiledStages), stageData = std::move(stageData)] { DrawPipeline::FollowEpoch(epoch); GuestMemory::SetCurrentPacket(commitPacket, commitQueue); commitDraw(localDevice, commitQueue, decode->state, drawParameters, stages, snapshots, recipe, recipeStages, drawKey); From 1df509baff05ad6fee6f7ebf805c65202fc701bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:34:27 +0200 Subject: [PATCH 17/18] test(agc): run the failure and draw cache reuse tests with pipelined draws, pin CMASK write ranges Co-Authored-By: Claude --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 10 +++++++--- .../tests/execution/DrawWriteRanges.cpp | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index ada0118bfc50..453dc8c5c666 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1643,8 +1643,10 @@ if(BUILD_TESTING) add_test(NAME agc_descriptor_pool_reset_failure COMMAND agc_driver_graphics_tests --descriptor-pool-reset-failure) add_test(NAME agc_driver_address_space_teardown COMMAND agc_driver_graphics_tests --address-space-teardown) set_tests_properties(agc_driver_address_space_teardown PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_pm4_failure_pipelined COMMAND agc_driver_pm4_tests failure) + set_tests_properties(agc_driver_pm4_failure_pipelined PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=1" SKIP_RETURN_CODE 77 TIMEOUT 30) if(WIN32) - set_tests_properties(agc_driver_pm4_failure agc_descriptor_pool_reset_failure agc_driver_address_space_teardown PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_driver_pm4_failure agc_driver_pm4_failure_pipelined agc_descriptor_pool_reset_failure agc_driver_address_space_teardown PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() if(NOT WIN32) @@ -1665,10 +1667,12 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_GPL=1" SKIP_RETURN_CODE 77) add_test(NAME agc_driver_draw_cache_reuse_verify COMMAND agc_driver_draw_cache_reuse_tests) set_tests_properties(agc_driver_draw_cache_reuse_verify PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_DRAW_DATA_HITS=1" SKIP_RETURN_CODE 77) - set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:") + add_test(NAME agc_driver_draw_cache_reuse_pipelined COMMAND agc_driver_draw_cache_reuse_tests) + set_tests_properties(agc_driver_draw_cache_reuse_pipelined PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PIPELINED_DRAWS=1;APS5_VERIFY_DRAW_RECIPE=1;APS5_VERIFY_REGISTER_KEYS=1" SKIP_RETURN_CODE 77) + set_tests_properties(agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_pipelined PROPERTIES FAIL_REGULAR_EXPRESSION "FATAL:") set_property(TEST agc_driver_draw_cache_reuse agc_driver_draw_cache_reuse_off APPEND PROPERTY ENVIRONMENT "APS5_VERIFY_DRAW_RECIPE=1" "APS5_VERIFY_REGISTER_KEYS=1") if(WIN32) - set_tests_properties(agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_driver_draw_cache_reuse_off agc_driver_draw_cache_reuse_verify agc_driver_draw_cache_reuse_pipelined PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp index 56a46eeb1204..d78479f1c3b1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawWriteRanges.cpp @@ -93,6 +93,8 @@ int main() { const AgcDriver::Graphics::ColorTargetLayout layout(1920, 1080, color.tileMode, 4); color.bytes = layout.Bytes(); color.dccAddress = 0x380000000ull; + color.cmaskAddress = 0x3c0000000ull; + color.cmaskBytes = AgcDriver::Graphics::CmaskBytes(1920, 1080); AgcDriver::Graphics::DepthTarget depth{}; depth.address = 0x400000000ull; depth.stencilAddress = 0x480000000ull; @@ -111,6 +113,10 @@ int main() { Require(Contains(exact, color.address, color.address + color.bytes), "the exact color range misses the layout"); Require(!Touches(exact, color.address + color.bytes, color.address + color.bytes + 1), "the exact color range runs past the layout"); Require(Contains(exact, color.dccAddress, color.dccAddress + color.bytes / 256) && !Touches(exact, color.dccAddress + (color.bytes + 255) / 256, color.dccAddress + (color.bytes + 255) / 256 + 1), "the exact DCC range is not one key byte per 256 surface bytes"); + for (const auto& ranges : {exact, estimated}) { + Require(Contains(ranges, color.cmaskAddress, color.cmaskAddress + color.cmaskBytes), "the CMASK of the color target is not a write range"); + } + Require(!Touches(exact, color.cmaskAddress + color.cmaskBytes, color.cmaskAddress + color.cmaskBytes + 1), "the exact CMASK range runs past CmaskBytes"); const auto depthBytes = AgcDriver::Graphics::DepthSliceBytes(depth.extent, 4); const auto stencilBytes = AgcDriver::Graphics::DepthSliceBytes(depth.extent, 1); Require(Contains(exact, depth.address, depth.address + depthBytes) && !Touches(exact, depth.address + depthBytes, depth.address + depthBytes + 1), "the exact depth range is not DepthSliceBytes"); From 43d7c11b80834b1bb83b69d4ef07a649c2cd0ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Thu, 15 Oct 2026 11:36:09 +0200 Subject: [PATCH 18/18] fix(agc): a pipelined draw owns its ShaderMemory through a unique_ptr now that it cannot be moved Co-Authored-By: Claude --- .../prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 6a6ae3111d04..a6af89efca2e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -210,7 +210,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe for (std::size_t i = 0; i < programs.size(); ++i) decodeVertexInfo(i); phaseTiming.Phase(DrawRowDecode); } - ShaderMemory shaderMemory(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); + auto shaderMemory = std::make_unique(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); std::vector results; std::vector> compiledStages; compiledStages.reserve(programs.size()); @@ -333,7 +333,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe break; } } - compiledStages.push_back(materializeDrawStage(i, pushCursorBytes, queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected)); + compiledStages.push_back(materializeDrawStage(i, pushCursorBytes, queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, *shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected)); if (!rejected.empty()) return DrawVerdict::Rejected; programResults[i] = compiledStages.back().get(); if (candidate != dataCandidates[i].end()) verifyDataStage(i, *candidate->first); @@ -482,7 +482,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe auto& result = results[resultIndex[programIndex]]; const auto pushBytes = result.pushConstants.size(); decodeVertexInfo(programIndex); - result = *materializeDrawStage(programIndex, pushOffsets[programIndex], queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, programs[programIndex].binary.stage == Stage::TessellationControl || programs[programIndex].binary.stage == Stage::TessellationEvaluation ? direct : drawParameters, localDevice, shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected); + result = *materializeDrawStage(programIndex, pushOffsets[programIndex], queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, programs[programIndex].binary.stage == Stage::TessellationControl || programs[programIndex].binary.stage == Stage::TessellationEvaluation ? direct : drawParameters, localDevice, *shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected); if (!rejected.empty()) return DrawVerdict::Rejected; require(result.pushConstants.size() == pushBytes, "patched program changed its push constant layout"); }