From d64657c5b0d29fe1732f8eb6da371374dedb48bd Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 21 Oct 2026 17:51:02 +0100 Subject: [PATCH 01/12] feat(agc): replay a capture several times in one process The first replay pass compiles every shader and pipeline the frames use, so its frame times mostly measure compilation. agc_frame_replay --repeat replays the frames n times: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot, registers the video outputs the snapshot had again and makes the EOP flip reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept). Frame times come from passes 2 to n, each pass is listed under passes, and agc_driver_frame_replay also replays its capture with --repeat 3. Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/tests/FrameReplay.cpp | 25 ++- .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 143 ++++++++++++++---- docs/dev/BUILD.md | 4 +- 3 files changed, 137 insertions(+), 35 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp index 9b557243eb61..d23bdebfb715 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp @@ -188,15 +188,26 @@ void testCaptureAndReplay(const std::filesystem::path& tool) { const auto firstDump = capture / "first.bin"; const auto secondDump = capture / "second.bin"; - auto command = "\"" + tool.string() + "\" \"" + capture.string() + "\" --strict --dump " + std::to_string(at(layout.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + firstDump.string() + - "\" --dump " + std::to_string(at(late.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + secondDump.string() + "\""; + const auto replay = [&](const std::string& options, const std::filesystem::path& output) { + std::filesystem::remove(firstDump); + std::filesystem::remove(secondDump); + auto command = "\"" + tool.string() + "\" \"" + capture.string() + "\" --strict " + options + " --dump " + std::to_string(at(layout.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + + firstDump.string() + "\" --dump " + std::to_string(at(late.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + secondDump.string() + "\" > \"" + output.string() + "\""; #ifdef _WIN32 - command = "\"" + command + "\""; + command = "\"" + command + "\""; #endif - const auto code = std::system(command.c_str()); - check(code == 0, "agc_frame_replay exited with " + std::to_string(code)); - check(readFile(firstDump) == first, "the replay did not reproduce the first copy"); - check(readFile(secondDump) == second, "the replay did not reproduce the second copy"); + const auto code = std::system(command.c_str()); + check(code == 0, "agc_frame_replay " + options + " exited with " + std::to_string(code)); + check(readFile(firstDump) == first, "the replay " + options + " did not reproduce the first copy"); + check(readFile(secondDump) == second, "the replay " + options + " did not reproduce the second copy"); + const auto json = readFile(output); + return std::string(reinterpret_cast(json.data()), json.size()); + }; + replay("", capture / "replay.json"); + const auto repeated = replay("--repeat 3", capture / "repeat.json"); + std::size_t passes = 0; + for (auto found = repeated.find("\"frames\":2,\"p50\""); found != std::string::npos; found = repeated.find("\"frames\":2,\"p50\"", found + 1)) ++passes; + check(passes == 3, "the repeated replay did not replay both frames in each of three passes: " + repeated); } void testConcurrentCopies(const std::filesystem::path& tool) { diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 0ddce118dbcd..815dd9a3c8de 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,7 @@ struct Options { std::chrono::milliseconds drainTimeout{10000}; bool release = true; bool synthesize = true; + std::size_t repeat = 1; }; class Frames { @@ -246,6 +248,21 @@ void mapHostRanges(const FrameCaptureReader& reader, Skipped& skipped) { #endif } +void restoreSnapshot(const FrameCaptureReader& reader, const Skipped& skipped) { + std::vector snapshot; + for (const auto& chunk : reader.Chunks()) { + if (chunk.kind == FrameMemoryKind::Snapshot && !skipped.Overlaps(chunk)) snapshot.push_back(&chunk); + } + const auto workers = std::max(1, std::thread::hardware_concurrency()); + for (std::size_t first = 0; first < snapshot.size(); first += workers) { + std::vector> pending; + for (auto index = first; index < std::min(snapshot.size(), first + workers); ++index) { + pending.push_back(std::async(std::launch::async, [&reader, chunk = snapshot[index]] { restore(reader, *chunk); })); + } + for (auto& future : pending) future.get(); + } +} + Skipped restoreMemory(const FrameCaptureReader& reader) { std::uintptr_t base = 0; std::size_t size = 0; @@ -280,19 +297,16 @@ Skipped restoreMemory(const FrameCaptureReader& reader) { mutation.Add(pointer, bytes, (chunk.access & FrameRangeAccess::Readable) != 0, (chunk.access & FrameRangeAccess::Writable) != 0, gpu); } } - std::vector snapshot; + restoreSnapshot(reader, skipped); + return skipped; +} + +void resetMemory(const FrameCaptureReader& reader, const Skipped& skipped) { + AgcDriverWaitIdle_nid_postfix(); for (const auto& chunk : reader.Chunks()) { - if (chunk.kind == FrameMemoryKind::Snapshot && !skipped.Overlaps(chunk)) snapshot.push_back(&chunk); + if (chunk.kind == FrameMemoryKind::Range && !skipped.Overlaps(chunk)) std::memset(reinterpret_cast(static_cast(chunk.address)), 0, static_cast(chunk.bytes)); } - const auto workers = std::max(1, std::thread::hardware_concurrency()); - for (std::size_t first = 0; first < snapshot.size(); first += workers) { - std::vector> pending; - for (auto index = first; index < std::min(snapshot.size(), first + workers); ++index) { - pending.push_back(std::async(std::launch::async, [&reader, chunk = snapshot[index]] { restore(reader, *chunk); })); - } - for (auto& future : pending) future.get(); - } - return skipped; + restoreSnapshot(reader, skipped); } std::pair progress() { @@ -321,9 +335,11 @@ struct Replay { std::atomic earlyReleases{0}; std::atomic lateReleases{0}; std::atomic syntheticReleases{0}; + std::atomic keptReservations{0}; std::atomic sequenced{0}; std::uint64_t unfinished = 0; std::size_t threads = 0; + std::size_t pass = 0; bool recording = false; Clock::time_point startedAt{}; Clock::time_point endedAt{}; @@ -418,7 +434,15 @@ struct Replay { return; case FrameCallKind::ReserveEopFlip: { const auto token = static_cast(call.values.at(4)); - if (AgcDriverReserveEopFlipAt_nid_postfix(flipInfo(call), token) != token) throw std::runtime_error("frame replay: EOP flip token " + std::to_string(token) + " was not reserved"); + std::uint32_t reserved = 0; + try { + reserved = AgcDriverReserveEopFlipAt_nid_postfix(flipInfo(call), token); + } catch (const std::exception&) { + if (pass == 0) throw; + ++keptReservations; + return; + } + if (reserved != token) throw std::runtime_error("frame replay: EOP flip token " + std::to_string(token) + " was not reserved"); return; } case FrameCallKind::Suspend: @@ -533,6 +557,8 @@ struct Replay { std::size_t started = 0; std::exception_ptr failure; std::atomic finished{false}; + sequenced.store(0); + stalled.store(false); std::thread breaker([&] { if (!options.release) return; auto lastCompleted = progress().second; @@ -626,7 +652,7 @@ struct Replay { int usage() { std::fprintf(stderr, "usage: agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--strict]\n" - " [--dump
::]...\n" + " [--repeat ] [--dump
::]...\n" " replays the driver calls and guest memory that APS5_FRAME_CAPTURE= recorded, through this\n" " build's AGC driver with no title running, and prints one JSON line with the frame times\n" " --no-pace issues calls without waiting for the in-flight depth the title had\n" @@ -634,6 +660,8 @@ int usage() { " --release-after-ms milliseconds without a completion before awaited dwords are released (3)\n" " --no-synthesize releases a wait only with a value the capture recorded, never with its reference value\n" " --strict fails on a pacing timeout or a failed registration inside the captured frames\n" + " --repeat replays the frames n times in this process, restoring the snapshot before each pass;\n" + " with n > 1 the frame times come from passes 2 to n, after the first warmed the caches\n" " --dump writes guest memory to a file after the replay (hexadecimal or decimal)\n" " --frames writes each recorded and replayed display buffer as reference-.bmp and replay-.bmp\n"); return 2; @@ -714,6 +742,20 @@ FrameCheck compareFrames(const FrameCaptureReader& reader, const Options& option return check; } +struct Pass { + std::size_t firstFrame = 0; + std::size_t frames = 0; + double seconds = 0.0; + std::uint64_t paceTimeouts = 0; + std::uint64_t synthesized = 0; +}; + +double percentile(std::vector values, double fraction) { + if (values.empty()) return 0.0; + std::sort(values.begin(), values.end()); + return values[std::min(values.size() - 1, static_cast(fraction * static_cast(values.size())))]; +} + int run(const Options& options) { const auto loadStart = Clock::now(); const FrameCaptureReader reader(options.directory); @@ -728,12 +770,45 @@ int run(const Options& options) { const auto& calls = reader.Calls(); const auto frameStart = std::find_if(calls.begin(), calls.end(), [](const FrameCall& call) { return call.kind == FrameCallKind::FrameStart; }); for (auto call = calls.begin(); call != frameStart; ++call) replay.execute(*call); - replay.recording = true; - replay.startedAt = Clock::now(); - if (frameStart != calls.end()) replay.runWindow(std::span(calls).subspan(static_cast(frameStart - calls.begin()) + 1)); - if (frameStart == calls.end()) AgcDriverWaitIdle_nid_postfix(); - replay.endedAt = Clock::now(); - if (const auto failure = replay.frames.Failure()) std::rethrow_exception(failure); + std::set initialOutputs; + for (const auto& [handle, output] : replay.outputs) initialOutputs.insert(handle); + std::vector passes; + for (std::size_t pass = 0; pass < options.repeat && (pass == 0 || replay.unfinished == 0); ++pass) { + if (pass > 0) { + resetMemory(reader, replay.skipped); + replay.pass = pass; + replay.recording = false; + for (auto output = replay.outputs.begin(); output != replay.outputs.end();) { + if (initialOutputs.contains(output->first)) { + ++output; + continue; + } + AgcDriverUnregisterVideoOutput_nid_postfix(output->first, output->second); + output = replay.outputs.erase(output); + } + for (const auto handle : initialOutputs) { + if (replay.outputs.contains(handle)) continue; + auto output = std::make_shared(replay.frames); + AgcDriverRegisterVideoOutput_nid_postfix(handle, output); + replay.outputs.emplace(handle, std::move(output)); + } + for (auto call = calls.begin(); call != frameStart; ++call) { + if (call->kind == FrameCallKind::ReserveEopFlip) replay.execute(*call); + } + } + Pass measured{replay.frames.Times().size(), 0, 0.0, replay.paceTimeouts.load(), replay.syntheticReleases.load()}; + replay.recording = true; + replay.startedAt = Clock::now(); + if (frameStart != calls.end()) replay.runWindow(std::span(calls).subspan(static_cast(frameStart - calls.begin()) + 1)); + else AgcDriverWaitIdle_nid_postfix(); + replay.endedAt = Clock::now(); + if (const auto failure = replay.frames.Failure()) std::rethrow_exception(failure); + measured.frames = replay.frames.Times().size() - measured.firstFrame; + measured.seconds = std::chrono::duration(replay.endedAt - replay.startedAt).count(); + measured.paceTimeouts = replay.paceTimeouts.load() - measured.paceTimeouts; + measured.synthesized = replay.syntheticReleases.load() - measured.synthesized; + passes.push_back(measured); + } FrameCheck check; if (replay.unfinished == 0) { for (const auto& dump : options.dumps) writeDump(dump); @@ -742,15 +817,30 @@ int run(const Options& options) { std::fprintf(stderr, "[frame-replay] no dumps written: the GPU work did not finish\n"); } const auto ready = replay.frames.Times(); + const std::size_t firstMeasured = passes.size() > 1 ? 1 : 0; std::vector frameMs; - for (std::size_t index = 1; index < ready.size(); ++index) frameMs.push_back(std::chrono::duration(ready[index] - ready[index - 1]).count()); std::string frameList; - for (const auto value : frameMs) frameList += (frameList.empty() ? "" : ",") + std::to_string(static_cast(value)); - std::sort(frameMs.begin(), frameMs.end()); - const auto percentile = [&](double fraction) { return frameMs.empty() ? 0.0 : frameMs[std::min(frameMs.size() - 1, static_cast(fraction * static_cast(frameMs.size())))]; }; - const auto seconds = std::chrono::duration(replay.endedAt - replay.startedAt).count(); - std::printf("{\"frames\":%zu,\"replay_s\":%.3f,\"fps\":%.3f,\"frame_ms\":{\"p50\":%.3f,\"p90\":%.3f,\"max\":%.3f,\"each\":[%s]},\"submits\":%llu,\"delta_mib\":%.1f,\"pace_timeouts\":%llu,\"pace_wait_s\":%.3f,\"stale_registrations\":%llu,\"failed_registrations\":%llu,\"skipped_mib\":%.1f,\"threads\":%zu,\"released_waits\":%llu,\"of_them_from_earlier_writes\":%llu,\"of_them_synthesized\":%llu,\"unfinished\":%llu,\"frames_compared\":%zu,\"pixels_identical\":%.6f,\"max_channel_diff\":%d,\"load_s\":%.3f}\n", - ready.size(), seconds, seconds > 0 ? static_cast(ready.size()) / seconds : 0.0, percentile(0.5), percentile(0.9), frameMs.empty() ? 0.0 : frameMs.back(), frameList.c_str(), + std::string passList; + double seconds = 0.0; + std::size_t frames = 0; + for (std::size_t index = 0; index < passes.size(); ++index) { + const auto& measured = passes[index]; + std::vector passMs; + for (auto frame = measured.firstFrame + 1; frame < measured.firstFrame + measured.frames; ++frame) passMs.push_back(std::chrono::duration(ready[frame] - ready[frame - 1]).count()); + char entry[192]; + std::snprintf(entry, sizeof(entry), "%s{\"replay_s\":%.3f,\"frames\":%zu,\"p50\":%.3f,\"pace_timeouts\":%llu,\"synthesized\":%llu}", passList.empty() ? "" : ",", measured.seconds, measured.frames, + percentile(passMs, 0.5), static_cast(measured.paceTimeouts), static_cast(measured.synthesized)); + passList += entry; + if (index < firstMeasured) continue; + for (const auto value : passMs) frameList += (frameList.empty() ? "" : ",") + std::to_string(static_cast(value)); + frameMs.insert(frameMs.end(), passMs.begin(), passMs.end()); + seconds += measured.seconds; + frames += measured.frames; + } + const auto slowest = frameMs.empty() ? 0.0 : *std::max_element(frameMs.begin(), frameMs.end()); + std::printf("{\"frames\":%zu,\"replay_s\":%.3f,\"fps\":%.3f,\"frame_ms\":{\"p50\":%.3f,\"p90\":%.3f,\"max\":%.3f,\"each\":[%s]},\"passes\":[%s],\"kept_reservations\":%llu,\"submits\":%llu,\"delta_mib\":%.1f,\"pace_timeouts\":%llu,\"pace_wait_s\":%.3f,\"stale_registrations\":%llu,\"failed_registrations\":%llu,\"skipped_mib\":%.1f,\"threads\":%zu,\"released_waits\":%llu,\"of_them_from_earlier_writes\":%llu,\"of_them_synthesized\":%llu,\"unfinished\":%llu,\"frames_compared\":%zu,\"pixels_identical\":%.6f,\"max_channel_diff\":%d,\"load_s\":%.3f}\n", + frames, seconds, seconds > 0 ? static_cast(frames) / seconds : 0.0, percentile(frameMs, 0.5), percentile(frameMs, 0.9), slowest, frameList.c_str(), passList.c_str(), + static_cast(replay.keptReservations.load()), static_cast(replay.submits.load()), static_cast(replay.deltaBytes.load()) / (1024.0 * 1024.0), static_cast(replay.paceTimeouts.load()), replay.paceWaitedSeconds(), static_cast(replay.staleRegistrations.load()), static_cast(replay.failedRegistrations.load()), static_cast(replay.skipped.bytes) / (1024.0 * 1024.0), replay.threads, static_cast(replay.earlyReleases.load()), static_cast(replay.lateReleases.load()), @@ -771,6 +861,7 @@ int main(int argc, char** argv) { else if (argument == "--no-synthesize") options.synthesize = false; else if (argument == "--release-after-ms" && index + 1 < argc) options.releaseAfter = std::chrono::milliseconds(std::strtoll(argv[++index], nullptr, 10)); else if (argument == "--strict") options.strict = true; + else if (argument == "--repeat" && index + 1 < argc) options.repeat = std::max(1, std::strtoull(argv[++index], nullptr, 10)); else if (argument == "--dump" && index + 1 < argc) options.dumps.push_back(parseDump(argv[++index])); else if (argument == "--frames" && index + 1 < argc) options.frames = argv[++index]; else if (!argument.starts_with("-") && options.directory.empty()) options.directory = argument; diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 5c9b4ac5a46f..31eabb83ddca 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -228,14 +228,14 @@ On Windows use `agc_pm4_replay.exe` and prepend `build/core/libs/libs/unpatched` On Windows and Linux, set `APS5_FRAME_CAPTURE` to a directory to record a few frames of a title at the AGC driver boundary, then replay them with `agc_frame_replay` through any build of the driver with no title running. From the start of the process the driver keeps every shader registration and ABI resolution, video output registration and EOP flip reservation. At the frame boundary picked by `APS5_FRAME_CAPTURE_AT_FLIP` (default 1), or by the appearance of the file `APS5_FRAME_CAPTURE_TRIGGER` names (a number in it replaces the frame count), it waits for the driver to go idle and writes every committed, readable allocation of the guest allocation registry, the title's image included, as zstd chunks without their all-zero 64 KiB blocks. A frame ends at a CPU flip or at a submission that flips (a flip packet, or the EOP release of a reserved flip token); the reservation itself only records its token. For the next `APS5_FRAME_CAPTURE_FRAMES` (10) frames it records each submission (queue, command buffer address and size), flip and suspend point with the thread that made it and the accepted and completed serials, each preceded by the guest pages the write tracker saw the CPU write since the previous call that handed work to the GPU (`GuestMemory::CollectWrittenBlocks`, per 4 KiB where the tracker kept page stamps), read once the submissions already accepted have completed (a recorded submission, flip or suspend point reaches the driver before the recorder takes its next call, so that wait covers every call recorded before the delta); an allocation the registry gained after the snapshot is recorded there as a new range with its nonzero 64 KiB blocks. A submission ends a frame only when it flips itself, not when another thread's submission flipped while it was being submitted. Shader registrations and ABI resolutions only collect the blocks of the shader header and the arrays it points to, and the pages of the title's image only when their content changed. At the end it stores each display buffer the title presented, decoded, as a reference image. Without the guest write watch (`APS5_NO_WRITE_WATCH`, or no write watch on the host) the capture throws when it starts, since it could not record CPU stores. A capture is `calls.bin`, `memory.bin` and `capture.json` (commit, switches, sizes, warnings), and the recorder stops with a warning when its volume has less than 4 GiB free. A 20-frame capture in the Demon's Souls Nexus is 5.3 GiB of snapshot and 11.5 GiB of deltas before compression, 3.9 GiB on disk. -`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. +`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The first pass compiles every shader and pipeline the frames use, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. ```sh cmake --build build --target agc_frame_replay build/core/libs/prx/libSceAgcDriver/agc_frame_replay capture ``` -On Windows use `agc_frame_replay.exe` and prepend `build/core/libs/libs/unpatched` to the process `PATH`. With `BUILD_TESTING=ON`, `agc_frame_capture` checks the file format (round trip, zero-block elision, truncated and corrupt files), and `agc_driver_frame_capture_eop` reserves and submits six EOP flips with `APS5_FRAME_CAPTURE_FRAMES=3` and checks that the capture starts after the first flipping submission and holds three frames of one reservation and one submission each, then that a flipping submission on another thread leaves this thread's flip count unchanged, and `agc_driver_frame_replay` captures two frames of CPU writes followed by DMA copies, the second from a block committed during the capture, replays them in `agc_frame_replay` and compares the copies. +On Windows use `agc_frame_replay.exe` and prepend `build/core/libs/libs/unpatched` to the process `PATH`. With `BUILD_TESTING=ON`, `agc_frame_capture` checks the file format (round trip, zero-block elision, truncated and corrupt files), and `agc_driver_frame_capture_eop` reserves and submits six EOP flips with `APS5_FRAME_CAPTURE_FRAMES=3` and checks that the capture starts after the first flipping submission and holds three frames of one reservation and one submission each, then that a flipping submission on another thread leaves this thread's flip count unchanged, and `agc_driver_frame_replay` captures two frames of CPU writes followed by DMA copies, the second from a block committed during the capture, replays them in `agc_frame_replay`, once and with `--repeat 3`, and compares the copies. ## Pipeline statistics From 8a17ba39da5279b4a278cd5c9e4aa7e3363cd19c Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 21 Oct 2026 17:51:19 +0100 Subject: [PATCH 02/12] fix(agc): keep draining a replay until no submission completes for ten minutes The drain gave up after 10 s without a completion even while the driver was still compiling or preparing the work, which reported unfinished submissions on full titles and left a process that could not exit. A blocked wait on one queue does not mean the replay is stuck either: another queue can still be preparing the work that releases it. The drain now gives up only after ten minutes without a completion and reports the waits the queues are blocked on every ten seconds and when it gives up. Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 815dd9a3c8de..c7a83a3c811a 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -69,6 +69,7 @@ struct Options { std::chrono::milliseconds memoryPaceTimeout{10000}; std::chrono::milliseconds releaseAfter{3}; std::chrono::milliseconds drainTimeout{10000}; + std::chrono::milliseconds busyDrainTimeout{600000}; bool release = true; bool synthesize = true; std::size_t repeat = 1; @@ -350,9 +351,24 @@ struct Replay { void report() const { const auto [accepted, completed] = progress(); - std::fprintf(stderr, "[frame-replay] %llu submits, %zu frames ready, %.1f s since the first frame, pacing waited %.1f s (%llu timeouts), serials %llu/%llu\n", + std::fprintf(stderr, "[frame-replay] %llu submits, %zu frames ready, %.1f s since the first frame, pacing waited %.1f s (%llu timeouts), serials %llu/%llu, blocked on%s\n", static_cast(submits.load()), frames.Times().size(), std::chrono::duration(Clock::now() - startedAt).count(), paceWaitedSeconds(), - static_cast(paceTimeouts.load()), static_cast(completed), static_cast(accepted)); + static_cast(paceTimeouts.load()), static_cast(completed), static_cast(accepted), describeBlocked().c_str()); + } + + static std::string describeBlocked() { + std::array awaited{}; + const auto count = AgcDriverAwaitedWaits_nid_postfix(awaited.data(), awaited.size()); + std::string text; + for (std::size_t slot = 0; slot < count; ++slot) { + const auto& wait = awaited[slot]; + if (wait.address == 0) continue; + char entry[128]; + std::snprintf(entry, sizeof(entry), " 0x%llx (function %u, reference 0x%llx, mask 0x%llx, now 0x%llx)", static_cast(wait.address), static_cast(wait.function), + static_cast(wait.reference), static_cast(wait.mask), static_cast(load(wait.address, wait.bytes == 8 ? 8 : 4))); + text += entry; + } + return text.empty() ? " nothing" : text; } static bool paced(FrameCallKind kind) { @@ -614,16 +630,23 @@ struct Replay { void drain() { auto lastCompleted = progress().second; auto lastProgress = Clock::now(); + auto lastReport = lastProgress; while (true) { const auto [accepted, completed] = progress(); unfinished = accepted - completed; if (unfinished == 0) return; + const auto now = Clock::now(); if (completed != lastCompleted) { lastCompleted = completed; - lastProgress = Clock::now(); - } else if (Clock::now() - lastProgress >= options.drainTimeout) { - std::fprintf(stderr, "[frame-replay] %llu submissions did not complete after the last captured call (serials %llu/%llu)\n", static_cast(unfinished), - static_cast(completed), static_cast(accepted)); + lastProgress = now; + } else if (now - lastReport >= options.drainTimeout) { + lastReport = now; + std::fprintf(stderr, "[frame-replay] draining: %llu submissions pending for %.0f s (serials %llu/%llu), blocked on%s\n", static_cast(unfinished), + std::chrono::duration(now - lastProgress).count(), static_cast(completed), static_cast(accepted), describeBlocked().c_str()); + } + if (now - lastProgress >= options.busyDrainTimeout) { + std::fprintf(stderr, "[frame-replay] %llu submissions did not complete after the last captured call (serials %llu/%llu), blocked on%s\n", static_cast(unfinished), + static_cast(completed), static_cast(accepted), describeBlocked().c_str()); return; } std::this_thread::sleep_for(std::chrono::milliseconds(5)); From f691bc9f415e524be1ffb508db323b97e62e4394 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 21 Oct 2026 12:55:06 +0100 Subject: [PATCH 03/12] fix(agc): replay frames with the title's pipeline cache and journal already applied A replay's device has no window, so it never loaded the title's Vulkan pipeline cache file and compiled every pipeline the captured frames used again, minutes for the first Demon's Souls frames. It also replayed the pipeline journal in the background during the measured frames, which the title did at boot. APS5_HEADLESS_PIPELINE_CACHE makes a device without a window load the file, and agc_frame_replay sets it unless --cold-pipelines is given. The driver exports AgcDriverWaitForPipelineJournal, and the replay calls it after the registrations and reports the wait as journal_s. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/Execution/include/Driver.hpp | 1 + .../Execution/include/Driver/Driver.hpp | 1 + .../Execution/src/Driver/EntryPoints.cpp | 4 ++++ .../Execution/src/Driver/Lifecycle.cpp | 10 ++++++++++ .../libSceAgcDriver/Execution/src/VulkanDevice.cpp | 3 ++- .../libs/prx/libSceAgcDriver/tools/FrameReplay.cpp | 14 +++++++++++--- docs/dev/BUILD.md | 2 +- docs/dev/ENVIRONMENT.md | 1 + 8 files changed, 31 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp index 03e68cd1b70b..7c89e376adeb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp @@ -32,6 +32,7 @@ extern "C" void AgcDriverRequestScreenshot_nid_postfix(); extern "C" void AgcDriverSuspendPoint_nid_postfix(); extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std::uint64_t* completed); extern "C" std::size_t AgcDriverAwaitedWaits_nid_postfix(AgcDriver::AwaitedWait* waits, std::size_t capacity); +extern "C" void AgcDriverWaitForPipelineJournal_nid_postfix(); extern "C" std::uint64_t AgcDriverDispatchesAdoptedAhead_nid_postfix(); extern "C" std::uint64_t AgcDriverDispatchesRejectedAhead_nid_postfix(); extern "C" void AgcDriverSetAheadAdoptHook_nid_postfix(void (*hook)()); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 7d89a2589f8d..602d3022efff 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -73,6 +73,7 @@ class Driver { std::pair SerialProgress(); std::uint64_t FlipSubmissions() const; std::vector AwaitedWaits(); + void WaitForPipelineJournal(); bool WaitIdleFor(std::chrono::milliseconds timeout); void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output); void UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp index fbe526d30a7b..b02459557bb8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp @@ -141,6 +141,10 @@ extern "C" std::size_t AgcDriverAwaitedWaits_nid_postfix(AgcDriver::AwaitedWait* return count; } +extern "C" void AgcDriverWaitForPipelineJournal_nid_postfix() { + AgcDriver::DriverDetail::Driver::Get().WaitForPipelineJournal(); +} + extern "C" void AgcDriverSerialProgress_nid_postfix(std::uint64_t* accepted, std::uint64_t* completed) { const auto [submitted, done] = AgcDriver::DriverDetail::Driver::Get().SerialProgress(); *accepted = submitted; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp index e12649fac403..9b4c42f4c8d7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp @@ -278,6 +278,16 @@ std::vector Driver::AwaitedWaits() { return result; } +void Driver::WaitForPipelineJournal() { + std::shared_ptr localDevice = device.Load(); + if (localDevice == nullptr) { + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (device == nullptr) device = std::make_shared(); + localDevice = device; + } + localDevice->WaitForPipelineJournalReplay(); +} + std::pair Driver::SerialProgress() { std::lock_guard lock(mutex); return {accepted, completed}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 89305ccae563..78f7a079d9e7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -2180,7 +2180,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : Graphics::PrepareImportWatch(graphicsContext()); state->bufferPool = std::make_shared(graphicsContext()); state->emptyBuffer = std::make_unique(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - state->pipelineCache = std::make_unique(graphicsContext(), state->properties, window != nullptr || compileOnly); + static const bool headlessPipelineCache = std::getenv("APS5_HEADLESS_PIPELINE_CACHE") != nullptr; + state->pipelineCache = std::make_unique(graphicsContext(), state->properties, window != nullptr || compileOnly || headlessPipelineCache); state->detiler = std::make_unique(graphicsContext()); state->textureCache = std::make_unique(graphicsContext()); state->colorTransfer = std::make_unique(graphicsContext()); diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index c7a83a3c811a..2235f258d6da 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -73,6 +73,7 @@ struct Options { bool release = true; bool synthesize = true; std::size_t repeat = 1; + bool coldPipelines = false; }; class Frames { @@ -675,7 +676,7 @@ struct Replay { int usage() { std::fprintf(stderr, "usage: agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--strict]\n" - " [--repeat ] [--dump
::]...\n" + " [--repeat ] [--cold-pipelines] [--dump
::]...\n" " replays the driver calls and guest memory that APS5_FRAME_CAPTURE= recorded, through this\n" " build's AGC driver with no title running, and prints one JSON line with the frame times\n" " --no-pace issues calls without waiting for the in-flight depth the title had\n" @@ -685,6 +686,7 @@ int usage() { " --strict fails on a pacing timeout or a failed registration inside the captured frames\n" " --repeat replays the frames n times in this process, restoring the snapshot before each pass;\n" " with n > 1 the frame times come from passes 2 to n, after the first warmed the caches\n" + " --cold-pipelines starts with an empty Vulkan pipeline cache instead of the title's pipeline cache file\n" " --dump writes guest memory to a file after the replay (hexadecimal or decimal)\n" " --frames writes each recorded and replayed display buffer as reference-.bmp and replay-.bmp\n"); return 2; @@ -793,6 +795,9 @@ int run(const Options& options) { const auto& calls = reader.Calls(); const auto frameStart = std::find_if(calls.begin(), calls.end(), [](const FrameCall& call) { return call.kind == FrameCallKind::FrameStart; }); for (auto call = calls.begin(); call != frameStart; ++call) replay.execute(*call); + const auto journalStart = Clock::now(); + AgcDriverWaitForPipelineJournal_nid_postfix(); + const auto journalSeconds = std::chrono::duration(Clock::now() - journalStart).count(); std::set initialOutputs; for (const auto& [handle, output] : replay.outputs) initialOutputs.insert(handle); std::vector passes; @@ -861,14 +866,14 @@ int run(const Options& options) { frames += measured.frames; } const auto slowest = frameMs.empty() ? 0.0 : *std::max_element(frameMs.begin(), frameMs.end()); - std::printf("{\"frames\":%zu,\"replay_s\":%.3f,\"fps\":%.3f,\"frame_ms\":{\"p50\":%.3f,\"p90\":%.3f,\"max\":%.3f,\"each\":[%s]},\"passes\":[%s],\"kept_reservations\":%llu,\"submits\":%llu,\"delta_mib\":%.1f,\"pace_timeouts\":%llu,\"pace_wait_s\":%.3f,\"stale_registrations\":%llu,\"failed_registrations\":%llu,\"skipped_mib\":%.1f,\"threads\":%zu,\"released_waits\":%llu,\"of_them_from_earlier_writes\":%llu,\"of_them_synthesized\":%llu,\"unfinished\":%llu,\"frames_compared\":%zu,\"pixels_identical\":%.6f,\"max_channel_diff\":%d,\"load_s\":%.3f}\n", + std::printf("{\"frames\":%zu,\"replay_s\":%.3f,\"fps\":%.3f,\"frame_ms\":{\"p50\":%.3f,\"p90\":%.3f,\"max\":%.3f,\"each\":[%s]},\"passes\":[%s],\"kept_reservations\":%llu,\"submits\":%llu,\"delta_mib\":%.1f,\"pace_timeouts\":%llu,\"pace_wait_s\":%.3f,\"stale_registrations\":%llu,\"failed_registrations\":%llu,\"skipped_mib\":%.1f,\"threads\":%zu,\"released_waits\":%llu,\"of_them_from_earlier_writes\":%llu,\"of_them_synthesized\":%llu,\"unfinished\":%llu,\"frames_compared\":%zu,\"pixels_identical\":%.6f,\"max_channel_diff\":%d,\"load_s\":%.3f,\"journal_s\":%.3f}\n", frames, seconds, seconds > 0 ? static_cast(frames) / seconds : 0.0, percentile(frameMs, 0.5), percentile(frameMs, 0.9), slowest, frameList.c_str(), passList.c_str(), static_cast(replay.keptReservations.load()), static_cast(replay.submits.load()), static_cast(replay.deltaBytes.load()) / (1024.0 * 1024.0), static_cast(replay.paceTimeouts.load()), replay.paceWaitedSeconds(), static_cast(replay.staleRegistrations.load()), static_cast(replay.failedRegistrations.load()), static_cast(replay.skipped.bytes) / (1024.0 * 1024.0), replay.threads, static_cast(replay.earlyReleases.load()), static_cast(replay.lateReleases.load()), static_cast(replay.syntheticReleases.load()), - static_cast(replay.unfinished), check.compared, check.identical, check.maxDifference, std::chrono::duration(loaded - loadStart).count()); + static_cast(replay.unfinished), check.compared, check.identical, check.maxDifference, std::chrono::duration(loaded - loadStart).count(), journalSeconds); std::fflush(stdout); return replay.unfinished == 0 ? 0 : 3; } @@ -885,6 +890,7 @@ int main(int argc, char** argv) { else if (argument == "--release-after-ms" && index + 1 < argc) options.releaseAfter = std::chrono::milliseconds(std::strtoll(argv[++index], nullptr, 10)); else if (argument == "--strict") options.strict = true; else if (argument == "--repeat" && index + 1 < argc) options.repeat = std::max(1, std::strtoull(argv[++index], nullptr, 10)); + else if (argument == "--cold-pipelines") options.coldPipelines = true; else if (argument == "--dump" && index + 1 < argc) options.dumps.push_back(parseDump(argv[++index])); else if (argument == "--frames" && index + 1 < argc) options.frames = argv[++index]; else if (!argument.starts_with("-") && options.directory.empty()) options.directory = argument; @@ -893,8 +899,10 @@ int main(int argc, char** argv) { if (options.directory.empty()) return usage(); #ifdef _WIN32 _putenv_s("APS5_FRAME_CAPTURE", ""); + if (!options.coldPipelines) _putenv_s("APS5_HEADLESS_PIPELINE_CACHE", "1"); #else unsetenv("APS5_FRAME_CAPTURE"); + if (!options.coldPipelines) setenv("APS5_HEADLESS_PIPELINE_CACHE", "1", 1); #endif int code = 2; try { diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 31eabb83ddca..5b9e00b8a6fb 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -228,7 +228,7 @@ On Windows use `agc_pm4_replay.exe` and prepend `build/core/libs/libs/unpatched` On Windows and Linux, set `APS5_FRAME_CAPTURE` to a directory to record a few frames of a title at the AGC driver boundary, then replay them with `agc_frame_replay` through any build of the driver with no title running. From the start of the process the driver keeps every shader registration and ABI resolution, video output registration and EOP flip reservation. At the frame boundary picked by `APS5_FRAME_CAPTURE_AT_FLIP` (default 1), or by the appearance of the file `APS5_FRAME_CAPTURE_TRIGGER` names (a number in it replaces the frame count), it waits for the driver to go idle and writes every committed, readable allocation of the guest allocation registry, the title's image included, as zstd chunks without their all-zero 64 KiB blocks. A frame ends at a CPU flip or at a submission that flips (a flip packet, or the EOP release of a reserved flip token); the reservation itself only records its token. For the next `APS5_FRAME_CAPTURE_FRAMES` (10) frames it records each submission (queue, command buffer address and size), flip and suspend point with the thread that made it and the accepted and completed serials, each preceded by the guest pages the write tracker saw the CPU write since the previous call that handed work to the GPU (`GuestMemory::CollectWrittenBlocks`, per 4 KiB where the tracker kept page stamps), read once the submissions already accepted have completed (a recorded submission, flip or suspend point reaches the driver before the recorder takes its next call, so that wait covers every call recorded before the delta); an allocation the registry gained after the snapshot is recorded there as a new range with its nonzero 64 KiB blocks. A submission ends a frame only when it flips itself, not when another thread's submission flipped while it was being submitted. Shader registrations and ABI resolutions only collect the blocks of the shader header and the arrays it points to, and the pages of the title's image only when their content changed. At the end it stores each display buffer the title presented, decoded, as a reference image. Without the guest write watch (`APS5_NO_WRITE_WATCH`, or no write watch on the host) the capture throws when it starts, since it could not record CPU stores. A capture is `calls.bin`, `memory.bin` and `capture.json` (commit, switches, sizes, warnings), and the recorder stops with a warning when its volume has less than 4 GiB free. A 20-frame capture in the Demon's Souls Nexus is 5.3 GiB of snapshot and 11.5 GiB of deltas before compression, 3.9 GiB on disk. -`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The first pass compiles every shader and pipeline the frames use, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. +`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--cold-pipelines] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The replay's device has no window, but it loads the title's Vulkan pipeline cache file (`vk-pipelines-*.bin` in the shader cache directory) as the windowed device does, so pipelines the title already compiled are not compiled again (`--cold-pipelines` starts with an empty one; the driver switch is `APS5_HEADLESS_PIPELINE_CACHE`). The title replayed its pipeline journal at boot, long before the captured frames, so after the registrations the replay waits for the driver's journal replay to finish before the first pass (`journal_s` in the JSON line). The first pass still compiles every shader and pipeline the frames use that neither cache holds, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. ```sh cmake --build build --target agc_frame_replay diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 13dd3116109b..79b0ee715a7a 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -172,6 +172,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_GPU_WAIT_TIMEOUT_MS` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp` | 1000 | milliseconds a GPU wait may run before a warning is printed | | `APS5_GROUP_CAPTURE_ALL_QUEUES` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp` | | resolves group-ahead capture on queue 0 as well | | `APS5_HANDOFF_SPIN_US` | integer | `core/libs/prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp` | | microseconds a thread spins for a handed-off draw's work or for the GPU mutex before blocking, clamped to 0..10000; unset spins 150 us when the process has more than 8 usable processors and not at all otherwise | +| `APS5_HEADLESS_PIPELINE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | loads the pipeline cache file on a device without a window too; `agc_frame_replay` sets it unless `--cold-pipelines` is given | | `APS5_HEAP_MIRROR_MIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | 24576 | budget in MiB for the heap mirrors held for registered allocations | | `APS5_HOOK_FLUSH_CPU_BLOCKS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | set to 0 flushes pending images whose every 64 KiB block the CPU wrote, instead of skipping them | | `APS5_HOOK_FULL_SYNC` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | makes the flush hook's recorded-store wait a full Sync under the GpuMutex | From 5584210c8f902653e869ede9bbdb781ece423660 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 27 Oct 2026 17:30:50 +0000 Subject: [PATCH 04/12] fix(agc): replay review fixes: inherited pipeline cache, shell quoting, reservation errors --cold-pipelines now clears an inherited APS5_HEADLESS_PIPELINE_CACHE on both platforms, the replay test builds its command with #11157's quoting so the redirect and --repeat work under sh and cmd, and a reservation that fails in a repeated pass is reported and fails the replay instead of being dropped. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 22 ++++- .../Execution/include/Driver/Driver.hpp | 2 +- .../Execution/include/VideoOutput.hpp | 5 + .../Execution/src/Driver/Presentation.cpp | 11 ++- .../prx/libSceAgcDriver/tests/FrameReplay.cpp | 72 ++++++++++++-- .../tests/FrameReplayState.cpp | 93 ++++++++++++++++++- .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 7 +- docs/dev/ENVIRONMENT.md | 2 +- 8 files changed, 192 insertions(+), 22 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 401e0a2b9d66..a1715c5d1d10 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -3594,11 +3594,11 @@ if(BUILD_TESTING) endif() add_test_executable(agc_driver_frame_replay_state_tests tests/FrameReplayState.cpp Execution/src/FrameCapture.cpp) - target_include_directories(agc_driver_frame_replay_state_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_include_directories(agc_driver_frame_replay_state_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) target_link_libraries(agc_driver_frame_replay_state_tests PRIVATE libSceAgcDriver libkernel libc libzstd_static) set_target_properties(agc_driver_frame_replay_state_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_frame_replay_state_tests) - set(agcFrameReplayModes order tokens blocking) + set(agcFrameReplayModes order tokens blocking reservations reservation_failure) if(WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND agcFrameReplayModes zero shader shader_host) endif() @@ -3635,12 +3635,28 @@ if(BUILD_TESTING) add_dependencies(agc_driver_frame_replay_tests agc_frame_replay) add_test(NAME agc_driver_frame_replay COMMAND agc_driver_frame_replay_tests $) set_tests_properties(agc_driver_frame_replay PROPERTIES - ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=2" + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame replay capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=2" TIMEOUT 120) if(WIN32) set_tests_properties(agc_driver_frame_replay PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test(NAME agc_driver_frame_replay_cold_pipelines COMMAND agc_driver_frame_replay_tests $ pipelines) + set_tests_properties(agc_driver_frame_replay_cold_pipelines PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=0;APS5_HEADLESS_PIPELINE_CACHE=1;ANYPS5_SHADER_CACHE_DIR=${CMAKE_CURRENT_BINARY_DIR}/frame replay pipeline cache;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame replay cold capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=2" + TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_driver_frame_replay_cold_pipelines PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test(NAME agc_driver_frame_replay_reservation_error COMMAND agc_driver_frame_replay_tests $ reservation_error) + set_tests_properties(agc_driver_frame_replay_reservation_error PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame replay reservation error" + TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_driver_frame_replay_reservation_error PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test(NAME agc_driver_frame_replay_concurrent COMMAND agc_driver_frame_replay_tests $ concurrent) set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-concurrent-capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=8" diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 602d3022efff..978b67587afd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -442,7 +442,7 @@ class Driver { std::map queues; std::map> outputs; struct EopFlip { - std::uint32_t handle; + FlipInfo info; std::shared_ptr request; }; std::map eopFlips; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp index c642371e8e8f..95696a705f50 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VideoOutput.hpp @@ -32,6 +32,11 @@ struct FlipInfo { std::int64_t argument; }; +class EopFlipAlreadyReserved final : public std::runtime_error { +public: + EopFlipAlreadyReserved() : std::runtime_error("EOP flip token is already reserved for this flip") {} +}; + class IFlipRequest { public: virtual ~IFlipRequest() = default; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp index d607e1ae5358..efbef80f3995 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp @@ -24,12 +24,17 @@ std::uint32_t Driver::ReserveEopFlipAt(const FlipInfo& info, std::uint32_t token std::uint32_t Driver::reserveEopFlip(const FlipInfo& info, std::uint32_t token) { rethrowFailure(); checkStopping(); - require(token >= EopFlipTokenBase && token <= EopFlipTokenLimit && !eopFlips.contains(token), "EOP flip token is not free"); + require(token >= EopFlipTokenBase && token <= EopFlipTokenLimit, "EOP flip token is out of range"); const auto output = outputs.find(info.handle); require(output != outputs.end(), "EOP flip references an unregistered video output"); + if (const auto found = eopFlips.find(token); found != eopFlips.end()) { + const auto& reserved = found->second.info; + require(reserved.handle == info.handle && reserved.index == info.index && reserved.mode == info.mode && reserved.argument == info.argument, "EOP flip token is reserved for a different flip"); + throw EopFlipAlreadyReserved{}; + } auto request = output->second->TryReserve(info); if (!request) return 0; - eopFlips.emplace(token, EopFlip{info.handle, std::move(request)}); + eopFlips.emplace(token, EopFlip{info, std::move(request)}); nextEopToken = std::max(nextEopToken, token + 1); return token; } @@ -53,7 +58,7 @@ void Driver::UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptrsecond == output, "video output registration mismatch"); - std::erase_if(eopFlips, [handle](const auto& entry) { return entry.second.handle == handle; }); + std::erase_if(eopFlips, [handle](const auto& entry) { return entry.second.info.handle == handle; }); outputs.erase(it); } diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp index d23bdebfb715..58f420fb6026 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp @@ -134,7 +134,7 @@ bool equal(const std::byte* bytes, const std::vector& expected) { return std::memcmp(bytes, expected.data(), expected.size()) == 0; } -void testCaptureAndReplay(const std::filesystem::path& tool) { +void testCaptureAndReplay(const std::filesystem::path& tool, bool pipelines) { const char* directory = std::getenv("APS5_FRAME_CAPTURE"); check(directory != nullptr && *directory != '\0', "APS5_FRAME_CAPTURE is not set"); const std::filesystem::path capture(directory); @@ -186,13 +186,13 @@ void testCaptureAndReplay(const std::filesystem::path& tool) { check(recorded(late.base), "the block committed during the capture is not in its ranges"); check(reader.Header().find("\"late_ranges\":1") != std::string::npos, "the capture header does not count one range committed during the capture: " + reader.Header()); - const auto firstDump = capture / "first.bin"; - const auto secondDump = capture / "second.bin"; + const auto firstDump = capture / "first dump.bin"; + const auto secondDump = capture / "second dump.bin"; const auto replay = [&](const std::string& options, const std::filesystem::path& output) { std::filesystem::remove(firstDump); std::filesystem::remove(secondDump); auto command = "\"" + tool.string() + "\" \"" + capture.string() + "\" --strict " + options + " --dump " + std::to_string(at(layout.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + - firstDump.string() + "\" --dump " + std::to_string(at(late.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + secondDump.string() + "\" > \"" + output.string() + "\""; + firstDump.string() + "\" --dump " + std::to_string(at(late.firstTarget)) + ":" + std::to_string(DataBytes) + ":\"" + secondDump.string() + "\" > \"" + output.string() + "\" 2> \"" + output.string() + ".log\""; #ifdef _WIN32 command = "\"" + command + "\""; #endif @@ -203,8 +203,20 @@ void testCaptureAndReplay(const std::filesystem::path& tool) { const auto json = readFile(output); return std::string(reinterpret_cast(json.data()), json.size()); }; - replay("", capture / "replay.json"); - const auto repeated = replay("--repeat 3", capture / "repeat.json"); + replay("", capture / "replay output.json"); + if (pipelines) { + check(std::getenv("APS5_HEADLESS_PIPELINE_CACHE") != nullptr, "the pipeline cache switch was not inherited"); + const auto warm = capture / "warm output.json"; + replay("", warm); + const auto warmLog = readFile(warm.string() + ".log"); + check(std::string(reinterpret_cast(warmLog.data()), warmLog.size()).find("[pipeline-cache] loaded ") != std::string::npos, "the warm replay did not load the seeded pipeline cache"); + const auto cold = capture / "cold output.json"; + replay("--cold-pipelines", cold); + const auto coldLog = readFile(cold.string() + ".log"); + check(std::string(reinterpret_cast(coldLog.data()), coldLog.size()).find("[pipeline-cache] loaded ") == std::string::npos, "--cold-pipelines loaded a cache with APS5_HEADLESS_PIPELINE_CACHE inherited"); + return; + } + const auto repeated = replay("--repeat 3", capture / "repeat output.json"); std::size_t passes = 0; for (auto found = repeated.find("\"frames\":2,\"p50\""); found != std::string::npos; found = repeated.find("\"frames\":2,\"p50\"", found + 1)) ++passes; check(passes == 3, "the repeated replay did not replay both frames in each of three passes: " + repeated); @@ -272,16 +284,56 @@ void testConcurrentCopies(const std::filesystem::path& tool) { } } +void testReservationError(const std::filesystem::path& tool) { + const char* directory = std::getenv("APS5_FRAME_CAPTURE"); + check(directory != nullptr && *directory != '\0', "APS5_FRAME_CAPTURE is not set"); + const std::filesystem::path capture(directory); + std::uintptr_t base = 0; + std::size_t size = 0; + GuestArena::GuestArenaRange_nid_postfix(&base, &size); + AgcDriver::FrameCaptureWriter writer(capture); + AgcDriver::FrameCall output; + output.kind = FrameCallKind::RegisterVideoOutput; + output.values = {Handle}; + writer.WriteCall(output); + AgcDriver::FrameCall reservation; + reservation.kind = FrameCallKind::ReserveEopFlip; + reservation.values = {Handle, static_cast(-1), 1, 0, AgcDriver::EopFlipTokenBase}; + writer.WriteCall(reservation); + AgcDriver::FrameCall start; + start.kind = FrameCallKind::FrameStart; + writer.WriteCall(start); + output.kind = FrameCallKind::UnregisterVideoOutput; + writer.WriteCall(output); + output.kind = FrameCallKind::RegisterVideoOutput; + writer.WriteCall(output); + reservation.values[3] = 1; + writer.WriteCall(reservation); + writer.Finish("{\"arena_base\":" + std::to_string(base) + "}"); + const auto result = capture / "reservation error.json"; + const auto log = capture / "reservation error.log"; + auto command = "\"" + tool.string() + "\" \"" + capture.string() + "\" --repeat 2 --strict > \"" + result.string() + "\" 2> \"" + log.string() + "\""; +#ifdef _WIN32 + command = "\"" + command + "\""; +#endif + check(std::system(command.c_str()) != 0, "the replay accepted an incompatible reservation on the second pass"); + const auto errors = readFile(log); + check(std::string(reinterpret_cast(errors.data()), errors.size()).find("EOP flip token is reserved for a different flip") != std::string::npos, "the repeated replay did not report the incompatible reservation"); + const auto report = readFile(result); + check(std::string(reinterpret_cast(report.data()), report.size()).find("\"p50\"") == std::string::npos, "a failed repeated reservation produced a successful replay report"); +} + } int main(int argc, char** argv) { - if (argc != 2 && !(argc == 3 && std::string(argv[2]) == "concurrent")) { - std::fprintf(stderr, "usage: agc_driver_frame_replay_tests [concurrent]\n"); + if (argc != 2 && !(argc == 3 && (std::string(argv[2]) == "concurrent" || std::string(argv[2]) == "pipelines" || std::string(argv[2]) == "reservation_error"))) { + std::fprintf(stderr, "usage: agc_driver_frame_replay_tests [concurrent|pipelines|reservation_error]\n"); return 2; } try { - if (argc == 3) testConcurrentCopies(argv[1]); - else testCaptureAndReplay(argv[1]); + if (argc == 3 && std::string(argv[2]) == "concurrent") testConcurrentCopies(argv[1]); + else if (argc == 3 && std::string(argv[2]) == "reservation_error") testReservationError(argv[1]); + else testCaptureAndReplay(argv[1], argc == 3); } catch (const std::exception& error) { std::fprintf(stderr, "FAILED: %s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp index 29c816ebcdf4..d210fcfad5e2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp @@ -2,8 +2,10 @@ #include "../tools/FrameReplay.cpp" #undef main #include "prx/libSceAgcDriver/Execution/include/FrameRecorder.hpp" +#include "prx/libSceAgcDriver/Execution/include/Presentation.hpp" #include "prx/libc/include/GuestWriteWatch.hpp" #include +#include namespace { @@ -182,6 +184,94 @@ void testEopTokens() { AgcDriverUnregisterVideoOutput_nid_postfix(7, output); } +class ReservationOutput final : public AgcDriver::IVideoOutput { +public: + explicit ReservationOutput(Frames& frames) : frames(frames) {} + std::shared_ptr Reserve(const AgcDriver::FlipInfo&) override { + ++reservations; + if (failure) std::rethrow_exception(failure); + return available ? std::make_shared(frames) : nullptr; + } + void Fail(std::exception_ptr error) noexcept override { frames.Failed(error); } + std::exception_ptr failure; + bool available = true; + std::size_t reservations = 0; + +private: + Frames& frames; +}; + +FrameCall reservationCall(std::uint32_t token) { + FrameCall call; + call.kind = FrameCallKind::ReserveEopFlip; + call.values = {7, static_cast(-1), 1, 99, token}; + return call; +} + +template +void expectReservationFailure(Replay& replay, const FrameCall& call) { + const auto kept = replay.keptReservations.load(); + bool failed = false; + try { replay.runWindow(std::span(&call, 1)); } + catch (const TException&) { failed = true; } + check(failed, "the replay accepted a failed EOP reservation"); + check(replay.keptReservations.load() == kept, "a failed EOP reservation was counted as kept"); +} + +void testRepeatedReservations(const std::filesystem::path& directory, bool driverFailure) { + FrameCaptureWriter writer(directory); + writer.Finish("{}"); + const FrameCaptureReader reader(directory); + Options options; + options.pace = false; + options.release = false; + Replay replay{options, reader}; + auto output = std::make_shared(replay.frames); + AgcDriverRegisterVideoOutput_nid_postfix(7, output); + AgcDriverRegisterVideoOutput_nid_postfix(8, output); + const auto call = reservationCall(AgcDriver::EopFlipTokenBase); + replay.execute(call); + expectReservationFailure(replay, call); + replay.pass = 1; + if (driverFailure) { + AgcDriverReportFailure_nid_postfix(std::make_exception_ptr(std::runtime_error("reservation driver failure"))); + expectReservationFailure(replay, call); + bool failed = false; + try { AgcDriverShutdown_nid_postfix(); } + catch (const std::runtime_error& error) { failed = std::string_view(error.what()) == "reservation driver failure"; } + check(failed, "the injected driver failure was not propagated at shutdown"); + return; + } + replay.runWindow(std::span(&call, 1)); + check(replay.keptReservations.load() == 1 && output->reservations == 1, "a compatible pending reservation was not kept without allocating again"); + for (std::size_t field = 0; field < 4; ++field) { + auto different = call; + ++different.values[field]; + expectReservationFailure(replay, different); + } + auto missing = call; + missing.values[0] = 9; + expectReservationFailure(replay, missing); + expectReservationFailure(replay, reservationCall(AgcDriver::EopFlipTokenBase - 1)); + expectReservationFailure(replay, reservationCall(AgcDriver::EopFlipTokenLimit + 1)); + const auto fresh = reservationCall(AgcDriver::EopFlipTokenBase + 1); + output->failure = std::make_exception_ptr(std::bad_alloc{}); + expectReservationFailure(replay, fresh); + output->failure = std::make_exception_ptr(std::runtime_error("reservation output failure")); + expectReservationFailure(replay, fresh); + output->failure = nullptr; + output->available = false; + expectReservationFailure(replay, fresh); + output->available = true; + AgcDriverUnregisterVideoOutput_nid_postfix(7, output); + auto replacement = std::make_shared(replay.frames); + AgcDriverRegisterVideoOutput_nid_postfix(7, replacement); + replay.runWindow(std::span(&call, 1)); + check(replay.keptReservations.load() == 1 && replacement->reservations == 1, "a closed output's token was kept after reopening"); + AgcDriverUnregisterVideoOutput_nid_postfix(7, replacement); + AgcDriverUnregisterVideoOutput_nid_postfix(8, output); +} + class WaitingOutput final : public AgcDriver::IVideoOutput { public: WaitingOutput(Frames& frames, const std::uint32_t* label) : frames(frames), label(label) {} @@ -302,10 +392,11 @@ int main(int argc, char** argv) { else if (mode == "host_neighbors") testHostNeighbors(directory); else if (mode == "unwatched") testUnwatched(); else if (mode == "tokens") testEopTokens(); + else if (mode == "reservations" || mode == "reservation_failure") testRepeatedReservations(directory, mode == "reservation_failure"); else if (mode == "blocking") testBlockingSubmission(directory); else if (mode == "shader" || mode == "shader_host") testShaderCode(directory, mode == "shader_host"); else throw std::runtime_error("unknown test"); - AgcDriverShutdown_nid_postfix(); + if (mode != "reservation_failure") AgcDriverShutdown_nid_postfix(); std::puts("frame replay state test passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 2235f258d6da..1e40481245ea 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -454,7 +454,7 @@ struct Replay { std::uint32_t reserved = 0; try { reserved = AgcDriverReserveEopFlipAt_nid_postfix(flipInfo(call), token); - } catch (const std::exception&) { + } catch (const AgcDriver::EopFlipAlreadyReserved&) { if (pass == 0) throw; ++keptReservations; return; @@ -899,10 +899,11 @@ int main(int argc, char** argv) { if (options.directory.empty()) return usage(); #ifdef _WIN32 _putenv_s("APS5_FRAME_CAPTURE", ""); - if (!options.coldPipelines) _putenv_s("APS5_HEADLESS_PIPELINE_CACHE", "1"); + _putenv_s("APS5_HEADLESS_PIPELINE_CACHE", options.coldPipelines ? "" : "1"); #else unsetenv("APS5_FRAME_CAPTURE"); - if (!options.coldPipelines) setenv("APS5_HEADLESS_PIPELINE_CACHE", "1", 1); + if (options.coldPipelines) unsetenv("APS5_HEADLESS_PIPELINE_CACHE"); + else setenv("APS5_HEADLESS_PIPELINE_CACHE", "1", 1); #endif int code = 2; try { diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 79b0ee715a7a..36683d55af18 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -172,7 +172,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_GPU_WAIT_TIMEOUT_MS` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp` | 1000 | milliseconds a GPU wait may run before a warning is printed | | `APS5_GROUP_CAPTURE_ALL_QUEUES` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp` | | resolves group-ahead capture on queue 0 as well | | `APS5_HANDOFF_SPIN_US` | integer | `core/libs/prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp` | | microseconds a thread spins for a handed-off draw's work or for the GPU mutex before blocking, clamped to 0..10000; unset spins 150 us when the process has more than 8 usable processors and not at all otherwise | -| `APS5_HEADLESS_PIPELINE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | loads the pipeline cache file on a device without a window too; `agc_frame_replay` sets it unless `--cold-pipelines` is given | +| `APS5_HEADLESS_PIPELINE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | loads the pipeline cache file on a device without a window too; `agc_frame_replay` sets it by default and clears it with `--cold-pipelines`, including inherited values | | `APS5_HEAP_MIRROR_MIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | 24576 | budget in MiB for the heap mirrors held for registered allocations | | `APS5_HOOK_FLUSH_CPU_BLOCKS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | set to 0 flushes pending images whose every 64 KiB block the CPU wrote, instead of skipping them | | `APS5_HOOK_FULL_SYNC` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | makes the flush hook's recorded-store wait a full Sync under the GpuMutex | From 0c8232d533984214645be8420ab21a5545ef9c1c Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 21 Oct 2026 17:54:04 +0100 Subject: [PATCH 05/12] feat(agc): trace the serials the driver accepts and completes under APS5_TRACE_GPU With APS5_TRACE_GPU the driver now prints the serial it gives each submission, CPU flip and suspend point when it accepts it, with the queue, size and command address, and each completion as it is marked, including queue 0 completions deferred to the draw pipeline. A replay that stops completing can then be followed from the call that made a submission to the serial that never completed. Co-Authored-By: Claude Opus 5.5 --- .../Execution/src/Driver/Queues/QueueWorker.cpp | 4 ++++ .../Execution/src/Driver/Queues/Submission.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index b6a5b5689cd7..24717b22bf7c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -50,6 +50,8 @@ FrameTiming* Driver::includeTimingSubmission(const Submission& submission, bool } void Driver::markCompleted(std::uint64_t serial) { + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f completed serial=%llu (in order up to %llu)\n", TraceMs(), static_cast(serial), static_cast(completed)); completedOutOfOrder.insert(serial); while (!completedOutOfOrder.empty() && *completedOutOfOrder.begin() == completed + 1) { completed = *completedOutOfOrder.begin(); @@ -70,6 +72,8 @@ void Driver::deferCompletion(const Submission& submission) { queue0Uncommitted.push_back(received); } completionDeferred() = true; + static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; + if (traceGpu) std::fprintf(stderr, "[gpu] %.1f deferred completion serial=%llu to the draw pipeline\n", TraceMs(), static_cast(submission.serial)); DrawPipeline::Queue0().Enqueue([this, serial = submission.serial, received, labelWrites = submission.labelWrites, progress = submission.progress] { completeSubmission(serial, received, labelWrites, progress); }, {}); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index eeaea3e12654..1d6388fe3c43 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -445,6 +445,8 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue, bool cpuFlip) { } submission.shaders = shaders; submission.serial = accepted + 1; + if (trace) std::fprintf(stderr, "[gpu] %.1f accepted serial=%llu queue=0x%x dwords=%zu at %p%s\n", TraceMs(), static_cast(submission.serial), queue, submission.commands.size(), + static_cast(descriptor.addr), cpuFlip ? " (cpu flip)" : ""); submission.received = ++eventSerial; const auto now = std::chrono::steady_clock::now(); @@ -501,6 +503,8 @@ void Driver::SuspendPoint() { Submission boundary{}; boundary.serial = accepted + 1; boundary.suspend = true; + static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr; + if (trace) std::fprintf(stderr, "[gpu] %.1f accepted serial=%llu suspend point\n", TraceMs(), static_cast(boundary.serial)); boundary.queue = 0; boundary.receivedAt = receivedAt; From c793788954d38ec2c70ec8a09820ca16e077c019 Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 21 Oct 2026 17:55:32 +0100 Subject: [PATCH 06/12] feat(agc): list a capture's shaders and submissions and trace a replay's serials Finding why a replay of a title stops, or which shaders a frame uses, meant running the whole capture. agc_frame_replay gains three modes: - --shaders replays nothing and maps no guest memory: it reads the snapshot, applies the captured writes in call order and prints one JSON line per shader registration with its code address, size, type, code hash and how many earlier registrations had the same code; --shader-code also writes each distinct code to a file - --submits lists the first n submissions and flips of the captured frames with their queue and serials, the packets of each and the fields of its waits and releases - --trace-serials prints each submission, flip and suspend point with its call ordinal and command address as it is issued, to match the serials APS5_TRACE_GPU prints agc_driver_frame_replay_shader and agc_driver_frame_replay_shader_host check that the listing reads the code the capture holds at the registration. Co-Authored-By: Claude Opus 5.5 --- .../tests/FrameReplayState.cpp | 15 + .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 277 +++++++++++++++++- docs/dev/BUILD.md | 2 +- 3 files changed, 287 insertions(+), 7 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp index d210fcfad5e2..e94b91c580b2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp @@ -377,6 +377,21 @@ void testShaderCode(const std::filesystem::path& directory, bool host) { } } check(registered && captured == expected, "the complete separate shader code was not captured before registration"); + CapturedMemory memory(reader); + std::map> groups; + for (const auto& chunk : reader.Chunks()) { + if (chunk.kind == FrameMemoryKind::Delta) groups[chunk.group].push_back(&chunk); + } + std::optional> listed; + for (const auto& call : reader.Calls()) { + if (call.kind == FrameCallKind::RegisterShader) { + listed = memory.Read(address(code), CodeBytes); + break; + } + if (call.kind != FrameCallKind::Memory) continue; + if (const auto found = groups.find(call.values.at(0)); found != groups.end()) memory.Apply(found->second); + } + check(listed && *listed == expected, "the offline shader listing read other code than the capture holds at the registration"); } } diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 1e40481245ea..04e9ca9c92ca 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -73,6 +73,10 @@ struct Options { bool release = true; bool synthesize = true; std::size_t repeat = 1; + bool shaders = false; + std::filesystem::path shaderCode; + std::size_t submits = 0; + bool traceSerials = false; bool coldPipelines = false; }; @@ -441,13 +445,15 @@ struct Replay { return; case FrameCallKind::Submit: { const Packet packet{reinterpret_cast(static_cast(call.address)), static_cast(call.size), static_cast(call.flags), {}}; - if (call.queue == 0) sceAgcDriverSubmitDcb(&packet); - else sceAgcDriverSubmitAcb(call.queue, &packet); + traced(call, [&] { + if (call.queue == 0) sceAgcDriverSubmitDcb(&packet); + else sceAgcDriverSubmitAcb(call.queue, &packet); + }); if (++submits % 25 == 0) report(); return; } case FrameCallKind::Flip: - AgcDriverSubmitFlip_nid_postfix(flipInfo(call)); + traced(call, [&] { AgcDriverSubmitFlip_nid_postfix(flipInfo(call)); }); return; case FrameCallKind::ReserveEopFlip: { const auto token = static_cast(call.values.at(4)); @@ -463,7 +469,7 @@ struct Replay { return; } case FrameCallKind::Suspend: - AgcDriverSuspendPoint_nid_postfix(); + traced(call, [] { AgcDriverSuspendPoint_nid_postfix(); }); return; case FrameCallKind::Memory: { std::lock_guard lock(memoryMutex); @@ -654,6 +660,15 @@ struct Replay { } } + template + void traced(const FrameCall& call, TFunction&& function) { + if (options.traceSerials) { + std::fprintf(stderr, "[frame-replay] call %llu (%s, thread %llu, queue 0x%x, %llu dwords at 0x%llx)\n", static_cast(call.ordinal), std::string(AgcDriver::FrameCallName(call.kind)).c_str(), + static_cast(call.thread), call.queue, static_cast(call.size), static_cast(call.address)); + } + function(); + } + template void registration(TFunction&& function) { try { @@ -687,6 +702,15 @@ int usage() { " --repeat replays the frames n times in this process, restoring the snapshot before each pass;\n" " with n > 1 the frame times come from passes 2 to n, after the first warmed the caches\n" " --cold-pipelines starts with an empty Vulkan pipeline cache instead of the title's pipeline cache file\n" + " --shaders replays nothing and maps no guest memory; lists each shader registration with its code\n" + " address, size and code hash as the snapshot and the captured writes before it hold them,\n" + " and how many registrations share that code\n" + " --shader-code as --shaders, and writes each distinct code as /type-.bin\n" + " --trace-serials prints each submission, flip and suspend point with its call ordinal and command\n" + " address as it is issued; with APS5_TRACE_GPU the driver prints the serial it accepts\n" + " for each address\n" + " --submits replays nothing; lists the first n submissions and flips of the captured frames with\n" + " their queue and serials, the packets of each and the fields of its waits and releases\n" " --dump writes guest memory to a file after the replay (hexadecimal or decimal)\n" " --frames writes each recorded and replayed display buffer as reference-.bmp and replay-.bmp\n"); return 2; @@ -767,6 +791,236 @@ FrameCheck compareFrames(const FrameCaptureReader& reader, const Options& option return check; } +bool readable(std::uint64_t address, std::uint64_t bytes) { +#ifdef _WIN32 + for (auto at = address; at < address + bytes;) { + MEMORY_BASIC_INFORMATION info{}; + if (VirtualQuery(reinterpret_cast(static_cast(at)), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT) return false; + const auto access = info.Protect & 0xffu; + if ((info.Protect & PAGE_GUARD) != 0 || access == PAGE_NOACCESS || access == PAGE_EXECUTE) return false; + at = reinterpret_cast(info.BaseAddress) + info.RegionSize; + } + return true; +#else + static_cast(address); + static_cast(bytes); + return false; +#endif +} + +std::uint64_t contentHash(const std::byte* bytes, std::size_t size) { + std::uint64_t hash = 0xcbf29ce484222325ull; + for (std::size_t index = 0; index < size; ++index) { + hash ^= std::to_integer(bytes[index]); + hash *= 0x100000001b3ull; + } + return hash; +} + +class CapturedMemory { +public: + explicit CapturedMemory(const FrameCaptureReader& reader) : reader(reader) { + for (const auto& chunk : reader.Chunks()) { + if (chunk.kind == FrameMemoryKind::Snapshot) snapshots.push_back(&chunk); + else if (chunk.kind == FrameMemoryKind::Range) ranges.push_back(&chunk); + } + const auto byAddress = [](const FrameMemoryChunk* a, const FrameMemoryChunk* b) { return a->address < b->address; }; + std::sort(snapshots.begin(), snapshots.end(), byAddress); + std::sort(ranges.begin(), ranges.end(), byAddress); + } + + std::optional> Read(std::uint64_t address, std::uint64_t bytes) { + std::vector result(static_cast(bytes)); + for (auto at = address; at < address + bytes;) { + const auto* range = covering(ranges, at); + if (range == nullptr) return std::nullopt; + const auto* snapshot = covering(snapshots, at); + auto end = std::min(address + bytes, range->address + range->bytes); + if (snapshot != nullptr) { + end = std::min(end, snapshot->address + snapshot->bytes); + const auto& data = load(*snapshot); + std::memcpy(result.data() + (at - address), data.data() + (at - snapshot->address), static_cast(end - at)); + } else { + const auto next = std::upper_bound(snapshots.begin(), snapshots.end(), at, [](std::uint64_t value, const FrameMemoryChunk* chunk) { return value < chunk->address; }); + if (next != snapshots.end()) end = std::min(end, (*next)->address); + } + at = end; + } + for (const auto* delta : deltas) { + const auto begin = std::max(address, delta->address); + const auto end = std::min(address + bytes, delta->address + delta->bytes); + if (begin >= end) continue; + const auto data = reader.Read(*delta); + std::memcpy(result.data() + (begin - address), data.data() + (begin - delta->address), static_cast(end - begin)); + } + return result; + } + + void Apply(std::span chunks) { + deltas.insert(deltas.end(), chunks.begin(), chunks.end()); + } + +private: + static const FrameMemoryChunk* covering(const std::vector& chunks, std::uint64_t address) { + const auto next = std::upper_bound(chunks.begin(), chunks.end(), address, [](std::uint64_t value, const FrameMemoryChunk* chunk) { return value < chunk->address; }); + if (next == chunks.begin()) return nullptr; + const auto* chunk = *std::prev(next); + return address < chunk->address + chunk->bytes ? chunk : nullptr; + } + + const std::vector& load(const FrameMemoryChunk& chunk) { + if (cached != &chunk) { + data = reader.Read(chunk); + cached = &chunk; + } + return data; + } + + const FrameCaptureReader& reader; + std::vector snapshots; + std::vector ranges; + std::vector deltas; + const FrameMemoryChunk* cached = nullptr; + std::vector data; +}; + +int listShaders(const Options& options) { + const FrameCaptureReader reader(options.directory); + CapturedMemory memory(reader); + if (!options.shaderCode.empty()) std::filesystem::create_directories(options.shaderCode); + std::map> groups; + for (const auto& chunk : reader.Chunks()) { + if (chunk.kind == FrameMemoryKind::Delta) groups[chunk.group].push_back(&chunk); + } + const auto& calls = reader.Calls(); + const auto frameStart = std::find_if(calls.begin(), calls.end(), [](const FrameCall& call) { return call.kind == FrameCallKind::FrameStart; }); + std::map copies; + std::size_t listed = 0, inWindow = 0, unreadable = 0; + for (auto call = calls.begin(); call != calls.end(); ++call) { + if (call->kind == FrameCallKind::Memory) { + if (const auto found = groups.find(call->values.at(0)); found != groups.end()) memory.Apply(found->second); + continue; + } + if (call->kind != FrameCallKind::RegisterShader) continue; + const auto header = memory.Read(call->address, sizeof(Shader)); + if (!header) { + ++unreadable; + continue; + } + Shader fields{}; + std::memcpy(&fields, header->data(), sizeof(Shader)); + const auto code = static_cast(reinterpret_cast(const_cast(fields.code))); + const auto bytes = fields.shader_size != 0 ? memory.Read(code, fields.shader_size) : std::nullopt; + if (!bytes) { + ++unreadable; + continue; + } + const auto hash = contentHash(bytes->data(), bytes->size()); + const auto copy = ++copies[hash]; + if (copy == 1 && !options.shaderCode.empty()) { + char name[64]; + std::snprintf(name, sizeof(name), "type%u-%016llx.bin", static_cast(fields.type), static_cast(hash)); + std::ofstream file(options.shaderCode / name, std::ios::binary | std::ios::trunc); + file.write(reinterpret_cast(bytes->data()), static_cast(bytes->size())); + } + const bool window = frameStart != calls.end() && call > frameStart; + ++listed; + inWindow += window ? 1 : 0; + std::printf("{\"ordinal\":%llu,\"in_window\":%s,\"header\":%llu,\"code\":%llu,\"bytes\":%u,\"type\":%u,\"hash\":\"%016llx\",\"copy\":%zu}\n", static_cast(call->ordinal), window ? "true" : "false", + static_cast(call->address), static_cast(code), fields.shader_size, static_cast(fields.type), static_cast(hash), copy); + } + std::printf("{\"registrations\":%zu,\"in_window\":%zu,\"distinct_code\":%zu,\"unreadable\":%zu}\n", listed, inWindow, copies.size(), unreadable); + std::fflush(stdout); + return 0; +} + +std::string_view packetName(std::uint32_t opcode) { + switch (opcode) { + case 0x10: return "NOP"; + case 0x15: return "DISPATCH_DIRECT"; + case 0x16: return "DISPATCH_INDIRECT"; + case 0x27: return "DRAW_INDEX_2"; + case 0x2d: return "DRAW_INDEX_AUTO"; + case 0x35: return "DRAW_INDEX_OFFSET_2"; + case 0x37: return "WRITE_DATA"; + case 0x3c: return "WAIT_REG_MEM"; + case 0x3f: return "INDIRECT_BUFFER"; + case 0x46: return "EVENT_WRITE"; + case 0x49: return "RELEASE_MEM"; + case 0x50: return "DMA_DATA"; + case 0x58: return "ACQUIRE_MEM"; + case 0x69: return "SET_CONTEXT_REG"; + case 0x76: return "SET_SH_REG"; + case 0x79: return "SET_UCONFIG_REG"; + default: return {}; + } +} + +void describeSubmission(const FrameCall& call) { + const auto* words = reinterpret_cast(static_cast(call.address)); + if (!readable(call.address, call.size * sizeof(std::uint32_t))) { + std::printf(" (command buffer not readable)\n"); + return; + } + std::map counts; + for (std::size_t at = 0; at < call.size;) { + const auto header = words[at]; + const auto type = header >> 30u; + const auto length = type == 3u ? static_cast((header >> 16u) & 0x3fffu) + 2u : 1u; + if (type == 3u) { + const auto opcode = (header >> 8u) & 0xffu; + const auto name = packetName(opcode); + char label[24]; + std::snprintf(label, sizeof(label), "0x%02x", opcode); + const std::string key = name.empty() ? std::string(label) : std::string(name); + const auto word = [&](std::size_t index) { return at + index < call.size ? words[at + index] : 0u; }; + if (opcode == 0x3c) { + std::printf(" @%zu WAIT_REG_MEM function %u space %u address 0x%llx reference 0x%x mask 0x%x\n", at, word(1) & 7u, (word(1) >> 4u) & 3u, + static_cast(word(2)) | (static_cast(word(3)) << 32u), word(4), word(5)); + } else if (opcode == 0x49 || opcode == 0x37 || opcode == 0x46) { + std::printf(" @%zu %s", at, key.c_str()); + for (std::size_t index = 1; index < length && index < 8; ++index) std::printf(" %08x", word(index)); + std::printf("\n"); + } + ++counts[key]; + } + at += length; + } + std::printf(" packets:"); + for (const auto& [name, count] : counts) std::printf(" %s x%zu", name.c_str(), count); + std::printf("\n"); +} + +int listSubmits(const Options& options) { + const FrameCaptureReader reader(options.directory); + const auto skipped = restoreMemory(reader); + std::map> groups; + for (const auto& chunk : reader.Chunks()) { + if (chunk.kind == FrameMemoryKind::Delta) groups[chunk.group].push_back(&chunk); + } + const auto& calls = reader.Calls(); + const auto frameStart = std::find_if(calls.begin(), calls.end(), [](const FrameCall& call) { return call.kind == FrameCallKind::FrameStart; }); + std::size_t listed = 0; + for (auto call = frameStart; call != calls.end() && listed < options.submits; ++call) { + if (call->kind == FrameCallKind::Memory) { + if (const auto found = groups.find(call->values.at(0)); found != groups.end()) { + for (const auto* chunk : found->second) { + if (!skipped.Overlaps(*chunk)) restore(reader, *chunk); + } + } + continue; + } + if (call->kind != FrameCallKind::Submit && call->kind != FrameCallKind::Flip && call->kind != FrameCallKind::ReserveEopFlip) continue; + ++listed; + std::printf("#%zu ordinal %llu %s thread %llu queue 0x%x serials %llu/%llu address 0x%llx dwords %llu\n", listed, static_cast(call->ordinal), std::string(AgcDriver::FrameCallName(call->kind)).c_str(), + static_cast(call->thread), call->queue, static_cast(call->completed), static_cast(call->accepted), + static_cast(call->address), static_cast(call->size)); + if (call->kind == FrameCallKind::Submit) describeSubmission(*call); + } + std::fflush(stdout); + return 0; +} + struct Pass { std::size_t firstFrame = 0; std::size_t frames = 0; @@ -890,6 +1144,16 @@ int main(int argc, char** argv) { else if (argument == "--release-after-ms" && index + 1 < argc) options.releaseAfter = std::chrono::milliseconds(std::strtoll(argv[++index], nullptr, 10)); else if (argument == "--strict") options.strict = true; else if (argument == "--repeat" && index + 1 < argc) options.repeat = std::max(1, std::strtoull(argv[++index], nullptr, 10)); + else if (argument == "--shaders") options.shaders = true; + else if (argument == "--shader-code" && index + 1 < argc) { + options.shaders = true; + options.shaderCode = argv[++index]; + } + else if (argument == "--submits" && index + 1 < argc) { + char* end = nullptr; + options.submits = std::strtoull(argv[++index], &end, 10); + if (options.submits == 0 || end == argv[index] || *end != '\0') return usage(); + } else if (argument == "--trace-serials") options.traceSerials = true; else if (argument == "--cold-pipelines") options.coldPipelines = true; else if (argument == "--dump" && index + 1 < argc) options.dumps.push_back(parseDump(argv[++index])); else if (argument == "--frames" && index + 1 < argc) options.frames = argv[++index]; @@ -907,8 +1171,9 @@ int main(int argc, char** argv) { #endif int code = 2; try { - code = run(options); - if (code == 0) { + const bool listing = options.shaders || options.submits != 0; + code = options.shaders ? listShaders(options) : options.submits != 0 ? listSubmits(options) : run(options); + if (code == 0 && !listing) { AgcDriverShutdown_nid_postfix(); return 0; } diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 5b9e00b8a6fb..bd57560d77a6 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -228,7 +228,7 @@ On Windows use `agc_pm4_replay.exe` and prepend `build/core/libs/libs/unpatched` On Windows and Linux, set `APS5_FRAME_CAPTURE` to a directory to record a few frames of a title at the AGC driver boundary, then replay them with `agc_frame_replay` through any build of the driver with no title running. From the start of the process the driver keeps every shader registration and ABI resolution, video output registration and EOP flip reservation. At the frame boundary picked by `APS5_FRAME_CAPTURE_AT_FLIP` (default 1), or by the appearance of the file `APS5_FRAME_CAPTURE_TRIGGER` names (a number in it replaces the frame count), it waits for the driver to go idle and writes every committed, readable allocation of the guest allocation registry, the title's image included, as zstd chunks without their all-zero 64 KiB blocks. A frame ends at a CPU flip or at a submission that flips (a flip packet, or the EOP release of a reserved flip token); the reservation itself only records its token. For the next `APS5_FRAME_CAPTURE_FRAMES` (10) frames it records each submission (queue, command buffer address and size), flip and suspend point with the thread that made it and the accepted and completed serials, each preceded by the guest pages the write tracker saw the CPU write since the previous call that handed work to the GPU (`GuestMemory::CollectWrittenBlocks`, per 4 KiB where the tracker kept page stamps), read once the submissions already accepted have completed (a recorded submission, flip or suspend point reaches the driver before the recorder takes its next call, so that wait covers every call recorded before the delta); an allocation the registry gained after the snapshot is recorded there as a new range with its nonzero 64 KiB blocks. A submission ends a frame only when it flips itself, not when another thread's submission flipped while it was being submitted. Shader registrations and ABI resolutions only collect the blocks of the shader header and the arrays it points to, and the pages of the title's image only when their content changed. At the end it stores each display buffer the title presented, decoded, as a reference image. Without the guest write watch (`APS5_NO_WRITE_WATCH`, or no write watch on the host) the capture throws when it starts, since it could not record CPU stores. A capture is `calls.bin`, `memory.bin` and `capture.json` (commit, switches, sizes, warnings), and the recorder stops with a warning when its volume has less than 4 GiB free. A 20-frame capture in the Demon's Souls Nexus is 5.3 GiB of snapshot and 11.5 GiB of deltas before compression, 3.9 GiB on disk. -`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--cold-pipelines] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The replay's device has no window, but it loads the title's Vulkan pipeline cache file (`vk-pipelines-*.bin` in the shader cache directory) as the windowed device does, so pipelines the title already compiled are not compiled again (`--cold-pipelines` starts with an empty one; the driver switch is `APS5_HEADLESS_PIPELINE_CACHE`). The title replayed its pipeline journal at boot, long before the captured frames, so after the registrations the replay waits for the driver's journal replay to finish before the first pass (`journal_s` in the JSON line). The first pass still compiles every shader and pipeline the frames use that neither cache holds, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. +`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--cold-pipelines] [--shaders] [--shader-code ] [--submits ] [--trace-serials] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The replay's device has no window, but it loads the title's Vulkan pipeline cache file (`vk-pipelines-*.bin` in the shader cache directory) as the windowed device does, so pipelines the title already compiled are not compiled again (`--cold-pipelines` starts with an empty one; the driver switch is `APS5_HEADLESS_PIPELINE_CACHE`). The title replayed its pipeline journal at boot, long before the captured frames, so after the registrations the replay waits for the driver's journal replay to finish before the first pass (`journal_s` in the JSON line). The first pass still compiles every shader and pipeline the frames use that neither cache holds, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. `--shaders` replays nothing and maps no guest memory: it reads the snapshot, applies the captured writes in call order and prints one JSON line per shader registration (code address, size, type, code hash and how many earlier registrations had the same code), then a summary; `--shader-code ` also writes each distinct code as `/type-.bin`. `--submits ` replays nothing either and lists the first n submissions and flips of the captured frames with their queue and serials, the packets of each and the fields of its waits and releases. `--trace-serials` prints each submission, flip and suspend point with its call ordinal and command address as it is issued; with `APS5_TRACE_GPU` the driver prints the serial it accepts for each address and each completion, so a replay that stops completing can be followed to the call that made the submission. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. ```sh cmake --build build --target agc_frame_replay From e6b0a9b4df15a082a2d07f9f466d9853018e86bc Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 27 Oct 2026 17:31:16 +0000 Subject: [PATCH 07/12] fix(agc): replay listing review fixes: non-Windows packet decoding, payload words, output errors --submits decodes the captured ranges on Linux and macOS as well, walks packets with Pm4::PacketWords and FillerPacket so type-0 payload dwords are not taken for headers, and the listing checks open, write, flush and close of its output file, naming the path and exiting nonzero on failure. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 18 ++- .../tests/FrameReplayState.cpp | 85 ++++++++++++++ .../tests/FrameReplayTool.cmake | 105 ++++++++++++++++++ .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 87 ++++++++------- docs/dev/BUILD.md | 2 +- 5 files changed, 252 insertions(+), 45 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index a1715c5d1d10..4c17b837e9c3 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -3598,7 +3598,19 @@ if(BUILD_TESTING) target_link_libraries(agc_driver_frame_replay_state_tests PRIVATE libSceAgcDriver libkernel libc libzstd_static) set_target_properties(agc_driver_frame_replay_state_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_frame_replay_state_tests) - set(agcFrameReplayModes order tokens blocking reservations reservation_failure) + add_dependencies(agc_driver_frame_replay_state_tests agc_frame_replay) + foreach(mode IN ITEMS submits shader_output) + add_test(NAME agc_driver_frame_replay_${mode} COMMAND ${CMAKE_COMMAND} + "-DTOOL=$" "-DWRITER=$" + "-DDIRECTORY=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-${mode}" "-DMODE=${mode}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/FrameReplayTool.cmake") + set_tests_properties(agc_driver_frame_replay_${mode} PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(agc_driver_frame_replay_${mode} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() + set(agcFrameReplayModes order tokens blocking reservations reservation_failure environment) if(WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND agcFrameReplayModes zero shader shader_host) endif() @@ -3620,6 +3632,10 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_frame_replay_${mode} PROPERTIES ENVIRONMENT "APS5_NO_WRITE_WATCH=1;APS5_FRAME_CAPTURE=${capture};APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=1") endif() + if(mode STREQUAL "environment") + set_tests_properties(agc_driver_frame_replay_${mode} PROPERTIES ENVIRONMENT + "APS5_HEADLESS_PIPELINE_CACHE=1;APS5_FRAME_CAPTURE=${capture}") + endif() if(WIN32) set_tests_properties(agc_driver_frame_replay_${mode} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp index e94b91c580b2..0b08d26e247a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplayState.cpp @@ -394,6 +394,86 @@ void testShaderCode(const std::filesystem::path& directory, bool host) { check(listed && *listed == expected, "the offline shader listing read other code than the capture holds at the registration"); } +void writeInspectionCapture(const std::filesystem::path& directory) { + constexpr std::uint64_t Commands = 0x1000000001ull; + constexpr std::uint64_t Header = 0x2000000000ull; + constexpr std::uint64_t Code = 0x2000010000ull; + const std::array commands{ + 0x00010000u, 0xc0053c00u, 0xc0064900u, 0x80000000u, 0xffff1000u, + 0xc0053c00u, 0x13u, 0x12345678u, 0x9abcdef0u, 0x10203040u, 0xf0f0f0f0u, 4u, + 0xc0064900u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, + 0xc0033700u, 0x500u, 0xdeadbeefu, 0x12u, 0x1337u, + 0x00030000u, 0xc0053c00u}; + std::array code{0, 1, 2, 3, 4, 5, 6, 7, 255, 128, 13, 10, 39, 36, 38, 0}; + Shader shader{}; + shader.code = reinterpret_cast(static_cast(Code)); + shader.shader_size = code.size(); + FrameCaptureWriter writer(directory); + writer.WriteRange(Commands, 6, FrameRangeKind::Host, ReadWrite, FrameRangeAccess::Readable); + writer.WriteRange(Commands + 6, sizeof(commands) - 6, FrameRangeKind::Host, ReadWrite, FrameRangeAccess::Readable); + writer.WriteRange(Header, sizeof(shader), FrameRangeKind::Host, ReadWrite, FrameRangeAccess::Readable); + writer.WriteRange(Code, code.size(), FrameRangeKind::Host, ReadWrite, FrameRangeAccess::Readable); + writer.WriteSnapshot(Header, std::as_bytes(std::span(&shader, 1))); + writer.WriteSnapshot(Code, std::as_bytes(std::span(code))); + writer.WriteDelta(1, 0, Commands, std::as_bytes(std::span(commands))); + const std::uint32_t reference = 0x55667788u; + writer.WriteDelta(2, 0, Commands + 9 * sizeof(std::uint32_t), std::as_bytes(std::span(&reference, 1))); + code[0] = 0x42; + writer.WriteDelta(3, 0, Code, std::as_bytes(std::span(code))); + FrameCall call; + const auto write = [&](FrameCallKind kind, std::uint64_t location, std::uint64_t size, std::vector values = {}) { + call.kind = kind; + ++call.ordinal; + call.address = location; + call.size = size; + call.values = std::move(values); + call.thread = 7; + call.queue = 0x42; + call.accepted = 12; + call.completed = 9; + writer.WriteCall(call); + }; + write(FrameCallKind::Submit, Commands, commands.size()); + write(FrameCallKind::Memory, 0, 0, {1}); + write(FrameCallKind::RegisterShader, Header, 0); + write(FrameCallKind::FrameStart, 0, 0); + write(FrameCallKind::Submit, Commands, commands.size()); + write(FrameCallKind::Memory, 0, 0, {2}); + write(FrameCallKind::Submit, Commands, commands.size()); + write(FrameCallKind::RegisterShader, Header, 0); + write(FrameCallKind::Memory, 0, 0, {3}); + write(FrameCallKind::RegisterShader, Header, 0); + write(FrameCallKind::Flip, 0, 0); + write(FrameCallKind::ReserveEopFlip, 0, 0); + write(FrameCallKind::Submit, 0x3000000000ull, 1); + write(FrameCallKind::Submit, Commands, std::numeric_limits::max()); + write(FrameCallKind::Submit, std::numeric_limits::max() - 1, 1); + write(FrameCallKind::Submit, Commands, 0); + writer.Finish("{}"); +} + +void testColdPipelines() { + const auto* inherited = std::getenv("APS5_HEADLESS_PIPELINE_CACHE"); + check(inherited != nullptr && std::string_view(inherited) == "1", "pipeline cache setting was not inherited"); + Options options; + options.coldPipelines = true; + configureEnvironment(options); + check(std::getenv("APS5_HEADLESS_PIPELINE_CACHE") == nullptr, "cold replay kept the inherited pipeline cache setting"); + check(std::getenv("APS5_FRAME_CAPTURE") == nullptr, "replay kept the capture setting"); + options.coldPipelines = false; + configureEnvironment(options); + const auto* warm = std::getenv("APS5_HEADLESS_PIPELINE_CACHE"); + check(warm != nullptr && std::string_view(warm) == "1", "warm replay did not enable the pipeline cache"); +#ifdef _WIN32 + _putenv_s("APS5_HEADLESS_PIPELINE_CACHE", "0"); +#else + setenv("APS5_HEADLESS_PIPELINE_CACHE", "0", 1); +#endif + options.coldPipelines = true; + configureEnvironment(options); + check(std::getenv("APS5_HEADLESS_PIPELINE_CACHE") == nullptr, "cold replay kept a present zero pipeline cache setting"); +} + } int main(int argc, char** argv) { @@ -401,6 +481,10 @@ int main(int argc, char** argv) { check(argc == 3, "expected a test name and capture directory"); const std::string mode = argv[1]; const std::filesystem::path directory = argv[2]; + if (mode == "inspect_capture") { + writeInspectionCapture(directory); + return 0; + } if (mode == "order") testMemoryOrder(directory); else if (mode == "zero") testReusedMemory(directory); else if (mode == "shared_page") testSharedPage(directory); @@ -409,6 +493,7 @@ int main(int argc, char** argv) { else if (mode == "tokens") testEopTokens(); else if (mode == "reservations" || mode == "reservation_failure") testRepeatedReservations(directory, mode == "reservation_failure"); else if (mode == "blocking") testBlockingSubmission(directory); + else if (mode == "environment") testColdPipelines(); else if (mode == "shader" || mode == "shader_host") testShaderCode(directory, mode == "shader_host"); else throw std::runtime_error("unknown test"); if (mode != "reservation_failure") AgcDriverShutdown_nid_postfix(); diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake b/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake new file mode 100644 index 000000000000..86da56139d8a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake @@ -0,0 +1,105 @@ +set(capture "${DIRECTORY}/capture 'quoted' & $scene %APS5_HEADLESS_PIPELINE_CACHE% ! (1)") +execute_process(COMMAND "${WRITER}" inspect_capture "${capture}" + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error) +if(NOT result STREQUAL "0") + message(FATAL_ERROR "capture writer failed: ${result}\n${output}\n${error}") +endif() +if(NOT EXISTS "${capture}/calls.bin") + message(FATAL_ERROR "capture path was not passed literally: ${capture}") +endif() + +function(runTool expected) + execute_process(COMMAND "${TOOL}" "${capture}" ${ARGN} + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error) + string(REPLACE "\\" "/" error "${error}") + if(NOT result STREQUAL "${expected}") + message(FATAL_ERROR "agc_frame_replay ${ARGN}: expected ${expected}, got ${result}\n${output}\n${error}") + endif() + set(output "${output}" PARENT_SCOPE) + set(error "${error}" PARENT_SCOPE) +endfunction() + +function(requireContains text needle) + string(FIND "${text}" "${needle}" found) + if(found EQUAL -1) + message(FATAL_ERROR "missing '${needle}' in:\n${text}") + endif() +endfunction() + +if(MODE STREQUAL "submits") + runTool(0 --submits 1) + requireContains("${output}" "#1 ordinal 5 submit thread 7 queue 0x42 serials 9/12 address 0x1000000001 dwords 27") + requireContains("${output}" "@5 WAIT_REG_MEM function 3 space 1 address 0x9abcdef012345678 reference 0x10203040 mask 0xf0f0f0f0") + requireContains("${output}" "@12 RELEASE_MEM 00000001 00000002 00000003 00000004 00000005 00000006 00000007") + requireContains("${output}" "@20 WRITE_DATA 00000500 deadbeef 00000012 00001337") + requireContains("${output}" "@25 (truncated packet: 5 dwords, 2 available)") + requireContains("${output}" "packets: RELEASE_MEM x1 WAIT_REG_MEM x1 WRITE_DATA x1") + if(output MATCHES "#2|ordinal 1 |reference 0x55667788|NOP|@1 WAIT_REG_MEM") + message(FATAL_ERROR "listing included a later submission, prologue or payload packet:\n${output}") + endif() + runTool(0 --submits 2) + requireContains("${output}" "#2 ordinal 7 submit") + requireContains("${output}" "reference 0x55667788") + runTool(0 --submits 4) + requireContains("${output}" "#3 ordinal 11 flip") + requireContains("${output}" "#4 ordinal 12 reserve-eop-flip") + if(output MATCHES "#5") + message(FATAL_ERROR "listing exceeded --submits 4:\n${output}") + endif() + runTool(0 --submits 8) + requireContains("${output}" "#8 ordinal 16 submit") + string(REGEX MATCHALL "command buffer not readable" unreadable "${output}") + list(LENGTH unreadable count) + if(NOT count EQUAL 3) + message(FATAL_ERROR "missing unrecorded and overflowing buffer diagnostics:\n${output}") + endif() + foreach(value IN ITEMS 0 -1 +1 1junk 18446744073709551616) + runTool(2 --submits "${value}") + requireContains("${error}" "usage: agc_frame_replay") + endforeach() + runTool(2 --submits) +elseif(MODE STREQUAL "shader_output") + runTool(0 --shaders) + requireContains("${output}" "\"ordinal\":3,\"in_window\":false") + requireContains("${output}" "\"ordinal\":8,\"in_window\":true") + requireContains("${output}" "\"hash\":\"541dd60be6dc3e20\",\"copy\":2") + requireContains("${output}" "\"hash\":\"b6df526defd638c2\",\"copy\":1") + requireContains("${output}" "{\"registrations\":3,\"in_window\":2,\"distinct_code\":2,\"unreadable\":0}") + set(binaries "${DIRECTORY}/shader code 'quoted' & $scene (2)") + runTool(0 --cold-pipelines --shader-code "${binaries}") + file(GLOB files "${binaries}/*.bin") + list(LENGTH files count) + if(NOT count EQUAL 2) + message(FATAL_ERROR "expected two distinct shader binaries, got ${count}") + endif() + file(READ "${binaries}/type0-541dd60be6dc3e20.bin" first HEX) + file(READ "${binaries}/type0-b6df526defd638c2.bin" second HEX) + if(NOT first STREQUAL "0001020304050607ff800d0a27242600" OR NOT second STREQUAL "4201020304050607ff800d0a27242600") + message(FATAL_ERROR "shader binaries differ from the bytes at registration: ${first}, ${second}") + endif() + set(blocked "${DIRECTORY}/blocked output 'quoted' & $scene (3)") + set(blockedFile "${blocked}/type0-541dd60be6dc3e20.bin") + file(MAKE_DIRECTORY "${blockedFile}") + runTool(2 --shader-code "${blocked}") + requireContains("${error}" "cannot open ${blockedFile}") + if(output MATCHES "\"registrations\"") + message(FATAL_ERROR "failed shader output reported success JSON:\n${output}") + endif() + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(full "${DIRECTORY}/full output") + file(MAKE_DIRECTORY "${full}") + set(fullFile "${full}/type0-541dd60be6dc3e20.bin") + file(REMOVE "${fullFile}") + file(CREATE_LINK /dev/full "${fullFile}" SYMBOLIC RESULT linked) + if(NOT linked STREQUAL "0") + message(FATAL_ERROR "cannot link /dev/full: ${linked}") + endif() + runTool(2 --shader-code "${full}") + requireContains("${error}" "${fullFile}") + if(NOT error MATCHES "cannot (write|flush|close)" OR NOT output STREQUAL "") + message(FATAL_ERROR "late shader output failure was not reported:\n${output}\n${error}") + endif() + endif() +else() + message(FATAL_ERROR "unknown inspection mode: ${MODE}") +endif() diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 04e9ca9c92ca..713406ccaef7 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/FrameCapture.hpp" #include "prx/libSceAgcDriver/Execution/include/FrameDumpIo.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -791,23 +794,6 @@ FrameCheck compareFrames(const FrameCaptureReader& reader, const Options& option return check; } -bool readable(std::uint64_t address, std::uint64_t bytes) { -#ifdef _WIN32 - for (auto at = address; at < address + bytes;) { - MEMORY_BASIC_INFORMATION info{}; - if (VirtualQuery(reinterpret_cast(static_cast(at)), &info, sizeof(info)) == 0 || info.State != MEM_COMMIT) return false; - const auto access = info.Protect & 0xffu; - if ((info.Protect & PAGE_GUARD) != 0 || access == PAGE_NOACCESS || access == PAGE_EXECUTE) return false; - at = reinterpret_cast(info.BaseAddress) + info.RegionSize; - } - return true; -#else - static_cast(address); - static_cast(bytes); - return false; -#endif -} - std::uint64_t contentHash(const std::byte* bytes, std::size_t size) { std::uint64_t hash = 0xcbf29ce484222325ull; for (std::size_t index = 0; index < size; ++index) { @@ -830,6 +816,7 @@ class CapturedMemory { } std::optional> Read(std::uint64_t address, std::uint64_t bytes) { + if (bytes > std::numeric_limits::max() || bytes > std::numeric_limits::max() - address) return std::nullopt; std::vector result(static_cast(bytes)); for (auto at = address; at < address + bytes;) { const auto* range = covering(ranges, at); @@ -920,8 +907,15 @@ int listShaders(const Options& options) { if (copy == 1 && !options.shaderCode.empty()) { char name[64]; std::snprintf(name, sizeof(name), "type%u-%016llx.bin", static_cast(fields.type), static_cast(hash)); - std::ofstream file(options.shaderCode / name, std::ios::binary | std::ios::trunc); + const auto path = options.shaderCode / name; + std::ofstream file(path, std::ios::binary | std::ios::trunc); + if (!file.is_open()) throw std::runtime_error("frame replay: cannot open " + path.string()); file.write(reinterpret_cast(bytes->data()), static_cast(bytes->size())); + if (!file) throw std::runtime_error("frame replay: cannot write " + path.string()); + file.flush(); + if (!file) throw std::runtime_error("frame replay: cannot flush " + path.string()); + file.close(); + if (!file) throw std::runtime_error("frame replay: cannot close " + path.string()); } const bool window = frameStart != calls.end() && call > frameStart; ++listed; @@ -956,24 +950,30 @@ std::string_view packetName(std::uint32_t opcode) { } } -void describeSubmission(const FrameCall& call) { - const auto* words = reinterpret_cast(static_cast(call.address)); - if (!readable(call.address, call.size * sizeof(std::uint32_t))) { +void describeSubmission(const FrameCall& call, CapturedMemory& memory) { + const auto bytes = call.size <= std::numeric_limits::max() / sizeof(std::uint32_t) ? memory.Read(call.address, call.size * sizeof(std::uint32_t)) : std::nullopt; + if (!bytes) { std::printf(" (command buffer not readable)\n"); return; } + std::vector words(static_cast(call.size)); + if (!bytes->empty()) std::memcpy(words.data(), bytes->data(), bytes->size()); std::map counts; for (std::size_t at = 0; at < call.size;) { const auto header = words[at]; const auto type = header >> 30u; - const auto length = type == 3u ? static_cast((header >> 16u) & 0x3fffu) + 2u : 1u; - if (type == 3u) { + const auto length = AgcDriver::Pm4::PacketWords(header); + if (length > words.size() - at) { + std::printf(" @%zu (truncated packet: %zu dwords, %zu available)\n", at, length, words.size() - at); + break; + } + if (type == 3u && !AgcDriver::Pm4::FillerPacket(header)) { const auto opcode = (header >> 8u) & 0xffu; const auto name = packetName(opcode); char label[24]; std::snprintf(label, sizeof(label), "0x%02x", opcode); const std::string key = name.empty() ? std::string(label) : std::string(name); - const auto word = [&](std::size_t index) { return at + index < call.size ? words[at + index] : 0u; }; + const auto word = [&](std::size_t index) { return index < length ? words[at + index] : 0u; }; if (opcode == 0x3c) { std::printf(" @%zu WAIT_REG_MEM function %u space %u address 0x%llx reference 0x%x mask 0x%x\n", at, word(1) & 7u, (word(1) >> 4u) & 3u, static_cast(word(2)) | (static_cast(word(3)) << 32u), word(4), word(5)); @@ -993,7 +993,7 @@ void describeSubmission(const FrameCall& call) { int listSubmits(const Options& options) { const FrameCaptureReader reader(options.directory); - const auto skipped = restoreMemory(reader); + CapturedMemory memory(reader); std::map> groups; for (const auto& chunk : reader.Chunks()) { if (chunk.kind == FrameMemoryKind::Delta) groups[chunk.group].push_back(&chunk); @@ -1001,21 +1001,18 @@ int listSubmits(const Options& options) { const auto& calls = reader.Calls(); const auto frameStart = std::find_if(calls.begin(), calls.end(), [](const FrameCall& call) { return call.kind == FrameCallKind::FrameStart; }); std::size_t listed = 0; - for (auto call = frameStart; call != calls.end() && listed < options.submits; ++call) { + for (auto call = calls.begin(); call != calls.end() && listed < options.submits; ++call) { if (call->kind == FrameCallKind::Memory) { - if (const auto found = groups.find(call->values.at(0)); found != groups.end()) { - for (const auto* chunk : found->second) { - if (!skipped.Overlaps(*chunk)) restore(reader, *chunk); - } - } + if (const auto found = groups.find(call->values.at(0)); found != groups.end()) memory.Apply(found->second); continue; } + if (call < frameStart) continue; if (call->kind != FrameCallKind::Submit && call->kind != FrameCallKind::Flip && call->kind != FrameCallKind::ReserveEopFlip) continue; ++listed; std::printf("#%zu ordinal %llu %s thread %llu queue 0x%x serials %llu/%llu address 0x%llx dwords %llu\n", listed, static_cast(call->ordinal), std::string(AgcDriver::FrameCallName(call->kind)).c_str(), static_cast(call->thread), call->queue, static_cast(call->completed), static_cast(call->accepted), static_cast(call->address), static_cast(call->size)); - if (call->kind == FrameCallKind::Submit) describeSubmission(*call); + if (call->kind == FrameCallKind::Submit) describeSubmission(*call, memory); } std::fflush(stdout); return 0; @@ -1132,6 +1129,17 @@ int run(const Options& options) { return replay.unfinished == 0 ? 0 : 3; } +void configureEnvironment(const Options& options) { +#ifdef _WIN32 + _putenv_s("APS5_FRAME_CAPTURE", ""); + _putenv_s("APS5_HEADLESS_PIPELINE_CACHE", options.coldPipelines ? "" : "1"); +#else + unsetenv("APS5_FRAME_CAPTURE"); + if (options.coldPipelines) unsetenv("APS5_HEADLESS_PIPELINE_CACHE"); + else setenv("APS5_HEADLESS_PIPELINE_CACHE", "1", 1); +#endif +} + } int main(int argc, char** argv) { @@ -1150,9 +1158,9 @@ int main(int argc, char** argv) { options.shaderCode = argv[++index]; } else if (argument == "--submits" && index + 1 < argc) { - char* end = nullptr; - options.submits = std::strtoull(argv[++index], &end, 10); - if (options.submits == 0 || end == argv[index] || *end != '\0') return usage(); + const std::string_view value = argv[++index]; + const auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), options.submits); + if (error != std::errc{} || end != value.data() + value.size() || options.submits == 0) return usage(); } else if (argument == "--trace-serials") options.traceSerials = true; else if (argument == "--cold-pipelines") options.coldPipelines = true; else if (argument == "--dump" && index + 1 < argc) options.dumps.push_back(parseDump(argv[++index])); @@ -1161,14 +1169,7 @@ int main(int argc, char** argv) { else return usage(); } if (options.directory.empty()) return usage(); -#ifdef _WIN32 - _putenv_s("APS5_FRAME_CAPTURE", ""); - _putenv_s("APS5_HEADLESS_PIPELINE_CACHE", options.coldPipelines ? "" : "1"); -#else - unsetenv("APS5_FRAME_CAPTURE"); - if (options.coldPipelines) unsetenv("APS5_HEADLESS_PIPELINE_CACHE"); - else setenv("APS5_HEADLESS_PIPELINE_CACHE", "1", 1); -#endif + configureEnvironment(options); int code = 2; try { const bool listing = options.shaders || options.submits != 0; diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index bd57560d77a6..6590403b5b0e 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -228,7 +228,7 @@ On Windows use `agc_pm4_replay.exe` and prepend `build/core/libs/libs/unpatched` On Windows and Linux, set `APS5_FRAME_CAPTURE` to a directory to record a few frames of a title at the AGC driver boundary, then replay them with `agc_frame_replay` through any build of the driver with no title running. From the start of the process the driver keeps every shader registration and ABI resolution, video output registration and EOP flip reservation. At the frame boundary picked by `APS5_FRAME_CAPTURE_AT_FLIP` (default 1), or by the appearance of the file `APS5_FRAME_CAPTURE_TRIGGER` names (a number in it replaces the frame count), it waits for the driver to go idle and writes every committed, readable allocation of the guest allocation registry, the title's image included, as zstd chunks without their all-zero 64 KiB blocks. A frame ends at a CPU flip or at a submission that flips (a flip packet, or the EOP release of a reserved flip token); the reservation itself only records its token. For the next `APS5_FRAME_CAPTURE_FRAMES` (10) frames it records each submission (queue, command buffer address and size), flip and suspend point with the thread that made it and the accepted and completed serials, each preceded by the guest pages the write tracker saw the CPU write since the previous call that handed work to the GPU (`GuestMemory::CollectWrittenBlocks`, per 4 KiB where the tracker kept page stamps), read once the submissions already accepted have completed (a recorded submission, flip or suspend point reaches the driver before the recorder takes its next call, so that wait covers every call recorded before the delta); an allocation the registry gained after the snapshot is recorded there as a new range with its nonzero 64 KiB blocks. A submission ends a frame only when it flips itself, not when another thread's submission flipped while it was being submitted. Shader registrations and ABI resolutions only collect the blocks of the shader header and the arrays it points to, and the pages of the title's image only when their content changed. At the end it stores each display buffer the title presented, decoded, as a reference image. Without the guest write watch (`APS5_NO_WRITE_WATCH`, or no write watch on the host) the capture throws when it starts, since it could not record CPU stores. A capture is `calls.bin`, `memory.bin` and `capture.json` (commit, switches, sizes, warnings), and the recorder stops with a warning when its volume has less than 4 GiB free. A 20-frame capture in the Demon's Souls Nexus is 5.3 GiB of snapshot and 11.5 GiB of deltas before compression, 3.9 GiB on disk. -`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--cold-pipelines] [--shaders] [--shader-code ] [--submits ] [--trace-serials] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The replay's device has no window, but it loads the title's Vulkan pipeline cache file (`vk-pipelines-*.bin` in the shader cache directory) as the windowed device does, so pipelines the title already compiled are not compiled again (`--cold-pipelines` starts with an empty one; the driver switch is `APS5_HEADLESS_PIPELINE_CACHE`). The title replayed its pipeline journal at boot, long before the captured frames, so after the registrations the replay waits for the driver's journal replay to finish before the first pass (`journal_s` in the JSON line). The first pass still compiles every shader and pipeline the frames use that neither cache holds, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. `--shaders` replays nothing and maps no guest memory: it reads the snapshot, applies the captured writes in call order and prints one JSON line per shader registration (code address, size, type, code hash and how many earlier registrations had the same code), then a summary; `--shader-code ` also writes each distinct code as `/type-.bin`. `--submits ` replays nothing either and lists the first n submissions and flips of the captured frames with their queue and serials, the packets of each and the fields of its waits and releases. `--trace-serials` prints each submission, flip and suspend point with its call ordinal and command address as it is issued; with `APS5_TRACE_GPU` the driver prints the serial it accepts for each address and each completion, so a replay that stops completing can be followed to the call that made the submission. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. +`agc_frame_replay [--no-pace] [--no-release] [--release-after-ms ] [--no-synthesize] [--strict] [--repeat ] [--cold-pipelines] [--shaders] [--shader-code ] [--submits ] [--trace-serials] [--dump
::]... [--frames ]` maps the captured ranges at their addresses and registers them with their access (a range the replaying process already maps writable, such as the driver's fixed memory, is only refilled, and a non-GPU range it uses itself is skipped and counted), repeats the registrations, then replays the calls of each captured thread on a thread of its own, started in captured order. Before a call that hands work to the GPU it waits until no more submissions are in flight than when the title made it (`--no-pace` turns this off). Titles release GPU waits from threads that never call the driver, so when the driver completes nothing for 3 ms the replay reads the waits its queues are blocked on (`AgcDriverAwaitedWaits`) and writes the next recorded CPU value of the awaited dword that satisfies the compare, else the last earlier one, else a value derived from the reference (`--no-synthesize` turns this last step off). It prints one JSON line with the frame count and times, the submissions, the pacing wait, the released waits and how many were derived, the submissions that never completed and, for captures with reference images, the share of identical pixels and the largest channel difference of the replayed display buffers (`--frames` writes both images). Effects that depend on timing (temporal antialiasing, particles) make pixels differ, so expect a visual match rather than identical frames. The replay's device has no window, but it loads the title's Vulkan pipeline cache file (`vk-pipelines-*.bin` in the shader cache directory) as the windowed device does, so pipelines the title already compiled are not compiled again (`--cold-pipelines` starts with an empty one and clears an inherited `APS5_HEADLESS_PIPELINE_CACHE`; that variable is the driver switch). The title replayed its pipeline journal at boot, long before the captured frames, so after the registrations the replay waits for the driver's journal replay to finish before the first pass (`journal_s` in the JSON line). The first pass still compiles every shader and pipeline the frames use that neither cache holds, so `--repeat ` replays the frames n times in one process: before each later pass it waits for the driver to go idle, clears the captured ranges, restores the snapshot and makes the reservations that were pending at the snapshot again (a token still reserved from the previous pass is kept), and the frame times then come from passes 2 to n, with each pass listed under `passes`. Later passes replay the same frames, so driver caches keyed on values that change every frame in the title (ring addresses in user data, for example) can hit entries of an earlier pass; compare such changes on the first pass with warm disk caches instead. `--shaders` replays nothing and maps no guest memory: it reads the snapshot, applies the captured writes in call order and prints one JSON line per shader registration (code address, size, type, code hash and how many earlier registrations had the same code), then a summary; `--shader-code ` also writes each distinct code as `/type-.bin`. `--submits ` also reads captured memory without mapping guest addresses, applies writes in call order, and lists the first n submissions and flips of the captured frames with their queue and serials, the packets of each and the fields of its waits and releases. `--trace-serials` prints each submission, flip and suspend point with its call ordinal and command address as it is issued; with `APS5_TRACE_GPU` the driver prints the serial it accepts for each address and each completion, so a replay that stops completing can be followed to the call that made the submission. Replay times follow the GPU clock state: lock the clocks (`nvidia-smi -lgc`, `-lmc`) when comparing two drivers. ```sh cmake --build build --target agc_frame_replay From 210bd86b58b015bd07eba4d84afc0c1081ebfc70 Mon Sep 17 00:00:00 2001 From: Sam <85789460+Sam-mon7@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:01:17 +0000 Subject: [PATCH 08/12] test(agc): accept the host CPU line in the late shader output failure check On Linux libc prints its host CPU line to stdout, so the check that a failed write left stdout empty failed even though the tool reported the failure and exited 2. Check that no success summary was printed instead, as the blocked output case does. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake b/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake index 86da56139d8a..dcb2bf078bb4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplayTool.cmake @@ -96,7 +96,7 @@ elseif(MODE STREQUAL "shader_output") endif() runTool(2 --shader-code "${full}") requireContains("${error}" "${fullFile}") - if(NOT error MATCHES "cannot (write|flush|close)" OR NOT output STREQUAL "") + if(NOT error MATCHES "cannot (write|flush|close)" OR output MATCHES "\"registrations\"") message(FATAL_ERROR "late shader output failure was not reported:\n${output}\n${error}") endif() endif() From df74f29c46e465ab1efde4b4e08a94bcfba4863f Mon Sep 17 00:00:00 2001 From: Jamie <55252868+chitters2004@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:48:07 +0000 Subject: [PATCH 09/12] fix(agc): relaunch the frame replay when a host region blocks a captured GPU range On Windows the startup thread stacks can sit inside a captured GPU-visible range, which the replay must map at its captured address. Start the replay again with a fresh address-space layout, up to 8 times, before mapping any memory. Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 0ddce118dbcd..96742f9df03c 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -714,9 +714,42 @@ FrameCheck compareFrames(const FrameCaptureReader& reader, const Options& option return check; } +#ifdef _WIN32 +void relaunchIfHostRegionBlocks(const FrameCaptureReader& reader) { + constexpr int Attempts = 8; + const auto& chunks = reader.Chunks(); + const auto blocked = std::find_if(chunks.begin(), chunks.end(), [](const FrameMemoryChunk& chunk) { + return chunk.kind == FrameMemoryKind::Range && chunk.rangeKind != FrameRangeKind::Host && (chunk.access & FrameRangeAccess::Gpu) != 0 && + GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(static_cast(chunk.address), static_cast(chunk.bytes)); + }); + if (blocked == chunks.end()) return; + const char* const previous = std::getenv("APS5_FRAME_REPLAY_ATTEMPT"); + const int attempt = previous == nullptr ? 1 : std::atoi(previous) + 1; + if (attempt > Attempts) return; + std::fprintf(stderr, "[frame-replay] a host region sits in the captured GPU range 0x%llx+0x%llx; relaunching (%d of %d)\n", static_cast(blocked->address), + static_cast(blocked->bytes), attempt, Attempts); + std::fflush(stdout); + std::fflush(stderr); + _putenv_s("APS5_FRAME_REPLAY_ATTEMPT", std::to_string(attempt).c_str()); + std::wstring commandLine = GetCommandLineW(); + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + PROCESS_INFORMATION process{}; + if (!CreateProcessW(nullptr, commandLine.data(), nullptr, nullptr, TRUE, 0, nullptr, nullptr, &startup, &process)) throw std::runtime_error("frame replay: cannot relaunch, error " + std::to_string(GetLastError())); + CloseHandle(process.hThread); + WaitForSingleObject(process.hProcess, INFINITE); + DWORD code = 2; + GetExitCodeProcess(process.hProcess, &code); + std::_Exit(static_cast(code)); +} +#endif + int run(const Options& options) { const auto loadStart = Clock::now(); const FrameCaptureReader reader(options.directory); +#ifdef _WIN32 + relaunchIfHostRegionBlocks(reader); +#endif if (reader.Truncated()) std::fprintf(stderr, "[frame-replay] the capture is truncated; replaying the complete records\n"); auto skipped = restoreMemory(reader); const auto loaded = Clock::now(); From 89db2b5ae2b0b99fac51742f73cbab3513631bbf Mon Sep 17 00:00:00 2001 From: Jamie <55252868+chitters2004@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:55:10 +0000 Subject: [PATCH 10/12] docs(env): list APS5_FRAME_REPLAY_ATTEMPT Co-Authored-By: Claude Opus 5.5 --- docs/dev/ENVIRONMENT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 05678c738d62..c7f6898408c8 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -170,6 +170,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_FRAME_CAPTURE_AT_FLIP` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp` | 0 | frame boundary (CPU flip or submission that flips) at which the frame capture starts when no trigger file is set; 0 and 1 both start at the first boundary; a value that is not a number fails | | `APS5_FRAME_CAPTURE_FRAMES` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp` | 10 | number of frame boundaries the frame capture records | | `APS5_FRAME_CAPTURE_TRIGGER` | string | `core/libs/prx/libSceAgcDriver/Execution/src/FrameRecorder.cpp` | | file whose appearance starts the frame capture at the next frame boundary; a number in it replaces the frame count | +| `APS5_FRAME_REPLAY_ATTEMPT` | integer | `core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp` | 0 | number of times `agc_frame_replay` has started itself again on Windows because a host region sat in a captured GPU-visible range; set by the replay for the process it starts, which stops relaunching after 8 | | `APS5_FULL_BARRIERS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | records every barrier instead of merging redundant ones | | `APS5_GPU_CHECKPOINTS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | attaches device diagnostic checkpoints through VK_NV_device_diagnostic_checkpoints, or buffer markers through VK_AMD_buffer_marker on a device without it | | `APS5_GPU_COPY_MAX_KIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | 1024 | size cap in KiB on a descriptor sub-range copied through the GPU path, larger ones staying on the CPU | From 93123ef32c576bb642d3281521a77cea92b90a17 Mon Sep 17 00:00:00 2001 From: Jamie <55252868+chitters2004@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:02:48 +0000 Subject: [PATCH 11/12] test(agc): skip the frame replay tests without a Vulkan driver Co-Authored-By: Claude Opus 5.5 --- .../prx/libSceAgcDriver/tests/FrameReplay.cpp | 2 ++ .../cmake/agc_driver_frame_replay_tests.cmake | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp index 58f420fb6026..2915200a3b9b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplay.cpp @@ -1,3 +1,4 @@ +#include "execution/VulkanTestDevice.hpp" #include "prx/libSceAgc/DcbFlow/include/Memory.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/FrameCapture.hpp" @@ -331,6 +332,7 @@ int main(int argc, char** argv) { return 2; } try { + if (!OpenVulkanTestDevice()) return VulkanTestSkipped; if (argc == 3 && std::string(argv[2]) == "concurrent") testConcurrentCopies(argv[1]); else if (argc == 3 && std::string(argv[2]) == "reservation_error") testReservationError(argv[1]); else testCaptureAndReplay(argv[1], argc == 3); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_tests.cmake index 2adff088124c..ac0f421ff238 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_tests.cmake @@ -1,7 +1,12 @@ if(BUILD_TESTING) if(WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Linux") add_test_executable(agc_driver_frame_replay_tests tests/FrameReplay.cpp Execution/src/FrameCapture.cpp) - target_include_directories(agc_driver_frame_replay_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_include_directories(agc_driver_frame_replay_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR} + ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR} + ) target_link_libraries(agc_driver_frame_replay_tests PRIVATE libSceAgc libSceAgcDriver libkernel libc libzstd_static) set_target_properties(agc_driver_frame_replay_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_frame_replay_tests) @@ -9,7 +14,7 @@ if(BUILD_TESTING) add_test(NAME agc_driver_frame_replay COMMAND agc_driver_frame_replay_tests $) set_tests_properties(agc_driver_frame_replay PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame replay capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=2" - TIMEOUT 120) + TIMEOUT 120 SKIP_RETURN_CODE 77) if(WIN32) set_tests_properties(agc_driver_frame_replay PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") @@ -17,7 +22,7 @@ if(BUILD_TESTING) add_test(NAME agc_driver_frame_replay_cold_pipelines COMMAND agc_driver_frame_replay_tests $ pipelines) set_tests_properties(agc_driver_frame_replay_cold_pipelines PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=0;APS5_HEADLESS_PIPELINE_CACHE=1;ANYPS5_SHADER_CACHE_DIR=${CMAKE_CURRENT_BINARY_DIR}/frame replay pipeline cache;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame replay cold capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=2" - TIMEOUT 30) + TIMEOUT 30 SKIP_RETURN_CODE 77) if(WIN32) set_tests_properties(agc_driver_frame_replay_cold_pipelines PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") @@ -25,7 +30,7 @@ if(BUILD_TESTING) add_test(NAME agc_driver_frame_replay_reservation_error COMMAND agc_driver_frame_replay_tests $ reservation_error) set_tests_properties(agc_driver_frame_replay_reservation_error PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame replay reservation error" - TIMEOUT 30) + TIMEOUT 30 SKIP_RETURN_CODE 77) if(WIN32) set_tests_properties(agc_driver_frame_replay_reservation_error PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") @@ -33,7 +38,7 @@ if(BUILD_TESTING) add_test(NAME agc_driver_frame_replay_concurrent COMMAND agc_driver_frame_replay_tests $ concurrent) set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_FRAME_CAPTURE=${CMAKE_CURRENT_BINARY_DIR}/frame-replay-concurrent-capture;APS5_FRAME_CAPTURE_AT_FLIP=1;APS5_FRAME_CAPTURE_FRAMES=8" - TIMEOUT 120) + TIMEOUT 120 SKIP_RETURN_CODE 77) if(WIN32) set_tests_properties(agc_driver_frame_replay_concurrent PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") From 9a2b40f2336b7c5f32f234d5a7ff1ca19692d7ed Mon Sep 17 00:00:00 2001 From: Jamie <55252868+chitters2004@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:55:53 +0000 Subject: [PATCH 12/12] fix(agc): bound the frame replay relaunch count and test the relaunch limit An inherited negative or malformed APS5_FRAME_REPLAY_ATTEMPT went through atoi, so the limit of eight relaunches did not hold, and a failed _putenv_s restarted the budget in the child. The count now has to be a whole number from 0 to 8, and the relaunch stops when it cannot be passed on. The new Windows test links without high-entropy addresses so its stack is an arena host region in every process, and covers the eight-relaunch limit, malformed counts and exit codes through the chain. Co-Authored-By: Claude Opus 5.5 --- .../tests/FrameReplayRelaunch.cpp | 98 +++++++++++++++++++ ...c_driver_frame_replay_relaunch_tests.cmake | 12 +++ .../prx/libSceAgcDriver/tools/FrameReplay.cpp | 23 +++-- 3 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/FrameReplayRelaunch.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_relaunch_tests.cmake diff --git a/core/libs/prx/libSceAgcDriver/tests/FrameReplayRelaunch.cpp b/core/libs/prx/libSceAgcDriver/tests/FrameReplayRelaunch.cpp new file mode 100644 index 000000000000..e5b984dae30c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FrameReplayRelaunch.cpp @@ -0,0 +1,98 @@ +#define main frameReplayToolMain +#include "../tools/FrameReplay.cpp" +#undef main +#include + +namespace { + +using AgcDriver::FrameCaptureWriter; + +constexpr int Skip = 77; +constexpr DWORD ChildLimit = 60000; + +void check(bool condition, const std::string& reason) { + if (!condition) throw std::runtime_error(reason); +} + +void append(const std::filesystem::path& log, const std::string& line) { + std::ofstream file(log, std::ios::app); + file << line << '\n'; + check(file.good(), "cannot write the relaunch log"); +} + +int child(const std::filesystem::path& directory, int exitCode) { + const int local = 0; + const auto address = reinterpret_cast(&local) & ~std::uintptr_t{0xfff}; + if (!GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(address, 4096)) return Skip; + const char* const attempt = std::getenv("APS5_FRAME_REPLAY_ATTEMPT"); + append(directory / "log.txt", attempt == nullptr ? "none" : attempt); + const auto capture = directory / std::to_string(GetCurrentProcessId()); + FrameCaptureWriter writer(capture); + writer.WriteRange(address, 4096, FrameRangeKind::Watched, 3, AgcDriver::FrameRangeAccess::Readable | AgcDriver::FrameRangeAccess::Gpu); + writer.Finish("{}"); + const FrameCaptureReader reader(capture); + relaunchIfHostRegionBlocks(reader); + append(directory / "log.txt", "continued"); + return exitCode; +} + +int start(const std::filesystem::path& directory, const char* attempt, int exitCode) { + std::filesystem::remove_all(directory); + std::filesystem::create_directories(directory); + check(_putenv_s("APS5_FRAME_REPLAY_ATTEMPT", attempt == nullptr ? "" : attempt) == 0, "cannot set the relaunch count"); + std::wstring self(32768, L'\0'); + self.resize(GetModuleFileNameW(nullptr, self.data(), static_cast(self.size()))); + std::wstring commandLine = L"\"" + self + L"\" child \"" + directory.wstring() + L"\" " + std::to_wstring(exitCode); + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + PROCESS_INFORMATION process{}; + check(CreateProcessW(nullptr, commandLine.data(), nullptr, nullptr, TRUE, 0, nullptr, nullptr, &startup, &process) != 0, "cannot start the replay child"); + CloseHandle(process.hThread); + const bool exited = WaitForSingleObject(process.hProcess, ChildLimit) == WAIT_OBJECT_0; + if (!exited) TerminateProcess(process.hProcess, 1); + DWORD code = 1; + GetExitCodeProcess(process.hProcess, &code); + CloseHandle(process.hProcess); + check(exited, "the replay child did not exit"); + return static_cast(code); +} + +std::vector lines(const std::filesystem::path& path) { + std::vector result; + std::ifstream file(path); + for (std::string line; std::getline(file, line);) result.push_back(line); + return result; +} + +bool expectRun(const std::filesystem::path& directory, const char* attempt, int exitCode, int expected, const std::vector& log) { + const int code = start(directory, attempt, exitCode); + if (code == Skip) return false; + const std::string name = attempt == nullptr ? "no count" : attempt; + check(code == expected, name + ": exited with " + std::to_string(code)); + check(lines(directory / "log.txt") == log, name + ": wrong process sequence"); + return true; +} + +} + +int main(int argc, char** argv) { + try { + if (argc == 4 && std::string_view(argv[1]) == "child") return child(argv[2], std::atoi(argv[3])); + check(argc == 2, "usage: agc_driver_frame_replay_relaunch_tests "); + const std::filesystem::path directory = argv[1]; + const std::vector full{"none", "1", "2", "3", "4", "5", "6", "7", "8", "continued"}; + if (!expectRun(directory, nullptr, 0, 0, full)) { + std::puts("skipped: the stack is not an arena host region"); + return Skip; + } + check(expectRun(directory, nullptr, 42, 42, full), "the failing run skipped"); + check(expectRun(directory, "7", 0, 0, {"7", "8", "continued"}), "the last relaunch skipped"); + check(expectRun(directory, "8", 5, 5, {"8", "continued"}), "the exhausted count skipped"); + for (const char* const malformed : {"-1", "-2147483648", "9", "99999999999", "abc", "1x", " 1", "+1"}) check(expectRun(directory, malformed, 0, 2, {malformed}), "a malformed count skipped"); + std::puts("frame replay relaunch tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "FAILED: %s\n", error.what()); + return argc == 4 ? 2 : 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_relaunch_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_relaunch_tests.cmake new file mode 100644 index 000000000000..a3bb15ba3d06 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_frame_replay_relaunch_tests.cmake @@ -0,0 +1,12 @@ +if(BUILD_TESTING AND WIN32) + add_test_executable(agc_driver_frame_replay_relaunch_tests tests/FrameReplayRelaunch.cpp Execution/src/FrameCapture.cpp) + target_include_directories(agc_driver_frame_replay_relaunch_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) + target_link_libraries(agc_driver_frame_replay_relaunch_tests PRIVATE libSceAgcDriver libkernel libc libzstd_static) + target_link_options(agc_driver_frame_replay_relaunch_tests PRIVATE -Wl,--disable-high-entropy-va) + set_target_properties(agc_driver_frame_replay_relaunch_tests PROPERTIES CXX_EXTENSIONS OFF) + configure_windows_unwind(agc_driver_frame_replay_relaunch_tests) + add_test(NAME agc_driver_frame_replay_relaunch COMMAND agc_driver_frame_replay_relaunch_tests "${CMAKE_CURRENT_BINARY_DIR}/frame replay relaunch") + set_tests_properties(agc_driver_frame_replay_relaunch PROPERTIES + TIMEOUT 120 SKIP_RETURN_CODE 77 + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") +endif() diff --git a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp index 199da031e32c..d9b7b3f0b953 100644 --- a/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/FrameReplay.cpp @@ -795,22 +795,33 @@ FrameCheck compareFrames(const FrameCaptureReader& reader, const Options& option } #ifdef _WIN32 +constexpr int RelaunchAttempts = 8; + +int relaunchAttempt() { + const char* const text = std::getenv("APS5_FRAME_REPLAY_ATTEMPT"); + if (text == nullptr) return 0; + const std::string_view value = text; + int attempt = 0; + const auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), attempt); + if (error != std::errc{} || end != value.data() + value.size() || attempt < 0 || attempt > RelaunchAttempts) + throw std::runtime_error("frame replay: APS5_FRAME_REPLAY_ATTEMPT must be a whole number from 0 to " + std::to_string(RelaunchAttempts) + ", not '" + std::string(value) + "'"); + return attempt; +} + void relaunchIfHostRegionBlocks(const FrameCaptureReader& reader) { - constexpr int Attempts = 8; + const int attempt = relaunchAttempt() + 1; const auto& chunks = reader.Chunks(); const auto blocked = std::find_if(chunks.begin(), chunks.end(), [](const FrameMemoryChunk& chunk) { return chunk.kind == FrameMemoryKind::Range && chunk.rangeKind != FrameRangeKind::Host && (chunk.access & FrameRangeAccess::Gpu) != 0 && GuestArena::GuestArenaHostRegionOverlaps_nid_postfix(static_cast(chunk.address), static_cast(chunk.bytes)); }); if (blocked == chunks.end()) return; - const char* const previous = std::getenv("APS5_FRAME_REPLAY_ATTEMPT"); - const int attempt = previous == nullptr ? 1 : std::atoi(previous) + 1; - if (attempt > Attempts) return; + if (attempt > RelaunchAttempts) return; std::fprintf(stderr, "[frame-replay] a host region sits in the captured GPU range 0x%llx+0x%llx; relaunching (%d of %d)\n", static_cast(blocked->address), - static_cast(blocked->bytes), attempt, Attempts); + static_cast(blocked->bytes), attempt, RelaunchAttempts); std::fflush(stdout); std::fflush(stderr); - _putenv_s("APS5_FRAME_REPLAY_ATTEMPT", std::to_string(attempt).c_str()); + if (_putenv_s("APS5_FRAME_REPLAY_ATTEMPT", std::to_string(attempt).c_str()) != 0) throw std::runtime_error("frame replay: cannot pass the relaunch count to the next process"); std::wstring commandLine = GetCommandLineW(); STARTUPINFOW startup{}; startup.cb = sizeof(startup);