diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 1d8e5eacf4fc..2f7237713a53 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -619,6 +619,7 @@ target_link_libraries(agc_driver_interpolation_modes_tests PRIVATE shader_recomp target_include_directories(agc_driver_prepared_shaders_tests PRIVATE $ ) +target_include_directories(agc_driver_user_data_swappc_tests PRIVATE $) if(BUILD_TESTING) add_test(NAME agc_shader_bindless COMMAND agc_shader_memory_tests --bindless) @@ -954,6 +955,16 @@ if(BUILD_TESTING) set_tests_properties(scalar_swappc PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(user_data_swappc_tests ${LIBS_INCLUDE_DIR}/tests/UserDataSwappc.cpp) + target_link_libraries(user_data_swappc_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(user_data_swappc_tests) + add_test(NAME user_data_swappc COMMAND user_data_swappc_tests) + set_tests_properties(user_data_swappc PROPERTIES TIMEOUT 30) + set_tests_properties(agc_driver_user_data_swappc PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(user_data_swappc PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(scalar_bitcount64_tests ${LIBS_INCLUDE_DIR}/tests/ScalarBitCount64.cpp) target_link_libraries(scalar_bitcount64_tests PRIVATE shader_recompiler_agc_driver libc) configure_windows_unwind(scalar_bitcount64_tests) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 84672986893f..fc18c32bfa2c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -79,7 +79,7 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr< ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId); std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); -ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); +ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request, ShaderMemory* memory = nullptr); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 457fd83602a6..a01d20b93dcd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -14,6 +14,7 @@ namespace ShaderRecompiler { struct SourceHandle; +struct CapturedCallProgram; class PreparedShaderInvocation; } @@ -63,6 +64,7 @@ class ShaderMemory { // resolution. std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle = nullptr); std::shared_ptr Capture(const ShaderRecompiler::PreparedShaderInvocation& invocation); + ShaderRecompiler::CapturedCallProgram ResolveCalls(const ShaderRecompiler::RecompileRequest& request); [[nodiscard]] std::vector Regions() const; // The page regions read since the previous call (or construction), a word read again // included, the initial regions excluded: one stage's own reads on the draw path's shared @@ -91,6 +93,7 @@ class ShaderMemory { // the caller keeps them alive for as long as the capture is used. std::map> initial; std::map pages; + std::vector callCodeReads; PendingWriteQuery pendingWrite = nullptr; PendingWriteObserver observe = nullptr; HookWaitCounter hookWaits = nullptr; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp index a13acf5c8bc4..ece186e26fb8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp @@ -13,7 +13,7 @@ void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffse std::abort(); }; auto verifyMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); - const auto invocation = InvocationFor(snapshot, codeOffset, request); + const auto invocation = InvocationFor(snapshot, codeOffset, request, verifyMemory.get()); const auto capture = verifyMemory->Capture(invocation); const auto regions = verifyMemory->Regions(); request.context.memory = regions; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index b0909f7214cc..e3b43a9d5ea5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -193,11 +193,12 @@ void Driver::dispatch(QueueState& queue, std::span packet, static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { - const auto invocation = InvocationFor(snapshot, codeOffset, request); - timing.Mark("prepared_invocation"); - const auto waitedBefore = traceCapSync() ? Graphics::Recorder::ThreadWaitedMs() : 0.0; forgetAtCapture = GuestMemory::ForgetSerial(); + const auto invocation = InvocationFor(snapshot, codeOffset, request, shaderMemory.get()); + request.shader = invocation.Request().shader; + timing.Mark("prepared_invocation"); + capture = [&] { const SampledReadScope sampling(evidenceReads); return shaderMemory->Capture(invocation); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 109b381a7fd8..e83e39c2dd76 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/LogMacros.hpp" #include "ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/UserDataCalls.hpp" #include "ControlFlow/Structurizer.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" @@ -175,9 +176,11 @@ std::shared_ptr SourceHandleFor(const Shad for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle; } - if (snapshot.header.empty()) { - if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); - APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); + if (snapshot.header.empty() || !request.shader.capturedCalls.empty()) { + if (snapshot.header.empty()) { + if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); + APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); + } auto handle = ShaderRecompiler::PrepareShader(request); snapshot.prepared->entries.push_back({codeOffset, handle}); return handle; @@ -202,26 +205,33 @@ ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapsh throw std::runtime_error("AGC driver: prepared rectangle artifacts are missing"); } -ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { +ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request, ShaderMemory* memory) { require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); const auto code = std::span(snapshot.code).subspan(codeOffset); require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code"); auto invocationRequest = request; + ShaderRecompiler::CapturedCallProgram captured; struct PreparedKeyStorage {}; auto& key = HostThreadLocal, PreparedKeyStorage>(); - ShaderRecompiler::BuildPreparedShaderKey(request, key); std::lock_guard lock(snapshot.prepared->mutex); + const bool captureCalls = std::ranges::none_of(snapshot.prepared->entries, [&](const auto& entry) { return entry.codeOffset == codeOffset; }) || std::ranges::any_of(snapshot.prepared->entries, [&](const auto& entry) { + return entry.codeOffset == codeOffset && !ShaderRecompiler::GetPreparedCalls(*entry.handle).empty(); + }); + if (memory != nullptr && captureCalls && request.shader.stage == ShaderRecompiler::ShaderStage::Compute) { + captured = memory->ResolveCalls(request); + if (!captured.calls.empty()) { + invocationRequest.shader.code = captured.code; + invocationRequest.shader.capturedCalls = captured.calls; + } + } + ShaderRecompiler::BuildPreparedShaderKey(invocationRequest, key); for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset != codeOffset) continue; - invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation); } - if (snapshot.header.empty()) { - if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); - APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); - auto handle = ShaderRecompiler::PrepareShader(request); - invocationRequest = request; - invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle); + if (snapshot.header.empty() || !captured.calls.empty()) { + if (snapshot.header.empty() && (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute)) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); + auto handle = ShaderRecompiler::PrepareShader(invocationRequest); auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key); if (!invocation.has_value()) throw std::runtime_error("AGC driver: raw compute artifact does not match its invocation"); snapshot.prepared->entries.push_back({codeOffset, std::move(handle)}); @@ -366,6 +376,7 @@ std::vector PrepareRegistered(const ShaderSnapshot& snap std::vector userData(userCount); if (stage != Stage::Compute && stage != Stage::Fragment && snapshot.type != 6) vertex = Graphics::DecodeVertexStageInfo(snapshot.header, snapshot.headerAddress, userData, nullptr, true); const ShaderRecompiler::SwappcInfo swappc{vertex.has_value(), firstUser, userCount}; + if (stage == Stage::Compute && !ShaderRecompiler::AnalyzeUserDataCalls(decoded, swappc).empty()) return {}; auto graph = ShaderRecompiler::GraphBuilder{}.Build(decoded, &swappc); ShaderRecompiler::Structurizer{}.Structurize(graph); const std::array memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index 8c07b7b74750..69e95a434a76 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -4,6 +4,7 @@ #include "Optimization/RequestMemoryView.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" +#include "ControlFlow/UserDataCalls.hpp" #include #include #include @@ -219,6 +220,21 @@ std::shared_ptr ShaderMemory::Capture(c return capture(invocation.Request(), nullptr, &invocation); } +ShaderRecompiler::CapturedCallProgram ShaderMemory::ResolveCalls(const ShaderRecompiler::RecompileRequest& request) { + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); + const auto reader = [](void* context, std::uint64_t address, std::uint32_t* value) { + auto& memory = *static_cast(context); + const auto next = memory.initial.upper_bound(address); + if (next != memory.initial.begin()) { + const auto previous = std::prev(next); + if (address - previous->first < previous->second.size()) throw std::runtime_error("AGC driver: captured callee code aliases a registered shader snapshot"); + } + memory.callCodeReads.push_back(address); + return read(context, address, value); + }; + return ShaderRecompiler::ResolveUserDataCalls(request, reader, this); +} + std::shared_ptr ShaderMemory::capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation) { // The capture's word and page reads (through `read`) are attributed to it ([hooksync], [guestmem]). const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); @@ -234,6 +250,14 @@ std::shared_ptr ShaderMemory::capture(c runtime.readMemory = &read; runtime.readSpecializationMemory = &read; auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); + if (!callCodeReads.empty()) { + auto traced = std::make_shared(*capture); + auto& reads = traced->readTrace.otherReads; + reads.insert(reads.end(), callCodeReads.begin(), callCodeReads.end()); + std::sort(reads.begin(), reads.end()); + reads.erase(std::unique(reads.begin(), reads.end()), reads.end()); + capture = std::move(traced); + } if (profile) { totals.captureNanoseconds += NanosecondsSince(started); totals.resolveNanoseconds += capture->sourceNanoseconds; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 0d8e318a3b9e..2961887412a6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -916,12 +916,12 @@ void verifyPixelRequestSerialization() { minimal.context.waveSize = 64; minimal.context.pixel = ShaderPixelStageInfo{}; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQQ0AAAA=", "new requests did not use serialization version 13"); + require(requestPrefix(encoded, 8u) == "NVNQQQ4AAAA=", "new requests did not use serialization version 14"); constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u; for (std::size_t bytes = 0; bytes < 8u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated version-12 pixel mapping was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQQ4AAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQQ8AAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/UserDataSwappc.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/UserDataSwappc.cpp new file mode 100644 index 000000000000..727fc419e4ad --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/UserDataSwappc.cpp @@ -0,0 +1,203 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +alignas(256) std::array Output; +alignas(256) std::array First{0x80148714u, 0xbe80200eu}; +alignas(256) std::array Second{0x80148b14u, 0xbe80200eu}; + +void CaptureChecks(AgcDriver::VulkanDevice& device) { + using AgcDriver::DriverDetail::InvocationFor; + using AgcDriver::DriverDetail::ShaderSnapshot; + ShaderSnapshot snapshot{0x10000u, 0, 0, {0xbe940382u, 0xbe8e0304u, 0xbe8f0305u, 0xbe8e210eu, 0xbf810000u}, {}}; + std::array users{}; + const auto address = reinterpret_cast(First.data()); + users[4] = static_cast(address); + users[5] = static_cast(address >> 32u); + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Compute, snapshot.codeAddress, snapshot.code, 0, {}}; + request.context.userData = users; + request.context.waveSize = 32; + request.context.compute = ShaderRecompiler::ShaderComputeStageInfo{{1, 1, 1}, 0, {false, false, false}, false, 0}; + request.target = device.ComputeTarget(32); + request.layout = {0, 0, 0, 128}; + AgcDriver::ShaderMemory memory({}); + const auto invocation = InvocationFor(snapshot, 0, request, &memory); + const auto capture = memory.Capture(invocation); + const auto handle = AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, invocation.Request()); + auto stale = invocation.Request(); + const auto otherAddress = reinterpret_cast(Second.data()); + users[4] = static_cast(otherAddress); + users[5] = static_cast(otherAddress >> 32u); + Require(!ShaderRecompiler::MatchesPreparedShader(stale, *handle), "prepared artifact accepts another invocation's function pointer"); + bool staleRejected = false; + try { + static_cast(ShaderRecompiler::PrepareShader(stale)); + } catch (const std::exception& error) { + staleRejected = std::string(error.what()).find("do not match invocation user data") != std::string::npos; + } + Require(staleRejected, "source cache accepts stale call bindings"); + users[4] = static_cast(address); + users[5] = static_cast(address >> 32u); + Require(std::ranges::find(capture->readTrace.otherReads, address) != capture->readTrace.otherReads.end(), "callee code may be masked as flat resource data"); + Require(std::ranges::find(capture->readTrace.otherReads, address + 4u) != capture->readTrace.otherReads.end(), "callee return bytes were not captured"); + const auto regions = memory.Regions(); + Require(std::ranges::any_of(regions, [&](const auto& region) { return region.guestAddress <= address && address - region.guestAddress + sizeof(First) <= region.bytes.size(); }), "dispatch cache does not observe callee code"); + const std::array frozenCode{{{address, std::as_bytes(std::span(First))}}}; + AgcDriver::ShaderMemory frozen(frozenCode); + bool aliasRejected = false; + try { + static_cast(InvocationFor(snapshot, 0, request, &frozen)); + } catch (const std::exception& error) { + aliasRejected = std::string(error.what()).find("aliases a registered shader snapshot") != std::string::npos; + } + Require(aliasRejected, "callee bytes were read from an immutable registration snapshot"); + const auto reject = [&](std::uint64_t target, const char* expected) { + users[4] = static_cast(target); + users[5] = static_cast(target >> 32u); + std::string previous; + for (unsigned attempt = 0; attempt < 2; ++attempt) { + AgcDriver::ShaderMemory invalid({}); + bool rejected = false; + try { + static_cast(InvocationFor(snapshot, 0, request, &invalid)); + } catch (const std::exception& error) { + const std::string reason = error.what(); + Require(reason.find(expected) != std::string::npos, "invalid target raised the wrong error: " + reason); + Require(previous.empty() || previous == reason, "invalid target did not fail deterministically"); + previous = reason; + rejected = true; + } + Require(rejected, "invalid target was accepted"); + } + Require(!previous.empty(), "invalid target was accepted"); + }; + reject(0, "null or misaligned"); + reject(address + 1u, "null or misaligned"); + reject(0x1000000000000000ull, "guest memory"); +} + +void Append(std::vector& code, std::uint32_t source, std::uint32_t output) { + code.insert(code.end(), {0xbe920300u | source, 0xbe930300u | (source + 1u), 0xbe8e0312u, 0xbe8f0313u, 0xbe8e210eu, + 0x7e020214u, 0xe0700000u | output * 8u, 0x80000100u, 0x80148314u, 0x7e020214u, 0xe0700004u | output * 8u, 0x80000100u}); +} + +void Run(std::span sources, bool loop, std::uint32_t wave) { + alignas(256) std::array code{}; + std::vector instructions{0xbe940382u, 0x7e000280u, 0xbe960382u}; + const auto loopStart = instructions.size(); + for (std::size_t index = 0; index < sources.size(); ++index) Append(instructions, sources[index], static_cast(index)); + if (loop) { + instructions.push_back(0x80968116u); + instructions.push_back(0xbf068016u); + const auto displacement = static_cast(loopStart) - static_cast(instructions.size()) - 1; + instructions.push_back(0xbf840000u | (static_cast(displacement) & 0xffffu)); + } + instructions.push_back(0xbf810000u); + Require(instructions.size() <= code.size(), "synthetic caller exceeds its storage"); + std::copy(instructions.begin(), instructions.end(), code.begin()); + struct Header { + Shader shader{}; + std::array registers{}; + ShaderSpecialRegs specials{}; + } header; + const auto address = reinterpret_cast(code.data()); + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18u; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.shader.specials = &header.specials; + header.specials.dispatch_modifier = wave == 32u ? 0x8000u : 0u; + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, + {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 24}}}; + AgcDriverRegisterShader_nid_postfix(&header.shader); + AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); + + std::array users{}; + const auto outputAddress = reinterpret_cast(Output.data()); + users[0] = static_cast(outputAddress); + users[1] = static_cast(outputAddress >> 32u) & 0xffffu; + users[2] = sizeof(Output); + users[3] = 0x31016facu; + const auto bind = [&](std::size_t index, const void* pointer) { + const auto value = reinterpret_cast(pointer); + users[index] = static_cast(value); + users[index + 1u] = static_cast(value >> 32u); + }; + bind(4, First.data()); + bind(8, Second.data()); + const auto dispatch = [&](std::uint32_t firstAdd, std::uint32_t secondAdd) { + Output.fill(0xdeadbeefu); + std::vector commands; + for (const auto reg : header.registers) commands.insert(commands.end(), {0xc0017600u, reg.offset, reg.value}); + for (std::size_t index = 0; index < users.size(); ++index) commands.insert(commands.end(), {0xc0017600u, 0x240u + static_cast(index), users[index]}); + commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x41u | (wave == 32u ? 0x8000u : 0u)}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(outputAddress, sizeof(Output)); + std::uint32_t expected = 2; + for (std::uint32_t iteration = 0; iteration < (loop ? 2u : 1u); ++iteration) { + for (std::size_t index = 0; index < sources.size(); ++index) { + expected += sources[index] == 4u ? firstAdd : secondAdd; + if (!loop || iteration == 1u) Require(Output[index * 2u] == expected, "callee result at site " + std::to_string(index) + " is " + std::to_string(Output[index * 2u]) + ", expected " + std::to_string(expected) + ", wave " + std::to_string(wave) + ", loop " + std::to_string(loop) + ", first add " + std::to_string(firstAdd) + ", second add " + std::to_string(secondAdd)); + expected += 3; + if (!loop || iteration == 1u) Require(Output[index * 2u + 1u] == expected, "caller result at site " + std::to_string(index) + " is " + std::to_string(Output[index * 2u + 1u]) + ", expected " + std::to_string(expected)); + } + } + }; + dispatch(7, 11); + dispatch(7, 11); + bind(4, Second.data()); + dispatch(11, 11); + bind(4, First.data()); + First[0] = 0x80148d14u; + dispatch(13, 11); + First[0] = 0x80148714u; + dispatch(7, 11); + Second[0] = 0x80149114u; + dispatch(7, 17); + Second[0] = 0x80148b14u; + dispatch(7, 11); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + CaptureChecks(*device); + const std::array single{4}; + const std::array multiple{4, 8, 8, 8}; + Run(single, false, 32); + Run(multiple, false, 32); + Run(multiple, true, 32); + Run(multiple, false, 64); + AgcDriverShutdown_nid_postfix(); + std::puts("user-data swappc dispatch and cache tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + try { AgcDriverShutdown_nid_postfix(); } catch (const std::exception&) {} + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp index b62ee64a7178..df22ae68b78d 100644 --- a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp @@ -3,6 +3,7 @@ #include "ControlFlow/GraphBuilder.hpp" #include "ControlFlow/RequestSerializer.hpp" #include "ControlFlow/Structurizer.hpp" +#include "ControlFlow/UserDataCalls.hpp" #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" #include "Recompiler.hpp" @@ -89,13 +90,15 @@ bool Replay(const char* path) { std::printf(" code -> %s\n", name.c_str()); } if (g_assembly) { - const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(request.request.shader.code); + const auto program = ShaderRecompiler::DecodeShaderProgram(request.request.shader); // Raw first words carry what the text omits (branch offsets, waitcnt fields). for (const auto& instruction : program.instructions) std::printf("raw=%08x %s\n", instruction.rawWords[0], ShaderRecompiler::RdnaInstructionToString(instruction).c_str()); } if (g_graph) { - const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(request.request.shader.code); - auto graph = ShaderRecompiler::GraphBuilder{}.Build(program); + const auto program = ShaderRecompiler::DecodeShaderProgram(request.request.shader); + const auto& context = request.request.context; + const ShaderRecompiler::SwappcInfo swappc{context.vertex.has_value(), context.userDataBaseRegister, static_cast(context.userData.size()), request.request.shader.capturedCalls}; + auto graph = ShaderRecompiler::GraphBuilder{}.Build(program, &swappc); std::printf("control flow graph:\n%s", ShaderRecompiler::GraphToString(graph).c_str()); try { ShaderRecompiler::Structurizer{}.Structurize(graph); diff --git a/core/libs/tests/UserDataSwappc.cpp b/core/libs/tests/UserDataSwappc.cpp new file mode 100644 index 000000000000..ba480701b178 --- /dev/null +++ b/core/libs/tests/UserDataSwappc.cpp @@ -0,0 +1,218 @@ +#include "ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/RequestSerializer.hpp" +#include "ControlFlow/UserDataCalls.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "CacheKey.hpp" +#include "ShaderDiskCache.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +void Require(bool value, const char* message) { + if (!value) throw std::runtime_error(message); +} + +template +void Reject(TFunction function, const char* message) { + try { + function(); + } catch (const std::exception& error) { + Require(std::string(error.what()).find(message) != std::string::npos, error.what()); + return; + } + throw std::runtime_error(std::string("negative control accepted: ") + message); +} + +struct Fixture { + std::vector code{0xbe920304u, 0xbe930305u, 0xbe8e0312u, 0xbe8f0313u, 0xbe8e210eu, 0x80148314u, 0xbf810000u}; + std::array users{}; + std::map> memory{{0x2000u, {0x80148714u, 0xbe80200eu}}, {0x3000u, {0x80148b14u, 0xbe80200eu}}}; + + Fixture() { users[4] = 0x2000u; users[8] = 0x3000u; } + + RecompileRequest Request() const { + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x1000u, code, 0, {}}; + request.context.userData = users; + request.context.waveSize = 32; + return request; + } + + static bool Read(void* context, std::uint64_t address, std::uint32_t* value) { + const auto& memory = static_cast(context)->memory; + for (const auto& [base, words] : memory) { + if (address >= base && address - base < words.size() * 4u) { + *value = words[(address - base) / 4u]; + return true; + } + } + return false; + } + + CapturedCallProgram Resolve() { return ResolveUserDataCalls(Request(), &Read, this); } +}; + +std::vector Key(const Fixture& fixture, const CapturedCallProgram& captured) { + auto request = fixture.Request(); + request.shader.code = captured.code; + request.shader.capturedCalls = captured.calls; + std::vector key; + RecompileCacheKey::Build(request, key); + return key; +} + +std::vector DiskKey(const Fixture& fixture, const CapturedCallProgram& captured) { + auto request = fixture.Request(); + request.shader.code = captured.code; + request.shader.capturedCalls = captured.calls; + std::vector key; + ShaderDiskCache::BuildKey(request, 32, key); + return key; +} + +void CallsAndContinuations() { + Fixture fixture; + const auto original = RdnaInstructionDecoder{}.Decode(fixture.code); + const SwappcInfo info{false, 0, 12}; + Reject([&] { static_cast(GraphBuilder{}.Build(original, &info)); }, "not statically resolvable"); + fixture.code = {0xbe8e0404u, 0xbe8e210eu, 0x80148314u, 0xbe8e0408u, 0xbe8e210eu, 0x80148314u, + 0xbe8e0408u, 0xbe8e210eu, 0x80148314u, 0xbe8e0408u, 0xbe8e210eu, 0x80148314u, 0xbf810000u}; + const auto captured = fixture.Resolve(); + Require(captured.calls.size() == 4u, "call sites were lost"); + Require(captured.calls[1].targetAddress == captured.calls[2].targetAddress, "same callee was not reused"); + Require(captured.calls[1].targetProgramCounter != captured.calls[2].targetProgramCounter, "same callee was not cloned per call site"); + auto request = fixture.Request(); + request.shader.code = captured.code; + request.shader.capturedCalls = captured.calls; + auto boundInfo = info; + boundInfo.capturedCalls = captured.calls; + const auto graph = GraphBuilder{}.Build(DecodeShaderProgram(request.shader), &boundInfo); + for (const auto& call : captured.calls) { + const auto returns = std::ranges::count_if(graph.blocks, [&](const auto& block) { + return block.startProgramCounter <= call.returnProgramCounter && block.endProgramCounter > call.returnProgramCounter && + block.successors.size() == 1 && graph.blocks[block.successors.front()].startProgramCounter == call.callProgramCounter + 4u; + }); + Require(returns == 1, "callee returns to the wrong continuation"); + } + const auto serialized = RequestSerializer{}.Serialize(request); + const auto restored = RequestSerializer{}.Deserialize(serialized); + Require(std::ranges::equal(restored.request.shader.capturedCalls, captured.calls), "capture serialization lost call bindings"); + Require(std::ranges::equal(restored.request.shader.code, captured.code), "capture serialization lost callee bytes"); + auto broken = captured; + broken.calls[2].returnProgramCounter = broken.calls[1].returnProgramCounter; + request.shader.capturedCalls = broken.calls; + Reject([&] { static_cast(DecodeShaderProgram(request.shader)); }, "boundaries"); +} + +void CacheIdentity() { + Fixture fixture; + const auto first = fixture.Resolve(); + const auto firstKey = Key(fixture, first); + const auto firstDiskKey = DiskKey(fixture, first); + fixture.users[4] = 0x3000u; + const auto second = fixture.Resolve(); + Require(firstKey != Key(fixture, second), "different targets reuse the same source key"); + Require(firstDiskKey != DiskKey(fixture, second), "different targets reuse the same disk key"); + fixture.users[4] = 0x2000u; + fixture.memory[0x2000u][0] = 0x80148b14u; + Require(firstKey != Key(fixture, fixture.Resolve()), "changed callee bytes reuse the same source key"); + Require(firstDiskKey != DiskKey(fixture, fixture.Resolve()), "changed callee bytes reuse the same disk key"); + fixture.memory[0x2000u] = fixture.memory[0x3000u]; + const auto sameBytes = Key(fixture, fixture.Resolve()); + fixture.users[4] = 0x3000u; + Require(sameBytes != Key(fixture, fixture.Resolve()), "target address is absent from the source key"); + auto changed = first; + changed.calls[0].targetProgramCounter += 4; + Require(firstKey != Key(fixture, changed), "call target mapping is absent from the key"); + changed = first; + changed.calls[0].returnProgramCounter += 4; + Require(firstKey != Key(fixture, changed), "call return mapping is absent from the key"); + changed = first; + changed.calls[0].userDataIndex += 2; + Require(firstKey != Key(fixture, changed), "call provenance is absent from the key"); +} + +void Failures() { + Fixture fixture; + fixture.users[4] = 0; + Reject([&] { static_cast(fixture.Resolve()); }, "null or misaligned"); + fixture.users[4] = 0x2001u; + Reject([&] { static_cast(fixture.Resolve()); }, "null or misaligned"); + fixture.users[4] = 0x4000u; + Reject([&] { static_cast(fixture.Resolve()); }, "unmapped"); + fixture.users[4] = 0x1000u; + Reject([&] { static_cast(fixture.Resolve()); }, "recursive"); + fixture.users[4] = 0xfffffffcu; + fixture.users[5] = 0xffffffffu; + Reject([&] { static_cast(fixture.Resolve()); }, "overflow"); + fixture.users[4] = 0x2000u; + fixture.users[5] = 0; + fixture.memory[0x2000u] = {0xffffffffu}; + Reject([&] { static_cast(fixture.Resolve()); }, "unknown RDNA"); + fixture.memory[0x2000u] = {0xbe9403ffu}; + Reject([&] { static_cast(fixture.Resolve()); }, "unmapped"); + fixture.memory[0x2000u] = {0xbe80200cu}; + Reject([&] { static_cast(fixture.Resolve()); }, "unsupported return"); + fixture.memory[0x2000u] = {0xbf810000u}; + Reject([&] { static_cast(fixture.Resolve()); }, "without returning"); + fixture.memory[0x2000u] = {0xbe8e2104u}; + Reject([&] { static_cast(fixture.Resolve()); }, "nested scalar calls"); + fixture.memory[0x2000u] = {0xbe941f00u}; + Reject([&] { static_cast(fixture.Resolve()); }, "PC-relative"); + fixture.memory[0x2000u] = {0xbe840380u, 0xbe80200eu}; + Reject([&] { static_cast(fixture.Resolve()); }, "clobbers a user-data call target"); + fixture.memory[0x2000u] = std::vector(16384, 0xbf800000u); + Reject([&] { static_cast(fixture.Resolve()); }, "size limit"); + fixture.memory[0x2000u] = {0x80148714u, 0xbe80200eu}; + fixture.code[2] = 0x800e0112u; + Reject([&] { static_cast(fixture.Resolve()); }, "not statically resolvable"); + fixture = Fixture{}; + fixture.code[2] = 0x7e1c0500u; + Reject([&] { static_cast(fixture.Resolve()); }, "not statically resolvable"); + fixture = Fixture{}; + fixture.code.insert(fixture.code.begin(), 0xbf840002u); + Reject([&] { static_cast(fixture.Resolve()); }, "not statically resolvable"); + fixture = Fixture{}; + fixture.code.insert(fixture.code.end() - 1, 0x800e810eu); + Reject([&] { static_cast(fixture.Resolve()); }, "link escapes"); + fixture = Fixture{}; + fixture.code.resize(262144); + fixture.memory[0x200000u] = fixture.memory[0x2000u]; + fixture.users[4] = 0x200000u; + Reject([&] { static_cast(fixture.Resolve()); }, "combined program size limit"); + fixture = Fixture{}; + fixture.code.clear(); + for (unsigned index = 0; index <= MaxCapturedShaderCalls; ++index) fixture.code.insert(fixture.code.end(), {0xbe8e0304u, 0xbe8f0305u, 0xbe8e210eu}); + fixture.code.push_back(0xbf810000u); + Reject([&] { static_cast(fixture.Resolve()); }, "too many captured scalar calls"); +} + +} + +int main() { + try { + Fixture fixture; + Require(fixture.Resolve().calls.size() == 1, "SGPR copy chain was not resolved"); + fixture.code = {0xbe842104u, 0xbf810000u}; + fixture.memory[0x2000u] = {0x80148714u, 0xbe802004u}; + Require(fixture.Resolve().calls.size() == 1, "direct user-data target sharing its link pair was not resolved"); + fixture.code = {0xbe842104u, 0xbf82fffeu, 0xbf810000u}; + Reject([&] { static_cast(fixture.Resolve()); }, "not statically resolvable"); + CallsAndContinuations(); + CacheIdentity(); + Failures(); + std::puts("user-data swappc analysis tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 324eca0f0dcc..caef8327a8bf 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -47,6 +47,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/RdnaDecoder/src/RdnaDescriptorFormat.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ControlFlow/src/ControlFlowGraph.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ControlFlow/src/GraphBuilder.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ControlFlow/src/UserDataCalls.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ControlFlow/src/Structurizer.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ControlFlow/src/RequestSerializer.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/IntermediateRepresentation/src/IrType.cpp @@ -206,6 +207,7 @@ function(add_shader_recompiler target runtimeTarget) string(TOUPPER "${CMAKE_BUILD_TYPE}" shaderCacheBuildType) set(shaderCacheExtra "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_BUILD_TYPE} spirv-tools=${ANYPS5_ENABLE_SPIRV_TOOLS} ${CMAKE_SYSTEM_PROCESSOR} flags=${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${shaderCacheBuildType}}") set(shaderCacheReviewedNames + AnalyzeUserDataCalls BuildPreparedShaderKey BuildRectListShaders CaptureResources @@ -222,11 +224,13 @@ function(add_shader_recompiler target runtimeTarget) RecompileResult PrepareShader GetPreparedCode + GetPreparedCalls PreparedShaderInvocation::TryCreate MatchesPreparedShader GetPreparedArtifact MaterializeShader ResolveSource + ResolveUserDataCalls ResourceMaterializer::SpecializationNanoseconds RuntimeAbi::RequireVersion RuntimeAbi::HeapCapacity diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 4207ad6ab22f..f0d38172bb81 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -46,6 +46,7 @@ class RecompileCacheKey { append(key, RuntimeAbi::Version); append(key, request.shader.stage); append(key, request.context.waveSize); + appendCalls(key, request); append(key, request.context.userDataBaseRegister); append(key, request.context.userData.size()); append(key, request.context.compute); @@ -76,7 +77,19 @@ class RecompileCacheKey { } private: + static void appendCalls(std::vector& key, const RecompileRequest& request) { + append(key, request.shader.capturedCalls.size()); + for (const auto& call : request.shader.capturedCalls) { + append(key, call.callProgramCounter); + append(key, call.targetProgramCounter); + append(key, call.returnProgramCounter); + append(key, call.targetAddress); + append(key, call.userDataIndex); + } + } + static void appendInterface(std::vector& key, const RecompileRequest& request) { + appendCalls(key, request); append(key, request.context.waveSize); append(key, request.context.userDataBaseRegister); append(key, request.context.userData.size()); diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index bca0da8c96ea..8c7ec3626e8c 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -3,6 +3,7 @@ #include "ControlFlow/ControlFlowGraph.hpp" #include "RdnaDecoder/RdnaProgram.hpp" +#include "Recompiler.hpp" #include #include @@ -12,6 +13,7 @@ struct SwappcInfo { bool fetchCallAllowed = false; std::uint32_t userDataBaseRegister = 0; std::uint32_t userDataCount = 0; + std::span capturedCalls{}; }; struct SwappcCall { @@ -22,8 +24,17 @@ struct SwappcCall { std::uint32_t targetProgramCounter = 0; std::uint32_t returnIndex = 0; std::uint32_t returnTargetProgramCounter = 0; + bool captured = false; }; +struct UserDataCall { + std::uint32_t callIndex; + std::uint32_t userDataIndex; +}; + +[[nodiscard]] std::vector AnalyzeUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc); +[[nodiscard]] bool WritesScalarRegister(const RdnaInstruction& instruction, std::uint32_t index); + class GraphBuilder { public: [[nodiscard]] ControlFlowGraph Build(const RdnaProgram& program, const SwappcInfo* swappc = nullptr) const; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp index 0f59c68558e0..23bb041e48dd 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp @@ -14,6 +14,7 @@ namespace ShaderRecompiler { std::vector code; std::vector header; std::vector userData; + std::vector calls; }; struct DeserializedGraphicsCompileContext { @@ -27,6 +28,7 @@ namespace ShaderRecompiler { struct DeserializedRequest { std::vector shaderCode; std::vector shaderHeader; + std::vector calls; std::vector userData; std::optional compute; std::optional pixel; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/UserDataCalls.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/UserDataCalls.hpp new file mode 100644 index 000000000000..ef3ac9a94ca8 --- /dev/null +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/UserDataCalls.hpp @@ -0,0 +1,20 @@ +#ifndef CORE_SHADER_RECOMPILER_CONTROLFLOW_USERDATACALLS_HPP +#define CORE_SHADER_RECOMPILER_CONTROLFLOW_USERDATACALLS_HPP + +#include "Recompiler.hpp" +#include "Optimization/SrtWalker.hpp" +#include "RdnaDecoder/RdnaProgram.hpp" + +namespace ShaderRecompiler { + +struct CapturedCallProgram { + std::vector code; + std::vector calls; +}; + +[[nodiscard]] CapturedCallProgram ResolveUserDataCalls(const RecompileRequest& request, SrtMemoryReader reader, void* context); +[[nodiscard]] RdnaProgram DecodeShaderProgram(const ShaderBinary& shader); + +} + +#endif diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 1bd3c7776f74..565025c6d59a 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -477,12 +477,37 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa SwappcCall call; call.callIndex = index; call.linkRegister = linkRegister; + const CapturedShaderCall* captured = nullptr; + if (swappc != nullptr) { + for (const auto& candidate : swappc->capturedCalls) { + if (candidate.callProgramCounter == instruction.programCounter) { + if (captured != nullptr) throw std::invalid_argument("duplicate captured scalar call"); + captured = &candidate; + } + } + } + if (captured != nullptr) { + if (instruction.op != RdnaOpcode::SSwappcB64) throw std::invalid_argument("captured scalar call is not s_swappc_b64"); + call.captured = true; + call.targetProgramCounter = captured->targetProgramCounter; + call.targetIndex = instructionIndexOfProgramCounter(program, captured->targetProgramCounter); + call.returnIndex = instructionIndexOfProgramCounter(program, captured->returnProgramCounter); + call.returnTargetProgramCounter = instructionEndProgramCounter(instruction); + if (call.targetIndex == InvalidControlFlowId || call.returnIndex == InvalidControlFlowId || call.targetIndex > call.returnIndex || + program.instructions[call.returnIndex].op != RdnaOpcode::SSetpcB64 || !isScalar(program.instructions[call.returnIndex].source0, linkRegister)) { + throw std::invalid_argument("invalid captured scalar call region"); + } + } for (const auto& other : calls) { - if (call.linkRegister <= other.linkRegister + 1u && other.linkRegister <= call.linkRegister + 1u) { + if ((!call.captured || !other.captured) && call.linkRegister <= other.linkRegister + 1u && other.linkRegister <= call.linkRegister + 1u) { throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(instruction.programCounter) + ": link register s[" + std::to_string(call.linkRegister) + "] is shared with another call"); } } + if (call.captured) { + calls.push_back(call); + continue; + } std::uint32_t target = instruction.branchTarget; const bool staticTarget = instruction.op == RdnaOpcode::SCallB64 || resolveSetpcTarget(program, index, target); call.targetIndex = staticTarget ? instructionIndexOfProgramCounter(program, target) : InvalidControlFlowId; @@ -511,6 +536,23 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa } for (auto& call : calls) { + if (call.captured) { + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + if (index == call.callIndex || index == call.returnIndex) continue; + const auto& instruction = program.instructions[index]; + const bool inside = index >= call.targetIndex && index <= call.returnIndex; + const auto* otherCall = findCallAt(calls, index); + const auto* otherReturn = findReturnAt(calls, index); + if (!inside && ((otherCall != nullptr && otherCall->captured) || (otherReturn != nullptr && otherReturn->captured))) continue; + auto reads = instruction; + reads.destination = {}; + reads.destination2 = {}; + if ((inside && touchesLinkRegister(instruction, call.linkRegister)) || (!inside && touchesLinkRegister(reads, call.linkRegister))) { + throw std::invalid_argument("captured scalar call link escapes the call/return model at program counter " + toHexString(instruction.programCounter)); + } + } + continue; + } std::uint32_t returnIndex = InvalidControlFlowId; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { if (index == call.callIndex) { @@ -764,7 +806,7 @@ std::vector GraphBuilder::splitIntoBlocks(const RdnaProgram& program } } } else if (instruction.op == RdnaOpcode::SEndpgm) { - labels.insert(nextProgramCounter); + if (nextProgramCounter == endProgramCounter || instructionProgramCounters.contains(nextProgramCounter)) labels.insert(nextProgramCounter); } } @@ -890,6 +932,63 @@ void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram } } +bool WritesScalarRegister(const RdnaInstruction& instruction, std::uint32_t index) { + return writesScalar(instruction, index); +} + +std::vector AnalyzeUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + std::vector result; + const auto immutableUserData = [&](std::uint32_t reg) { + return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount && + std::ranges::none_of(program.instructions, [&](const auto& instruction) { return writesScalar(instruction, reg); }); + }; + const auto origin = [&](std::uint32_t callIndex, std::uint32_t reg) { + for (auto index = callIndex; ; ) { + if (immutableUserData(reg)) return reg; + if (index == 0u) { + const bool user = reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; + const bool revisited = std::ranges::any_of(program.instructions, [&](const auto& branch) { + return IsDirectBranchOpcode(branch.op) && branch.programCounter >= program.instructions[callIndex].programCounter && branch.branchTarget <= program.instructions[callIndex].programCounter; + }); + return user && !revisited ? reg : NoScalarRegister; + } + const auto& instruction = program.instructions[--index]; + if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SSetpcB64 || instruction.op == RdnaOpcode::SSwappcB64 || + instruction.op == RdnaOpcode::SCallB64 || instruction.op == RdnaOpcode::SEndpgm) return NoScalarRegister; + if (!writesScalar(instruction, reg)) continue; + if ((instruction.op != RdnaOpcode::SMovB32 && instruction.op != RdnaOpcode::SMovB64) || + instruction.destination.kind != RdnaOperandKind::ScalarRegister || instruction.source0.kind != RdnaOperandKind::ScalarRegister) return NoScalarRegister; + if (instruction.op == RdnaOpcode::SMovB64 && ((instruction.destination.reg & 1u) != 0u || (instruction.source0.reg & 1u) != 0u)) return NoScalarRegister; + if (std::ranges::any_of(program.instructions, [&](const auto& branch) { + return IsDirectBranchOpcode(branch.op) && branch.branchTarget > instruction.programCounter && branch.branchTarget <= program.instructions[callIndex].programCounter; + })) return NoScalarRegister; + reg = instruction.source0.reg + reg - instruction.destination.reg; + } + }; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (instruction.op != RdnaOpcode::SSwappcB64) continue; + std::uint32_t target = 0; + const bool resolved = resolveSetpcTarget(program, index, target); + if (resolved && instructionIndexOfProgramCounter(program, target) != InvalidControlFlowId) continue; + const auto targetRegister = scalarIndex(instruction.source0); + const bool positional = std::all_of(program.instructions.begin(), program.instructions.begin() + index, isFetchCallPrefixOpcode); + const bool userPair = targetRegister != NoScalarRegister && immutableUserData(targetRegister) && immutableUserData(targetRegister + 1u); + if (swappc.fetchCallAllowed && positional && (userPair || resolved)) continue; + if (targetRegister == NoScalarRegister || targetRegister > 104u || (targetRegister & 1u) != 0u) throw std::invalid_argument("unsupported captured scalar call target register"); + const auto low = targetRegister == NoScalarRegister ? NoScalarRegister : origin(index, targetRegister); + const auto high = targetRegister == NoScalarRegister ? NoScalarRegister : origin(index, targetRegister + 1u); + if (low == NoScalarRegister || high != low + 1u) { + throw std::invalid_argument("computed/data-dependent s_swappc_b64 call target at program counter " + toHexString(instruction.programCounter) + " is not statically resolvable"); + } + const auto link = scalarIndex(instruction.destination); + if (link == NoScalarRegister || link % 2u != 0u || link > 104u) throw std::invalid_argument("unsupported captured scalar call link register"); + result.push_back({index, low - swappc.userDataBaseRegister}); + if (result.size() > MaxCapturedShaderCalls) throw std::invalid_argument("too many captured scalar calls"); + } + return result; +} + ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program, const SwappcInfo* swappc) const { const std::vector calls = analyzeSwappcCalls(program, swappc); ControlFlowGraph graph; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index 48386cd206b1..0aebddd7d3e9 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -714,7 +714,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(13u); + writer.WriteU32(14u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -733,6 +733,18 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteU32(request.target.srgbDecodeFormats); if (request.context.compute.has_value()) writer.WriteU32(request.context.compute->scratchDwords); writer.WriteBool(request.target.narrowSubgroupClock); + const auto writeCalls = [&](std::span calls) { + writer.WriteU32(static_cast(calls.size())); + for (const auto& call : calls) { + writer.WriteU32(call.callProgramCounter); + writer.WriteU32(call.targetProgramCounter); + writer.WriteU32(call.returnProgramCounter); + writer.WriteU64(call.targetAddress); + writer.WriteU32(call.userDataIndex); + } + }; + writeCalls(request.shader.capturedCalls); + if (request.graphics) for (const auto& program : request.graphics->linkedPrograms) writeCalls(program.binary.capturedCalls); return base64Encode(buffer); } @@ -741,7 +753,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 13u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 14u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -761,6 +773,30 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const if (version >= 9u) result.request.target.srgbDecodeFormats = reader.ReadU32(); if (version >= 10u && result.request.context.compute.has_value()) result.request.context.compute->scratchDwords = reader.ReadU32(); if (version >= 12u) result.request.target.narrowSubgroupClock = reader.ReadBool(); + if (version >= 14u) { + const auto readCalls = [&](std::vector& calls) { + const auto count = reader.ReadU32(); + if (count > MaxCapturedShaderCalls) throw std::runtime_error("RequestSerializer: too many captured scalar calls"); + for (std::uint32_t index = 0; index < count; ++index) { + CapturedShaderCall call; + call.callProgramCounter = reader.ReadU32(); + call.targetProgramCounter = reader.ReadU32(); + call.returnProgramCounter = reader.ReadU32(); + call.targetAddress = reader.ReadU64(); + call.userDataIndex = reader.ReadU32(); + calls.push_back(call); + } + }; + readCalls(result.calls); + result.request.shader.capturedCalls = result.calls; + if (result.graphicsStorage) { + for (std::size_t index = 0; index < result.graphicsStorage->linkedPrograms.size(); ++index) { + auto& calls = result.graphicsStorage->linkedProgramStorage[index].calls; + readCalls(calls); + result.graphicsStorage->linkedPrograms[index].binary.capturedCalls = calls; + } + } + } return result; } diff --git a/core/shader/recompiler/ControlFlow/src/UserDataCalls.cpp b/core/shader/recompiler/ControlFlow/src/UserDataCalls.cpp new file mode 100644 index 000000000000..92991430e4e7 --- /dev/null +++ b/core/shader/recompiler/ControlFlow/src/UserDataCalls.cpp @@ -0,0 +1,125 @@ +#include "ControlFlow/UserDataCalls.hpp" +#include "ControlFlow/GraphBuilder.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include +#include +#include + +namespace ShaderRecompiler { + +namespace { + +constexpr std::size_t MaxCalleeWords = 16384; +constexpr std::size_t MaxCombinedWords = 262144; + +void requireLeafInstruction(const RdnaInstruction& instruction, bool last, std::uint32_t link) { + if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { + throw std::invalid_argument("nested scalar calls in a captured callee are unsupported"); + } + if (instruction.op == RdnaOpcode::SGetpcB64) throw std::invalid_argument("PC-relative captured callees are unsupported"); + if (instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd) { + throw std::invalid_argument("captured callee ends without returning"); + } + if (instruction.op == RdnaOpcode::SSetpcB64 && (!last || instruction.source0.kind != RdnaOperandKind::ScalarRegister || instruction.source0.reg != link)) { + throw std::invalid_argument("captured callee has an unsupported return"); + } +} + +} + +CapturedCallProgram ResolveUserDataCalls(const RecompileRequest& request, SrtMemoryReader reader, void* context) { + if (!request.shader.capturedCalls.empty()) throw std::invalid_argument("shader calls have already been captured"); + if (request.shader.code.size() > MaxCombinedWords) throw std::invalid_argument("captured shader exceeds the combined program size limit"); + const auto program = RdnaInstructionDecoder{}.Decode(request.shader.code); + const SwappcInfo info{request.context.vertex.has_value(), request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + const auto targets = AnalyzeUserDataCalls(program, info); + CapturedCallProgram result; + if (targets.empty()) return result; + if (reader == nullptr) throw std::invalid_argument("captured scalar calls require a memory reader"); + result.code.assign(request.shader.code.begin(), request.shader.code.end()); + for (const auto& target : targets) { + const auto address = static_cast(request.context.userData[target.userDataIndex]) | + (static_cast(request.context.userData[target.userDataIndex + 1u]) << 32u); + if (address == 0 || (address & 3u) != 0u) throw std::invalid_argument("captured scalar call target is null or misaligned"); + if (address >= request.shader.codeAddress && address - request.shader.codeAddress < request.shader.code.size_bytes()) { + throw std::invalid_argument("recursive captured scalar call targets the caller"); + } + const auto& call = program.instructions[target.callIndex]; + const auto start = static_cast(result.code.size() * 4u); + std::vector callee; + bool returned = false; + while (callee.size() < MaxCalleeWords) { + const auto wordIndex = static_cast(callee.size()); + RdnaInstruction instruction; + for (std::size_t words = 0; ; ++words) { + if (words >= MaxRdnaInstructionRawWords || callee.size() >= MaxCalleeWords) throw std::invalid_argument("captured callee instruction exceeds the size limit"); + const auto offset = callee.size() * 4u; + if (address > std::numeric_limits::max() - offset - 4u) throw std::invalid_argument("captured scalar call target arithmetic overflow"); + std::uint32_t word = 0; + if (!reader(context, address + offset, &word)) throw std::invalid_argument("captured scalar call target is unmapped"); + callee.push_back(word); + try { + instruction = DecodeRdnaInstruction(wordIndex * 4u, callee, wordIndex); + break; + } catch (const std::out_of_range&) { + if (words + 1u == MaxRdnaInstructionRawWords) throw; + } + } + returned = instruction.op == RdnaOpcode::SSetpcB64; + requireLeafInstruction(instruction, returned, call.destination.reg); + for (const auto& source : targets) { + const auto reg = request.context.userDataBaseRegister + source.userDataIndex; + if (WritesScalarRegister(instruction, reg) || WritesScalarRegister(instruction, reg + 1u)) { + throw std::invalid_argument("captured callee clobbers a user-data call target"); + } + } + if (returned) break; + } + if (!returned) throw std::invalid_argument("captured callee exceeds the size limit without returning"); + if (callee.size() > MaxCombinedWords - result.code.size()) throw std::invalid_argument("captured shader exceeds the combined program size limit"); + result.calls.push_back({call.programCounter, start, static_cast(start + callee.size() * 4u - 4u), address, target.userDataIndex}); + result.code.insert(result.code.end(), callee.begin(), callee.end()); + } + auto shader = request.shader; + shader.code = result.code; + shader.capturedCalls = result.calls; + const auto combined = DecodeShaderProgram(shader); + auto capturedInfo = info; + capturedInfo.capturedCalls = result.calls; + static_cast(GraphBuilder{}.Build(combined, &capturedInfo)); + return result; +} + +RdnaProgram DecodeShaderProgram(const ShaderBinary& shader) { + if (shader.capturedCalls.size() > MaxCapturedShaderCalls) throw std::invalid_argument("too many captured scalar calls"); + if (!shader.capturedCalls.empty() && shader.code.size() > MaxCombinedWords) throw std::invalid_argument("captured shader exceeds the combined program size limit"); + auto program = RdnaInstructionDecoder{}.Decode(shader.code); + const auto callerEnd = program.instructions.back().programCounter + program.instructions.back().wordCount * 4u; + auto end = callerEnd; + for (const auto& call : shader.capturedCalls) { + const auto* instruction = FindInstructionAtProgramCounter(program, call.callProgramCounter); + if (instruction == nullptr || instruction->op != RdnaOpcode::SSwappcB64 || call.callProgramCounter >= callerEnd || + call.targetProgramCounter < end || call.returnProgramCounter < call.targetProgramCounter || (call.targetProgramCounter & 3u) != 0 || + (call.returnProgramCounter & 3u) != 0 || static_cast(call.returnProgramCounter) + 4u > shader.code.size_bytes()) { + throw std::invalid_argument("invalid captured scalar call boundaries"); + } + const auto bytes = static_cast(call.returnProgramCounter) + 4u - call.targetProgramCounter; + if (bytes > MaxCalleeWords * 4u || call.targetAddress == 0 || (call.targetAddress & 3u) != 0 || call.targetAddress > std::numeric_limits::max() - bytes) { + throw std::invalid_argument("invalid captured scalar call target or size"); + } + const auto link = instruction->destination.reg; + for (auto pc = call.targetProgramCounter; pc <= call.returnProgramCounter; ) { + auto decoded = DecodeRdnaInstruction(pc, shader.code, pc / 4u); + const bool last = pc == call.returnProgramCounter; + requireLeafInstruction(decoded, last, link); + if (last && decoded.op != RdnaOpcode::SSetpcB64) throw std::invalid_argument("captured callee has no return"); + if (decoded.wordCount * 4u > call.returnProgramCounter + 4u - pc) throw std::invalid_argument("captured callee return is not an instruction boundary"); + pc += decoded.wordCount * 4u; + program.instructions.push_back(std::move(decoded)); + } + end = call.returnProgramCounter + 4u; + } + return program; +} + +} diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index a0c470a7bfae..fddeaf4fae36 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -54,6 +54,7 @@ class PreparedShaderInvocation { std::shared_ptr handle; }; [[nodiscard]] std::span GetPreparedCode(const SourceHandle& handle); +[[nodiscard]] std::span GetPreparedCalls(const SourceHandle& handle); [[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request); [[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle); void BuildPreparedShaderKey(const RecompileRequest& request, std::vector& key); diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 358b31a2427f..0bb0cb49f1d4 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -18,6 +18,7 @@ #include #include #include "ControlFlow/include/ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/UserDataCalls.hpp" #include "ControlFlow/include/ControlFlow/Structurizer.hpp" #include "RdnaDecoder/include/RdnaDecoder/RdnaInstructionDecoder.hpp" #include "IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp" @@ -95,20 +96,39 @@ ShaderStageInputInfo RequestInputInfo(const RecompileRequest& request) { return BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request), mesh, tessellation); } +bool capturedCallsMatchUserData(std::span calls, std::span userData) { + for (const auto& call : calls) { + if (call.userDataIndex >= userData.size() || userData.size() - call.userDataIndex < 2u) return false; + const auto address = static_cast(userData[call.userDataIndex]) | (static_cast(userData[call.userDataIndex + 1u]) << 32u); + if (address != call.targetAddress) return false; + } + return true; +} + } IrProgram PrepareResourceProgram(const RecompileRequest& request) { + if (!capturedCallsMatchUserData(request.shader.capturedCalls, request.context.userData)) throw std::invalid_argument("captured scalar call targets do not match invocation user data"); const auto stageKind = toShaderStageKind(request.shader.stage); const auto inputInfo = RequestInputInfo(request); - constexpr RdnaInstructionDecoder decoder; - const auto decoded = decoder.Decode(request.shader.code); + const auto decoded = DecodeShaderProgram(request.shader); constexpr GraphBuilder graphBuilder; SwappcInfo swappcInfo; swappcInfo.fetchCallAllowed = inputInfo.vertex != nullptr; swappcInfo.userDataBaseRegister = request.context.userDataBaseRegister; swappcInfo.userDataCount = static_cast(request.context.userData.size()); + swappcInfo.capturedCalls = request.shader.capturedCalls; + if (!request.shader.capturedCalls.empty()) { + const auto targets = AnalyzeUserDataCalls(decoded, swappcInfo); + if (targets.size() != request.shader.capturedCalls.size()) throw std::invalid_argument("captured scalar call provenance does not match the program"); + for (std::size_t index = 0; index < targets.size(); ++index) { + if (decoded.instructions[targets[index].callIndex].programCounter != request.shader.capturedCalls[index].callProgramCounter || targets[index].userDataIndex != request.shader.capturedCalls[index].userDataIndex) { + throw std::invalid_argument("captured scalar call provenance does not match the call site"); + } + } + } auto cfg = graphBuilder.Build(decoded, &swappcInfo); constexpr Structurizer structurizer; @@ -206,6 +226,7 @@ struct SourceEntry { // accepted only when its code matches word for word. Owned here because the request's span // points into a registration the driver may replace while the entry lives on. std::vector code; + std::vector capturedCalls; std::shared_ptr plan; // A plan build that threw (an unsupported resource chain or control flow) is remembered and // rethrown: the front end ran every pass before failing, ~13 ms per dispatch of a shader the @@ -249,6 +270,7 @@ bool FailureMemo() { } std::shared_ptr getSource(const RecompileRequest& request) { + if (!capturedCallsMatchUserData(request.shader.capturedCalls, request.context.userData)) throw std::invalid_argument("captured scalar call targets do not match invocation user data"); static std::shared_mutex mutex; // Entries whose code hashes alike share a bucket; the code comparison picks the right one. static std::unordered_map, std::vector>, SourceKeyHash> sources; @@ -275,6 +297,7 @@ std::shared_ptr getSource(const RecompileRequest& request) { if (source == nullptr) { source = std::make_shared(); source->code.assign(request.shader.code.begin(), request.shader.code.end()); + source->capturedCalls.assign(request.shader.capturedCalls.begin(), request.shader.capturedCalls.end()); auto& bucket = sources[key]; if (!bucket.empty()) { // A second entry under one key is a code hash collision (or the unhashed key with @@ -959,6 +982,7 @@ void materializeCapture(ResourceCapture& capture, const SrtRuntime& runtime) { } std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime) { + if (!capturedCallsMatchUserData(request.shader.capturedCalls, runtime.userData)) throw std::invalid_argument("captured scalar call targets do not match capture user data"); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; // Validates the stage inputs once per request, as GetResourcePlan and Recompile(request) do. @@ -1008,6 +1032,7 @@ void BuildPreparedShaderKey(const RecompileRequest& request, std::vector key) { + if (!capturedCallsMatchUserData(request.shader.capturedCalls, request.context.userData)) return false; if (handle.source == nullptr || handle.artifact == nullptr) return false; const auto& layout = request.layout; const auto& prepared = handle.artifact->bindings.layout; @@ -1025,7 +1050,15 @@ std::span GetPreparedCode(const SourceHandle& handle) { return handle.source->code; } -PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) {} +std::span GetPreparedCalls(const SourceHandle& handle) { + if (handle.source == nullptr || handle.artifact == nullptr) throw std::runtime_error("ShaderRecompiler: prepared artifact is missing"); + return handle.source->capturedCalls; +} + +PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) { + this->request.shader.code = handle->source->code; + this->request.shader.capturedCalls = handle->source->capturedCalls; +} std::optional PreparedShaderInvocation::TryCreate(const RecompileRequest& request, const std::shared_ptr& handle) { struct PreparedKeyStorage {}; @@ -1041,6 +1074,7 @@ std::optional PreparedShaderInvocation::TryCreate(cons } std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { + if (!capturedCallsMatchUserData(handle->source->capturedCalls, runtime.userData)) throw std::invalid_argument("captured scalar call targets do not match capture user data"); auto capture = std::make_shared(); capture->source = handle->source; capture->plan = handle->source->plan; @@ -1049,6 +1083,7 @@ std::shared_ptr PreparedShaderInvocation::Capture(const S } std::shared_ptr PreparedShaderInvocation::Materialize(const ResourceCapture& capture) const { + if (!capturedCallsMatchUserData(handle->source->capturedCalls, request.context.userData)) throw std::invalid_argument("captured scalar call targets do not match invocation user data"); if (capture.source != handle->source || capture.plan != handle->source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle->source, handle->artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle->artifact, request, capture.snapshot)); @@ -1072,6 +1107,7 @@ std::shared_ptr MaterializeShader(const RecompileRequest& std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle) { if (handle.source == nullptr) throw std::runtime_error("ShaderRecompiler: source handle is missing"); + if (!capturedCallsMatchUserData(handle.source->capturedCalls, runtime.userData)) throw std::invalid_argument("captured scalar call targets do not match capture user data"); if (handle.artifact != nullptr && !MatchesPreparedShader(request, handle)) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the static ABI"); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 3ca2409ea5ea..766b1c271bb8 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -31,12 +31,25 @@ struct MemoryRegion { std::span bytes; }; +inline constexpr std::uint32_t MaxCapturedShaderCalls = 64; + +struct CapturedShaderCall { + std::uint32_t callProgramCounter; + std::uint32_t targetProgramCounter; + std::uint32_t returnProgramCounter; + std::uint64_t targetAddress; + std::uint32_t userDataIndex; + + bool operator==(const CapturedShaderCall&) const = default; +}; + struct ShaderBinary { ShaderStage stage; std::uint64_t codeAddress; std::span code; std::uint64_t headerAddress; std::span header; + std::span capturedCalls{}; }; struct ShaderComputeStageInfo { diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 8f2178189d5b..096ad4c0e736 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -350,7 +350,7 @@ Throughout the project, every function at every stage either **does exactly what - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. - [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontCreateWritingLine`, `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics` and `sceFontWritingLineClear` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libFont.cpp): runs are placed left to right, each at the line's advance so far, with no spacing or adjustment. Only horizontal and left-to-right lines, writing attribute 0 and non-null run metrics are modelled; other values, the metrics of an empty line and unknown or null handles throw, since no reference gives their result or error code. `sceFontWritingLineGetOrderingSpace` still throws (KytyPS5's values are placeholders). Lines are allocated on the host, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. -- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store, or a second `s_swappc_b64` sharing it) throws; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, keeping the existing decode quirk. +- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store, or a second static `s_swappc_b64` sharing it) throws; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, keeping the existing decode quirk. - The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. - [sceImeKeyboardGetResourceId](../../core/libs/prx/libSceIme/Export.cpp) (libSceIme) reports that no USB keyboard is connected (`0x80bc0004`, as [shadPS4](https://github.com/shadps4-emu/shadPS4) does) because IME keyboard events are not delivered to the handler, although [libSceKeyboard](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) exposes the host keyboard - [Player review dialog](../../core/libs/prx/libSceCdlgPlayerReview/Export.cpp) (libSceCdlgPlayerReview) shows nothing: `scePlayerReviewDialogOpen` finishes at once and the result reports that the user canceled, as `sceWebBrowserDialogOpen` does @@ -358,6 +358,7 @@ Throughout the project, every function at every stage either **does exactly what - [Running application ID](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - `sceSystemServiceGetAppIdOfRunningBigApp` reports 1 as the running title's app ID. The value is AnyPS5's own: the console's shell assigns app IDs and their values are not known here, so other calls that take an app ID must use the same one. `sceSystemServiceKillApp` with that ID and the arguments -1, 0 and 0, which [SwanStationPS5](https://github.com/darkxex/SwanStationPS5/blob/4cdc981d1a64ffb7f2af73f7a27a9e91a3cb5eb4/src/main.c#L1050-L1058) passes to close itself and [ps5-payload-dev's hbldr](https://github.com/ps5-payload-dev/websrv/blob/46374edee6627885abac717de24da7a1d73af6f1/src/ps5/hbldr.c#L425-L435) passes to close the running game, ends the title as its own `exit(0)` would: the guest's shutdown and `atexit` handlers run and the process exits with status 0. On the console the call returns and the shell then ends the title, and whether those handlers run there is unknown; here there is no shell, so the call does not return. Another app ID or other arguments throw. - [Temporary app-content storage](../../core/libs/prx/libSceAppContent/Export.cpp) tracks one `/temp0` mount, preserves its files on unmount, and rejects new path resolutions until remount. Mount, format, space-query and unmount errors follow [fpPS4](https://github.com/red-prig/fpPS4/blob/04cefd43e6fddd1ab033e7980cd356d14c964905/sys/sys_path.pas); the pointer ABI is also declared by [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/AppContent/AppContent.cs). Existing open host descriptors remain usable after unmount; console behaviour with open files is unverified. Host filesystem failures still throw. - [Guest path resolution](../../core/libs/prx/libc/src/General.cpp) falls back to ASCII case-insensitive component lookup, following [shadPS4's host filesystem](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/file_sys/backends/host_fs.cpp). PPSA03747 requests `Skyrim.INI`, `Data/Video` and lowercase archive/plugin names that differ from the dump's filenames. Exact host names take precedence; multiple fallback matches throw instead of selecting an arbitrary file. Existing parent names are resolved when creating a new file, and guest mount aliases use the same ASCII comparison. Results are not cached, so renames and newly created files remain visible. Unicode case folding and PS5 behaviour for case-colliding host entries are unverified; host roots are used as supplied. -- [Static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require one call per ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and one later `s_setpc_b64` return reading that exact pair. Regions must nest or be disjoint; recursion, shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. +- [Static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require one call per ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and one later `s_setpc_b64` return reading that exact pair. Regions must nest or be disjoint; recursion, shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Capture-resolved user-data calls use separate callee regions per call site and may reuse a link pair; arbitrary indirect calls and general call frames are not implemented. +- [Capture-resolved scalar calls](../../core/shader/recompiler/ControlFlow/src/UserDataCalls.cpp) are integrated at compute dispatch only. Targets must trace through scalar moves in a branch-free definition chain to invocation user data. A source pair that is written elsewhere in the caller is accepted only before its first write, without a back edge across the call; callees cannot write any source pair. A branch entering after a defining move is rejected. Callees must be leaves with one final return to the link pair, at most 64 call sites, 64 KiB per callee and 1 MiB combined with the caller. Nested calls, targets within a registration snapshot, recursion, PC-relative callee code, escaping links, other shader stages and targets computed by arithmetic or GPU data remain unsupported. Native RDNA2/PS5 call equivalence and retail-title compatibility have not been measured. - [s_call_b64](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) saves `PC + 4` over `GetShaderBase` and branches to `PC + 4 + signext(SIMM16) * 4`, following [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) sections 3.6.3, 12.2 and 12.3. Forward callees after `s_endpgm` are decoded. Native RDNA2/PS5 execution and title compatibility for these bounded scalar calls remain unverified; `s_rfe_b64` still throws in translation. - [sceVideodec2CreateDecoder](../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2) decodes HEVC (codec type 974921) and VP9 (2382845) through FFmpeg, besides H.264. The tags come from SharpProspero's `Videodec2CodecType` and EVO-PLAYER-PS5's `sce_videodec2.h`, which records them from hardware runs. HEVC input is converted from length-prefixed NAL units to Annex B as for H.264, and VP9 frames are passed through. Output is the same 8-bit NV12 for every codec; a decoder for HEVC Main10 or VP9 Profile 2 is created, but a frame deeper than 8 bits throws, since the console's 10-bit output layout is unknown, and `QueryDecoderMemoryInfo` reports the 8-bit frame size for it. `sceVideodec2GetPictureInfo` and `sceVideodec2GetAvcPictureInfo` throw for HEVC and VP9 pictures, which use other layouts (`SceVideodec2HevcPictureInfo`, `SceVideodec2Vp9PictureInfo`), and `sceVideodec2GetHevcPictureInfo` and `sceVideodec2GetVp9PictureInfo` are not exported