diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 9a4a6e63eb73..9e5979d516c3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -29,9 +29,11 @@ #ifdef __GLIBC__ #include #endif +#include #include #include #include +#include "Optimization/ShaderStageInputInfo.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" namespace AgcDriver::DriverDetail { @@ -538,14 +540,48 @@ std::vector ReadUserDataCallee(std::uint64_t address, std::uint32 throw std::runtime_error("its callee at " + hexText(address) + " does not return within " + std::to_string(limitWords * 4u) + " bytes"); } -std::vector LinkUserDataCallees(const ShaderRecompiler::RecompileRequest& request) { +std::vector ReadGuestBytes(std::uint64_t address, std::uint64_t bytes, const char* what) { + if (address > UINTPTR_MAX - bytes || !GuestMemory::Accessible(reinterpret_cast(static_cast(address)), static_cast(bytes))) { + throw std::runtime_error(std::string("its ") + what + " at " + hexText(address) + " + " + hexText(bytes) + " is not readable"); + } + std::vector data(static_cast(bytes)); + GuestMemory::Read(address, std::as_writable_bytes(std::span(data))); + return data; +} + +std::vector ReadTableCallees(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::TableCall& call) { + constexpr std::uint64_t limitBytes = 64u << 20u; + constexpr std::size_t limitCallees = 64u; + const auto slot = call.userDataRegister - request.context.userDataBaseRegister; + const auto pointer = static_cast(request.context.userData[slot]) | static_cast(request.context.userData[slot + 1u]) << 32u; + const auto words = ReadGuestBytes(pointer + call.descriptorOffset, 16u, "call table descriptor"); + ShaderRecompiler::ShaderBufferResource descriptor{}; + std::memcpy(descriptor.fields.data(), words.data(), words.size()); + const auto bytes = descriptor.GetSize(); + if (bytes > limitBytes) throw std::runtime_error("its call table of " + std::to_string(bytes) + " bytes is larger than the " + std::to_string(limitBytes) + " bytes that are searched for callees"); + const auto table = bytes == 0u ? std::vector{} : ReadGuestBytes(descriptor.Base48(), bytes, "call table"); + std::set targets; + for (std::uint64_t offset = call.entryOffset; offset + 8u <= bytes; offset += call.entryStride) { + std::uint64_t target = 0; + std::memcpy(&target, table.data() + offset, sizeof(target)); + if (target != 0u) targets.insert(target); + if (call.entryStride == 0u) break; + } + if (targets.size() > limitCallees) throw std::runtime_error("its call table holds " + std::to_string(targets.size()) + " distinct targets; at most " + std::to_string(limitCallees) + " are linked"); + std::vector callees; + for (const auto target : targets) callees.push_back({target, ReadUserDataCallee(target, call.linkRegister)}); + return callees; +} + +ShaderRecompiler::LinkedCalls LinkUserDataCallees(const ShaderRecompiler::RecompileRequest& request) { if (request.shader.stage != ShaderRecompiler::ShaderStage::Compute) return {}; const auto code = request.shader.code; if (std::none_of(code.begin(), code.end(), [](std::uint32_t word) { return (word & 0xff80ff00u) == 0xbe802100u; })) return {}; const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); - const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), request.context.waveSize}; const auto calls = ShaderRecompiler::FindUserDataCalls(program, swappc); - if (calls.empty()) return {}; + const auto tableCalls = ShaderRecompiler::FindTableCalls(program, swappc); + if (calls.empty() && tableCalls.empty()) return {}; std::vector> callees; for (const auto& call : calls) { const auto slot = call.userDataRegister - swappc.userDataBaseRegister; @@ -557,7 +593,15 @@ std::vector LinkUserDataCallees(const ShaderRecompiler::Recompile std::to_string(call.userDataRegister + 1u) + "] at program counter " + hexText(call.programCounter) + ", but " + error.what()); } } - return ShaderRecompiler::LinkUserDataCalls(program, calls, callees); + std::vector> tableCallees; + for (const auto& call : tableCalls) { + try { + tableCallees.push_back(ReadTableCallees(request, call)); + } catch (const std::exception& error) { + throw std::runtime_error("AGC driver: compute shader " + hexText(request.shader.codeAddress) + " calls a target loaded from a table at program counter " + hexText(call.programCounter) + ", but " + error.what()); + } + } + return ShaderRecompiler::LinkCalls(program, calls, callees, tableCalls, tableCallees); } } @@ -567,9 +611,12 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s const auto code = std::span(snapshot.code).subspan(codeOffset); require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code"); auto linked = LinkUserDataCallees(request); - if (linked.empty()) linked = LinkGraphicsUserDataTail(request, snapshot); + if (linked.code.empty()) linked.code = LinkGraphicsUserDataTail(request, snapshot); auto source = request; - if (!linked.empty()) source.shader.code = linked; + if (!linked.code.empty()) { + source.shader.code = linked.code; + source.context.linkedCallFallbacks = linked.fallbacks; + } auto invocationRequest = source; struct PreparedKeyStorage {}; auto& key = HostThreadLocal, PreparedKeyStorage>(); @@ -579,7 +626,7 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset != codeOffset) continue; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); - if (!linked.empty() && !std::ranges::equal(invocationRequest.shader.code, linked)) continue; + if (!linked.code.empty() && !std::ranges::equal(invocationRequest.shader.code, linked.code)) continue; if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation); } ReportPreparedAtUse(snapshot, source); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index f4f4750f14a5..148b5e4f55a6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -275,6 +275,10 @@ void BdaResources::CheckFault() const { throw std::runtime_error(message.str()); } std::ostringstream message; + if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::UnlinkedCall) { + message << "shader stage " << report.stage << " called address 0x" << std::hex << report.address << " at program counter 0x" << report.instruction << ", which no linked callee of that call matches"; + throw std::runtime_error(message.str()); + } message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast(report.reason); if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::Permission && table != nullptr) { const auto bytes = table->Bytes(); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 59cf1d71b5c3..4c987e96de0d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2836,7 +2836,13 @@ void verifyPixelRequestSerialization() { minimal.context.pixel->layerExported = true; minimal.context.pixel->ditheredAlphaToCoverage = true; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQRkAAAA=", "new requests did not use serialization version 25"); + require(requestPrefix(encoded, 8u) == "NVNQQRoAAAA=", "new requests did not use serialization version 26"); + const std::array fallbacks{0x40u, 0x1ac4u}; + auto withFallbacks = minimal; + withFallbacks.context.linkedCallFallbacks = fallbacks; + const auto fallbackRequest = serializer.Deserialize(serializer.Serialize(withFallbacks)); + require(std::ranges::equal(fallbackRequest.request.context.linkedCallFallbacks, fallbacks), "the linked call fallbacks did not round-trip"); + require(serializer.Deserialize(encoded).request.context.linkedCallFallbacks.empty(), "a request without linked call fallbacks gained some"); require(serializer.Deserialize(encoded).request.context.pixel->rasterPrimitive == RasterPrimitive::Rectangle && serializer.Deserialize(encoded).request.context.pixel->ditheredAlphaToCoverage, "the pixel primitive class or dither mode did not round-trip"); require(serializer.Deserialize(encoded).request.context.pixel->layerExported, "the pixel layer export did not round-trip"); const auto legacy16 = serializer.Deserialize( @@ -2948,7 +2954,7 @@ void verifyPixelRequestSerialization() { for (std::size_t bytes = 0; bytes < 23u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag, primitive class, layer export or dither flag was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRoAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRsAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } static constexpr std::string_view version18Request{ @@ -3067,6 +3073,7 @@ void verifyVertexInfoVersion19() { while (flagOffset < legacyPixel.size() && legacyPixel[flagOffset] == exportedBytes[flagOffset]) ++flagOffset; require(flagOffset < legacyPixel.size(), "the pixel layer export flag is not serialized"); legacyPixel.erase(legacyPixel.begin() + static_cast(flagOffset), legacyPixel.begin() + static_cast(flagOffset + 2u)); + legacyPixel.resize(legacyPixel.size() - sizeof(std::uint64_t)); for (const auto version : {19u, 20u, 21u, 22u}) { auto versionPixel = legacyPixel; versionPixel[4] = static_cast(version); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp new file mode 100644 index 000000000000..6b3bb066da31 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp @@ -0,0 +1,164 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "SceShaders.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Untouched = 0xcafebabeu; + +alignas(256) constexpr std::array Caller{ + 0x7e020280u, 0xf4080400u, 0xfa000000u, 0x8f148402u, 0xbf8cc07fu, 0xf4280208u, 0x28000000u, 0xbf8cc07fu, + 0xbe8e0308u, 0xbe8f0309u, 0xbf068103u, 0xbe8e210eu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, + 0xbf9f0000u, 0x30306c73u, 0x00000040u, 0x00000008u}; +alignas(256) constexpr std::array SelectSixteen{0x85159091u, 0x7e020215u, 0xbe80200eu}; +alignas(256) constexpr std::array SelectThirtyTwo{0x8515a0a1u, 0x7e020215u, 0xbe80200eu}; +alignas(16) std::array Table{}; +alignas(16) std::array Descriptors{}; +alignas(256) std::array Output{}; + +void Fill() { + const auto first = reinterpret_cast(SelectSixteen.data()); + const auto second = reinterpret_cast(SelectThirtyTwo.data()); + Table = {static_cast(first), static_cast(first >> 32u), 0u, 0u, static_cast(second), static_cast(second >> 32u), 0u, 0u}; + const auto table = reinterpret_cast(Table.data()); + Descriptors = {static_cast(table), static_cast((table >> 32u) & 0xffffu) | (16u << 16u), 2u, 0x31016facu}; +} + +std::array UserData(std::uint32_t index, bool scc) { + const auto descriptors = reinterpret_cast(Descriptors.data()); + const auto output = reinterpret_cast(Output.data()); + return {static_cast(descriptors), static_cast(descriptors >> 32u), index, scc ? 1u : 0u, + static_cast(output), static_cast((output >> 32u) & 0xffffu), static_cast(sizeof(Output)), 0x31016facu}; +} + +std::vector Commands(const std::array& data, bool wave32) { + std::vector words; + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{Threads, 1, 1}; + registers(0x207, shape); + const auto address = reinterpret_cast(Caller.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array resource{static_cast(data.size()) << 1u}; + registers(0x213, resource); + registers(0x240, data); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x41u | (wave32 ? 0x8000u : 0u)}); + return words; +} + +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +std::string Dispatch(std::uint32_t index, bool scc, bool wave32) { + Output.fill(Untouched); + const auto words = Commands(UserData(index, scc), wave32); + Packet packet{const_cast(words.data()), static_cast(words.size()), 0, {}}; + StderrCapture log; + std::string failure; + try { + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); + } catch (const std::exception& error) { + failure = error.what(); + } + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); + return log.Finish() + failure; +} + +void Check(std::uint32_t index, bool scc, bool wave32, std::uint32_t expected, const std::string& what) { + const auto log = Dispatch(index, scc, wave32); + Require(log.find("no linked callee") == std::string::npos, what + ": a linked callee was reported missing: " + log); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + Require(Output[lane] == expected, what + (wave32 ? " (wave32)" : " (wave64)") + ": lane " + std::to_string(lane) + " is " + std::to_string(Output[lane]) + ", expected " + std::to_string(expected)); + } +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + Fill(); + for (const bool wave32 : {false, true}) { + Check(0u, false, wave32, 16u, "the first record's callee with SCC 0"); + Check(0u, true, wave32, 17u, "the first record's callee with SCC 1"); + Check(1u, false, wave32, 32u, "the second record's callee with SCC 0"); + Check(1u, true, wave32, 33u, "the second record's callee with SCC 1"); + const auto log = Dispatch(2u, false, wave32); + Require(log.find("called address 0x0 at program counter") != std::string::npos && log.find("which no linked callee of that call matches") != std::string::npos, + std::string("a record past the table did not report an unlinked call") + (wave32 ? " (wave32)" : " (wave64)") + ": " + log); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + Require(Output[lane] == Untouched, "a lane stored after the unlinked call: lane " + std::to_string(lane) + " is " + std::to_string(Output[lane])); + } + } + AgcDriverShutdown_nid_postfix(); + std::puts("compute table call tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception&) { + } + return 1; + } +} diff --git a/core/libs/tests/ScalarCall.cpp b/core/libs/tests/ScalarCall.cpp index e797b7aeba84..dda7b2fea4ae 100644 --- a/core/libs/tests/ScalarCall.cpp +++ b/core/libs/tests/ScalarCall.cpp @@ -151,10 +151,8 @@ static void CheckUserDataCalls() { Require(std::equal(addFive.begin(), addFive.end(), linked.begin() + 12) && std::equal(addSixtyFour.begin(), addSixtyFour.end(), linked.begin() + 14)); auto readsPc = caller; readsPc[1] = 0xbe941f00u; - ExpectThrow("cannot have user-data callees linked", [&] { - const auto readsPcProgram = RdnaInstructionDecoder{}.Decode(readsPc); - static_cast(LinkUserDataCalls(readsPcProgram, FindUserDataCalls(readsPcProgram, swappc), callees)); - }); + const auto readsPcProgram = RdnaInstructionDecoder{}.Decode(readsPc); + Require(LinkUserDataCalls(readsPcProgram, FindUserDataCalls(readsPcProgram, swappc), callees).size() == 17u); const auto linkedProgram = RdnaInstructionDecoder{}.Decode(linked); const auto cfg = GraphBuilder{}.Build(linkedProgram, &swappc); @@ -181,6 +179,101 @@ static void CheckUserDataCalls() { ExpectThrow("is shared with another call", [&] { static_cast(GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(sameCopy), &swappc)); }); } +static std::vector TableCaller() { + return {0x8f148414u, 0xf4080406u, 0xfa000270u, 0xbf8cc07fu, 0xf4280208u, 0x28000000u, 0xbf8cc07fu, 0xbe8e0308u, 0xbe8f0309u, 0xbe8e210eu, 0xbf810000u}; +} + +static ControlFlowGraph BuildCode(std::span code, const SwappcInfo& swappc) { + return GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(code), &swappc); +} + +static IrProgram TranslateCode(std::span code, const SwappcInfo& swappc) { + const auto program = RdnaInstructionDecoder{}.Decode(code); + const auto cfg = GraphBuilder{}.Build(program, &swappc); + ShaderComputeInputInfo computeInfo{}; + TranslateOptions options{}; + options.stage = ShaderStageKind::Compute; + options.inputInfo.compute = &computeInfo; + options.userDataCount = swappc.userDataCount; + return InstructionTranslator{}.Translate(program, cfg, options); +} + +static void CheckTableCalls() { + const SwappcInfo swappc{false, 0u, 16u}; + const auto caller = TableCaller(); + const auto program = RdnaInstructionDecoder{}.Decode(caller); + const auto calls = FindTableCalls(program, swappc); + Require(calls.size() == 1u); + Require(calls[0].programCounter == 0x24u && calls[0].linkRegister == 14u && calls[0].targetRegister == 14u && calls[0].userDataRegister == 12u); + Require(calls[0].descriptorOffset == 0x270u && calls[0].entryOffset == 0u && calls[0].entryStride == 16u); + Require(FindUserDataCalls(program, swappc).empty()); + const auto table = [&](std::size_t word, std::uint32_t value, std::uint32_t userDataCount = 16u) { + auto code = caller; + code[word] = value; + return FindTableCalls(RdnaInstructionDecoder{}.Decode(code), SwappcInfo{false, 0u, userDataCount}); + }; + Require(table(0u, 0x8f148114u).empty()); + Require(table(2u, 0x28000270u).empty()); + Require(table(8u, 0xbe8f030au).empty()); + Require(table(10u, 0xbf810000u, 12u).empty()); + Require(table(5u, 0x28000008u).size() == 1u && table(5u, 0x28000008u)[0].entryOffset == 8u); + Require(table(0u, 0xbe940380u).empty()); + auto readsLane = caller; + readsLane.insert(readsLane.begin(), {0xd4c2000bu, 0x00020300u}); + const auto readsLaneProgram = RdnaInstructionDecoder{}.Decode(readsLane); + Require(FindTableCalls(readsLaneProgram, SwappcInfo{false, 0u, 16u, 32u}).size() == 1u); + Require(FindTableCalls(readsLaneProgram, SwappcInfo{false, 0u, 16u, 64u}).empty()); + + ExpectThrow("not statically resolvable", [&] { static_cast(BuildCode(caller, swappc)); }); + const std::array fallback{0x24u}; + SwappcInfo listed = swappc; + listed.linkedCallFallbacks = fallback; + Require(BuildCode(caller, listed).unlinkedCallProgramCounters == std::vector{0x24u}); + const auto unlinked = TranslateCode(caller, listed); + Require(unlinked.Info().usesFaultBuffer && ProgramToString(unlinked).find("UnlinkedCall") != std::string::npos); + + const std::vector first{0xbf068000u, 0x4a020285u, 0xbe80200eu}; + const std::vector second{0x4a0202c0u, 0xbf068001u, 0xbe80200eu}; + const std::vector> callees{{{0x1122334455ull, first}, {0x66778899aaull, second}}}; + const auto linkedCalls = LinkCalls(program, {}, {}, calls, callees); + const auto& linked = linkedCalls.code; + Require(linkedCalls.fallbacks == std::vector{50u * 4u}); + ExpectThrow("is shared with another call", [&] { static_cast(BuildCode(linked, swappc)); }); + SwappcInfo linkedSwappc = swappc; + linkedSwappc.linkedCallFallbacks = linkedCalls.fallbacks; + Require(linked.size() == 59u && linked[9] == 0xbf820001u && linked[11] == 0xbf850010u && linked[58] == 0xbf9f0000u); + Require(linked[12] == 0xbf07ff0eu && linked[13] == 0x22334455u && linked[14] == 0xbf850004u && linked[15] == 0xbf07ff0fu && linked[16] == 0x11u && linked[17] == 0xbf850001u && linked[18] == 0xbf82001bu); + Require(linked[20] == 0x778899aau && linked[26] == 0xbf078080u && linked[27] == 0xbf820016u && linked[34] == 0xbf068080u && linked[35] == 0xbf82000au && linked[44] == 0xbf068080u && linked[45] == 0xbf820004u); + Require(linked[46] == 0xbb0e0005u && linked[47] == 0xbf82ffdau && linked[48] == 0xbb0e0006u && linked[49] == 0xbf82ffd8u && linked[50] == 0xbe8e210eu && linked[51] == 0xbf82ffd6u); + Require(std::equal(first.begin(), first.end(), linked.begin() + 52) && std::equal(second.begin(), second.end(), linked.begin() + 55)); + const auto cfg = BuildCode(linked, linkedSwappc); + const auto block = [&](std::uint32_t word) { return cfg.FindBlockByProgramCounter(word * 4u).id; }; + Require(cfg.FindBlockByProgramCounter(46u * 4u).terminator.trueBlock == block(52u)); + Require(cfg.FindBlockByProgramCounter(52u * 4u).terminator.trueBlock == block(47u)); + Require(cfg.FindBlockByProgramCounter(48u * 4u).terminator.trueBlock == block(55u)); + Require(cfg.FindBlockByProgramCounter(55u * 4u).terminator.trueBlock == block(49u)); + Require(cfg.unlinkedCallProgramCounters == std::vector{50u * 4u}); + static_cast(TranslateCode(linked, linkedSwappc)); + + std::vector large(0x8000u, 0xbf800000u); + const std::vector> far{{{0x10ull, large}, {0x20ull, first}}}; + ExpectThrow("out of s_call_b64 range", [&] { static_cast(LinkCalls(program, {}, {}, calls, far)); }); + + const std::vector jumpTable{ + 0xbe8c1f00u, 0x800cff0cu, 0x00000058u, 0x820d800du, 0x83928202u, 0xd4c40016u, 0x02000500u, 0x8f128212u, 0xf4000386u, 0x24000000u, + 0xbf8cc07fu, 0x7d820003u, 0x808c0e0cu, 0x828d800du, 0xbefd210cu, 0x7e020281u, 0xbf820003u, 0x7e020282u, 0xbf820001u, 0x7e020283u, + 0xe0700000u, 0x80000100u, 0xbf810000u, 0x00000020u, 0x00000018u, 0x00000010u}; + ExpectThrow("jumps through a table in its code", [&] { static_cast(LinkCalls(RdnaInstructionDecoder{}.Decode(jumpTable), {}, {}, {}, {})); }); + const std::array compare{0xbf068008u}; + const auto comparesLink = Build({DecodeRdnaSopc(0u, compare, 0u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + Require(comparesLink.FindBlockByProgramCounter(0u).terminator.trueBlock == comparesLink.FindBlockByProgramCounter(12u).id); + const auto readsFirst = Build({Sop1(0u, 0u, 3u, 8u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + Require(readsFirst.FindBlockByProgramCounter(0u).terminator.trueBlock == readsFirst.FindBlockByProgramCounter(12u).id); + ExpectThrow("escapes the constant-offset call/return model", [] { + Build({Sop1(0u, 0u, 3u, 8u), Call(4u, 8u, 2u), Sopp(8u, 2u, 0xfffdu), Sopp(12u, 1u), Sop1(16u, 0u, 0x20u, 8u)}); + }); +} + static void CheckRejections() { ExpectThrow("invalid instruction boundary", [] { Build({Call(0u, 8u, 16u), Sopp(4u, 1u)}); @@ -280,6 +373,7 @@ int main(int argc, char** argv) { CheckCallAfterEnd(); CheckRejections(); CheckUserDataCalls(); + CheckTableCalls(); CheckGuards(""); } else { Require(argc == 2); diff --git a/core/libs/tests/ScalarSwappc.cpp b/core/libs/tests/ScalarSwappc.cpp index 6ab8dc4f0512..7776183052a0 100644 --- a/core/libs/tests/ScalarSwappc.cpp +++ b/core/libs/tests/ScalarSwappc.cpp @@ -135,7 +135,7 @@ static void CheckThrows() { ExpectThrow("escapes the constant-offset call/return model", [] { Build(Program({ Sop1(0x00u, 4u, 0x1fu, 0u), - Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x20u), Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), Sop1(0x14u, 8u, 0x21u, 4u), Sop1(0x18u, 8u, 0x03u, 8u), diff --git a/core/shader/recompiler/BdaAbi.hpp b/core/shader/recompiler/BdaAbi.hpp index 0b055a525e8f..7a79928c76b5 100644 --- a/core/shader/recompiler/BdaAbi.hpp +++ b/core/shader/recompiler/BdaAbi.hpp @@ -28,7 +28,7 @@ struct Range { }; enum class FaultState : std::uint32_t { Empty, Writing, Ready }; -enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor, ImageTable }; +enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor, ImageTable, UnlinkedCall }; struct Fault { FaultState state; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 3068aca764df..08b8bbb7035c 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -260,6 +260,8 @@ function(add_shader_recompiler target runtimeTarget) FindUserDataCalls UserDataCalleeLength LinkUserDataCalls + FindTableCalls + LinkCalls WriteFileAtomically ) set(shaderCacheManifest) diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 33e899a8188e..01e252df30d0 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -52,6 +52,7 @@ class RecompileCacheKey { append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.context.floatMode); + appendFallbacks(key, request); appendMesh(key, request); appendTessellation(key, request); std::uint64_t hash = 0xcbf29ce484222325ull; @@ -84,6 +85,7 @@ class RecompileCacheKey { append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.context.floatMode); + appendFallbacks(key, request); appendMesh(key, request); appendTessellation(key, request); append(key, request.target); @@ -92,6 +94,11 @@ class RecompileCacheKey { append(key, RayTracingMiss()); } + static void appendFallbacks(std::vector& key, const RecompileRequest& request) { + append(key, request.context.linkedCallFallbacks.size()); + for (const auto programCounter : request.context.linkedCallFallbacks) append(key, programCounter); + } + static void appendMesh(std::vector& key, const RecompileRequest& request) { if (request.shader.stage != ShaderStage::Mesh) return; const auto* mesh = request.graphics && request.graphics->mesh ? &*request.graphics->mesh : nullptr; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp index d93905ba942d..3199579688c2 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp @@ -107,6 +107,7 @@ struct ControlFlowGraph { std::vector components; std::vector codeTableLoadProgramCounters; std::vector codeTableLoads; + std::vector unlinkedCallProgramCounters; std::uint32_t entryBlock = InvalidControlFlowId; bool hasFetchCall = false; std::uint32_t fetchCallProgramCounter = 0; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index 27bbdae67dfe..49d52a67f03e 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -23,6 +23,8 @@ struct SwappcInfo { bool fetchCallAllowed = false; std::uint32_t userDataBaseRegister = 0; std::uint32_t userDataCount = 0; + std::uint32_t waveSize = 0; + std::span linkedCallFallbacks; }; struct SwappcCall { @@ -41,10 +43,33 @@ struct UserDataCall { std::uint32_t userDataRegister = 0; }; +struct TableCall { + std::uint32_t programCounter = 0; + std::uint32_t linkRegister = 0; + std::uint32_t targetRegister = 0; + std::uint32_t userDataRegister = 0; + std::uint32_t descriptorOffset = 0; + std::uint32_t entryOffset = 0; + std::uint32_t entryStride = 0; +}; + +struct TableCallee { + std::uint64_t target = 0; + std::vector code; +}; + [[nodiscard]] std::optional UnresolvableSwappcTarget(const RdnaProgram& program, const SwappcInfo* swappc); [[nodiscard]] std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc); [[nodiscard]] std::optional UserDataCalleeLength(std::span callee, std::uint32_t linkRegister); [[nodiscard]] std::vector LinkUserDataCalls(const RdnaProgram& program, std::span calls, std::span> callees); +[[nodiscard]] std::vector FindTableCalls(const RdnaProgram& program, const SwappcInfo& swappc); +struct LinkedCalls { + std::vector code; + std::vector fallbacks; +}; + +[[nodiscard]] LinkedCalls LinkCalls(const RdnaProgram& program, std::span userDataCalls, std::span> userDataCallees, + std::span tableCalls, std::span> tableCallees); class GraphBuilder { public: diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp index 0f59c68558e0..bf76e576bdd5 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp @@ -33,6 +33,7 @@ namespace ShaderRecompiler { std::optional vertex; std::vector> memoryBytesStorage; std::vector memory; + std::vector linkedCallFallbacks; std::vector supportedCapabilities; std::vector supportedExtensionStorage; std::vector supportedExtensions; diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 7a487915c8b4..a67f30d1ec38 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -2,6 +2,7 @@ #include "ControlFlow/ControlFlowHelpers.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include +#include #include #include #include @@ -257,7 +258,7 @@ bool resolveJumpTable(const RdnaProgram& program, std::uint32_t index, BoundedJu return false; } -bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { +bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index, std::uint32_t waveSize = 0u) { std::uint32_t count = instruction.dataDwordCount; switch (instruction.family) { case RdnaInstructionFamily::SOP1: @@ -273,7 +274,7 @@ bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { case RdnaInstructionFamily::VOP3: case RdnaInstructionFamily::VOP3P: case RdnaInstructionFamily::VOPC: - count = 2u; + count = instruction.op == RdnaOpcode::VReadlaneB32 || instruction.op == RdnaOpcode::VReadfirstlaneB32 || waveSize == 32u ? 1u : 2u; break; case RdnaInstructionFamily::SOPC: case RdnaInstructionFamily::SOPP: @@ -293,11 +294,11 @@ bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { return covers(instruction.destination) || covers(instruction.destination2); } -bool findLastWriter(const RdnaProgram& program, std::uint32_t end, std::initializer_list indices, std::uint32_t& writer) { +bool findLastWriter(const RdnaProgram& program, std::uint32_t end, std::initializer_list indices, std::uint32_t& writer, std::uint32_t waveSize = 0u) { for (auto position = end; position-- > 0u;) { const auto& instruction = program.instructions[position]; if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SSetpcB64 || instruction.op == RdnaOpcode::SEndpgm) return false; - if (std::ranges::any_of(indices, [&](std::uint32_t index) { return writesScalar(instruction, index); })) { + if (std::ranges::any_of(indices, [&](std::uint32_t index) { return writesScalar(instruction, index, waveSize); })) { writer = position; return true; } @@ -475,7 +476,298 @@ bool isFetchCall(const RdnaProgram& program, std::uint32_t index, const SwappcIn return swappc != nullptr && swappc->fetchCallAllowed && positional && (userDataPair || outsideProgram); } -std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc) { +bool overwritesScalar(const RdnaInstruction& instruction, std::uint32_t reg, std::uint32_t waveSize) { + const auto first = scalarIndex(instruction.destination); + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + return isScalarLoad(instruction.op) && first != NoScalarRegister && reg >= first && reg - first < instruction.dataDwordCount; + case RdnaInstructionFamily::SOP1: + switch (instruction.op) { + case RdnaOpcode::SMovreldB32: + case RdnaOpcode::SMovreldB64: + case RdnaOpcode::SMovrelsd2B32: + case RdnaOpcode::SCmovB32: + case RdnaOpcode::SCmovB64: + case RdnaOpcode::SBitset0B32: + case RdnaOpcode::SBitset0B64: + case RdnaOpcode::SBitset1B32: + case RdnaOpcode::SBitset1B64: + case RdnaOpcode::SSwappcB64: + case RdnaOpcode::SSetpcB64: return false; + case RdnaOpcode::SMovB64: + case RdnaOpcode::SGetpcB64: return first != NoScalarRegister && reg >= first && reg - first < 2u; + default: return first != NoScalarRegister && reg == first; + } + case RdnaInstructionFamily::SOP2: return first != NoScalarRegister && reg == first; + case RdnaInstructionFamily::SOPK: return instruction.op == RdnaOpcode::SMovkI32 && reg == first; + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: { + const std::uint32_t count = instruction.op != RdnaOpcode::VReadlaneB32 && instruction.op != RdnaOpcode::VReadfirstlaneB32 && waveSize == 64u ? 2u : 1u; + return std::ranges::any_of(std::array{&instruction.destination, &instruction.destination2}, [&](const RdnaOperand* operand) { + const auto slot = scalarIndex(*operand); + return slot != NoScalarRegister && reg >= slot && reg - slot < count; + }); + } + default: return false; + } +} + +bool readsVectorSources64(RdnaOpcode op) { + switch (op) { + case RdnaOpcode::VMadU64U32: + case RdnaOpcode::VMadI64I32: + case RdnaOpcode::VFmaF64: + case RdnaOpcode::VAddF64: + case RdnaOpcode::VMulF64: + case RdnaOpcode::VMinF64: + case RdnaOpcode::VMaxF64: + case RdnaOpcode::VLdexpF64: + case RdnaOpcode::VCvtI32F64: + case RdnaOpcode::VCvtF64I32: + case RdnaOpcode::VCvtF32F64: + case RdnaOpcode::VCvtF64F32: + case RdnaOpcode::VCvtU32F64: + case RdnaOpcode::VCvtF64U32: + case RdnaOpcode::VTruncF64: + case RdnaOpcode::VCeilF64: + case RdnaOpcode::VRndneF64: + case RdnaOpcode::VFloorF64: + case RdnaOpcode::VFrexpExpI32F64: + case RdnaOpcode::VFrexpMantF64: + case RdnaOpcode::VFractF64: + case RdnaOpcode::VRcpF64: + case RdnaOpcode::VRsqF64: + case RdnaOpcode::VSqrtF64: + case RdnaOpcode::VTrigPreopF64: + case RdnaOpcode::VDivScaleF64: + case RdnaOpcode::VDivFmasF64: + case RdnaOpcode::VDivFixupF64: + case RdnaOpcode::VLshlrevB64: + case RdnaOpcode::VLshrrevB64: + case RdnaOpcode::VAshrrevI64: + case RdnaOpcode::VCmpEqI64: + case RdnaOpcode::VCmpLtU64: + case RdnaOpcode::VCmpEqU64: + case RdnaOpcode::VCmpGtU64: + case RdnaOpcode::VCmpNeU64: + case RdnaOpcode::VCmpxNeI64: + case RdnaOpcode::VCmpxNeU64: + case RdnaOpcode::VCmpFI64: + case RdnaOpcode::VCmpLtI64: + case RdnaOpcode::VCmpLeI64: + case RdnaOpcode::VCmpGtI64: + case RdnaOpcode::VCmpNeI64: + case RdnaOpcode::VCmpGeI64: + case RdnaOpcode::VCmpTI64: + case RdnaOpcode::VCmpxFI64: + case RdnaOpcode::VCmpxLtI64: + case RdnaOpcode::VCmpxEqI64: + case RdnaOpcode::VCmpxLeI64: + case RdnaOpcode::VCmpxGtI64: + case RdnaOpcode::VCmpxGeI64: + case RdnaOpcode::VCmpxTI64: + case RdnaOpcode::VCmpFU64: + case RdnaOpcode::VCmpLeU64: + case RdnaOpcode::VCmpGeU64: + case RdnaOpcode::VCmpTU64: + case RdnaOpcode::VCmpxFU64: + case RdnaOpcode::VCmpxLtU64: + case RdnaOpcode::VCmpxEqU64: + case RdnaOpcode::VCmpxLeU64: + case RdnaOpcode::VCmpxGtU64: + case RdnaOpcode::VCmpxGeU64: + case RdnaOpcode::VCmpxTU64: + case RdnaOpcode::VCmpFF64: + case RdnaOpcode::VCmpLtF64: + case RdnaOpcode::VCmpEqF64: + case RdnaOpcode::VCmpLeF64: + case RdnaOpcode::VCmpGtF64: + case RdnaOpcode::VCmpLgF64: + case RdnaOpcode::VCmpGeF64: + case RdnaOpcode::VCmpOF64: + case RdnaOpcode::VCmpUF64: + case RdnaOpcode::VCmpNgeF64: + case RdnaOpcode::VCmpNlgF64: + case RdnaOpcode::VCmpNgtF64: + case RdnaOpcode::VCmpNleF64: + case RdnaOpcode::VCmpNeqF64: + case RdnaOpcode::VCmpNltF64: + case RdnaOpcode::VCmpTruF64: + case RdnaOpcode::VCmpClassF64: + case RdnaOpcode::VCmpxFF64: + case RdnaOpcode::VCmpxLtF64: + case RdnaOpcode::VCmpxEqF64: + case RdnaOpcode::VCmpxLeF64: + case RdnaOpcode::VCmpxGtF64: + case RdnaOpcode::VCmpxLgF64: + case RdnaOpcode::VCmpxGeF64: + case RdnaOpcode::VCmpxOF64: + case RdnaOpcode::VCmpxUF64: + case RdnaOpcode::VCmpxNgeF64: + case RdnaOpcode::VCmpxNlgF64: + case RdnaOpcode::VCmpxNgtF64: + case RdnaOpcode::VCmpxNleF64: + case RdnaOpcode::VCmpxNeqF64: + case RdnaOpcode::VCmpxNltF64: + case RdnaOpcode::VCmpxTruF64: + case RdnaOpcode::VCmpxClassF64: + return true; + default: return false; + } +} + +bool readsScalarDword(const RdnaInstruction& instruction, std::uint32_t reg, std::uint32_t waveSize) { + if (instruction.op == RdnaOpcode::SMovrelsB32 || instruction.op == RdnaOpcode::SMovrelsB64) return true; + const auto overlaps = [&](const RdnaOperand& operand, std::uint32_t count) { + const auto slot = scalarIndex(operand); + return slot != NoScalarRegister && slot <= reg && reg - slot < count; + }; + std::uint32_t source0Count = 1u, source1Count = 1u, source2Count = 1u, source3Count = 1u; + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + switch (instruction.op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: source0Count = 4u; break; + default: source0Count = 2u; break; + } + break; + case RdnaInstructionFamily::MUBUF: + case RdnaInstructionFamily::MTBUF: source1Count = 4u; break; + case RdnaInstructionFamily::MIMG: + source1Count = instruction.imageR128 ? 4u : 8u; + source2Count = 4u; + break; + case RdnaInstructionFamily::FLAT: + source0Count = 2u; + source1Count = 2u; + break; + case RdnaInstructionFamily::SOPK: + if (instruction.op != RdnaOpcode::SMovkI32 && instruction.op != RdnaOpcode::SCallB64 && overlaps(instruction.destination, 1u)) return true; + break; + case RdnaInstructionFamily::SOP1: + case RdnaInstructionFamily::SOP2: + case RdnaInstructionFamily::SOPC: + switch (instruction.op) { + case RdnaOpcode::SBcnt0I32B64: + case RdnaOpcode::SBcnt1I32B64: + case RdnaOpcode::SFf0I32B64: + case RdnaOpcode::SFf1I32B64: + case RdnaOpcode::SFlbitI32B64: + case RdnaOpcode::SFlbitI32I64: + case RdnaOpcode::SCmpEqU64: + case RdnaOpcode::SCmpLgU64: + case RdnaOpcode::SBitcmp0B64: + case RdnaOpcode::SBitcmp1B64: source0Count = source1Count = 2u; break; + default: source0Count = source1Count = instruction.family == RdnaInstructionFamily::SOPC ? 1u : std::max(instruction.dataDwordCount, 1u); break; + } + break; + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: + source0Count = source1Count = source2Count = source3Count = readsVectorSources64(instruction.op) ? 2u : 1u; + switch (instruction.op) { + case RdnaOpcode::VCndmaskB32: + case RdnaOpcode::VAddcU32: + case RdnaOpcode::VSubCoCiU32: + case RdnaOpcode::VSubrevCoCiU32: + case RdnaOpcode::VDivFmasF32: + case RdnaOpcode::VDivFmasF64: source2Count = std::max(source2Count, waveSize == 32u ? 1u : 2u); break; + default: break; + } + break; + default: break; + } + switch (instruction.op) { + case RdnaOpcode::SBitset0B32: + case RdnaOpcode::SBitset0B64: + case RdnaOpcode::SBitset1B32: + case RdnaOpcode::SBitset1B64: + case RdnaOpcode::SCmovB32: + case RdnaOpcode::SCmovB64: + if (overlaps(instruction.destination, 2u)) return true; + break; + default: break; + } + return overlaps(instruction.source0, source0Count) || overlaps(instruction.source1, source1Count) || overlaps(instruction.source2, source2Count) || overlaps(instruction.source3, source3Count); +} + +template +void requireReturnAddressesStayInCalls(const RdnaProgram& program, const std::vector& calls, const std::vector& unlinked, std::uint32_t waveSize, const TError& escapeError) { + std::set links; + for (const auto& call : calls) { + if (!call.fetch) links.insert(call.linkRegister); + } + const auto size = static_cast(program.instructions.size()); + const auto successors = [&](std::uint32_t index) { + std::vector next; + const auto& instruction = program.instructions[index]; + if (instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd) return next; + if (IsDirectBranchOpcode(instruction.op)) { + next.push_back(instructionIndexOfProgramCounter(program, instruction.branchTarget)); + if (IsConditionalBranchOpcode(instruction.op)) next.push_back(index + 1u); + } else if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { + const SwappcCall* call = findCallAt(calls, index); + next.push_back(call == nullptr || call->fetch ? index + 1u : call->targetIndex); + } else if (instruction.op == RdnaOpcode::SSetpcB64) { + std::uint32_t target = 0; + if (const SwappcCall* call = findReturnAt(calls, index); call != nullptr) { + next.push_back(call->callIndex + 1u); + } else if (resolveSetpcTarget(program, index, target)) { + next.push_back(instructionIndexOfProgramCounter(program, target)); + } + } else { + next.push_back(index + 1u); + } + std::erase_if(next, [&](std::uint32_t successor) { return successor >= size; }); + return next; + }; + for (const auto link : links) { + const auto writesLink = [&](std::uint32_t index) { + const SwappcCall* call = findCallAt(calls, index); + return (call != nullptr && call->linkRegister == link) || (std::ranges::find(unlinked, index) != unlinked.end() && scalarIndex(program.instructions[index].destination) == link); + }; + std::vector holds(size, 0u); + std::vector work; + for (std::uint32_t index = 0; index < size; ++index) { + if (writesLink(index)) work.push_back(index); + } + const auto after = [&](std::uint32_t index) -> std::uint8_t { + if (writesLink(index)) return 3u; + std::uint8_t value = holds[index]; + if (overwritesScalar(program.instructions[index], link, waveSize)) value &= 2u; + if (overwritesScalar(program.instructions[index], link + 1u, waveSize)) value &= 1u; + return value; + }; + while (!work.empty()) { + const auto index = work.back(); + work.pop_back(); + const auto value = after(index); + for (const auto successor : successors(index)) { + if ((holds[successor] | value) == holds[successor]) continue; + holds[successor] |= value; + work.push_back(successor); + } + } + for (std::uint32_t index = 0; index < size; ++index) { + const auto& instruction = program.instructions[index]; + if (holds[index] == 0u || writesLink(index) || instruction.family == RdnaInstructionFamily::SOPC || findReturnAt(calls, index) != nullptr) continue; + const bool inside = std::ranges::any_of(calls, [&](const SwappcCall& call) { return !call.fetch && call.linkRegister == link && index >= call.targetIndex && index <= call.returnIndex; }); + if (inside || !(((holds[index] & 1u) != 0u && readsScalarDword(instruction, link, waveSize)) || ((holds[index] & 2u) != 0u && readsScalarDword(instruction, link + 1u, waveSize)))) continue; + const auto owner = std::ranges::find_if(calls, [&](const SwappcCall& call) { return !call.fetch && call.linkRegister == link; }); + throw escapeError(*owner, instruction); + } + } +} + +std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc, std::vector& unlinked) { std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { const auto& instruction = program.instructions[index]; @@ -502,6 +794,10 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa continue; } if (!staticTarget || call.targetIndex == InvalidControlFlowId) { + if (swappc != nullptr && std::ranges::find(swappc->linkedCallFallbacks, instruction.programCounter) != swappc->linkedCallFallbacks.end()) { + unlinked.push_back(index); + continue; + } if (std::ranges::any_of(calls, [&](const SwappcCall& other) { return other.linkRegister == call.linkRegister; })) { throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(instruction.programCounter) + ": link register s[" + std::to_string(call.linkRegister) + "] is shared with another call"); @@ -574,7 +870,8 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa continue; } const bool outsideRegion = regionEnd != InvalidControlFlowId && (index < call.targetIndex || index > regionEnd); - if (touchesLinkRegister(instruction, call.linkRegister) && !(outsideRegion && onlyWritesLinkRegister(instruction, call.linkRegister))) { + if (outsideRegion) continue; + if (touchesLinkRegister(instruction, call.linkRegister)) { throw escapeError(call, instruction); } } @@ -609,14 +906,12 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa if (std::ranges::any_of(calls, [&](const SwappcCall& other) { return other.callIndex == index && other.linkRegister == call.linkRegister; })) continue; const auto& instruction = program.instructions[index]; const auto region = std::ranges::find_if(calls, [&](const SwappcCall& other) { return other.linkRegister == call.linkRegister && index >= other.targetIndex && index <= other.returnIndex; }); - if (region != calls.end()) { - if (index != region->returnIndex && touchesLinkRegister(instruction, call.linkRegister)) throw escapeError(*region, instruction); - } else if (touchesLinkRegister(instruction, call.linkRegister) && !onlyWritesLinkRegister(instruction, call.linkRegister)) { - throw escapeError(call, instruction); - } + if (region != calls.end() && index != region->returnIndex && touchesLinkRegister(instruction, call.linkRegister)) throw escapeError(*region, instruction); } } + requireReturnAddressesStayInCalls(program, calls, unlinked, swappc != nullptr ? swappc->waveSize : 0u, escapeError); + for (std::size_t first = 0; first < calls.size(); ++first) { if (!calls[first].fetch && calls[first].callIndex >= calls[first].targetIndex && calls[first].callIndex <= calls[first].returnIndex) { throw std::invalid_argument("recursive scalar call at program counter " + toHexString(program.instructions[calls[first].callIndex].programCounter) + @@ -664,7 +959,7 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa } if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { const SwappcCall* call = findCallAt(calls, index); - transfers.push_back(Transfer{index, call->fetch ? index + 1u : call->targetIndex}); + transfers.push_back(Transfer{index, call == nullptr || call->fetch ? index + 1u : call->targetIndex}); continue; } if (instruction.op != RdnaOpcode::SEndpgm && instruction.op != RdnaOpcode::SCodeEnd) transfers.push_back(Transfer{index, index + 1u}); @@ -958,23 +1253,75 @@ std::optional UnresolvableSwappcTarget(const RdnaProgram& program return std::nullopt; } -std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { +namespace { + +std::set branchLabels(const RdnaProgram& program) { std::set labels; for (const auto& instruction : program.instructions) { if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) labels.insert(instruction.branchTarget); } - const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; - const auto unchangedUntil = [&](std::uint32_t reg, std::uint32_t reader) { - for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { - if (!writesScalar(program.instructions[index], reg)) continue; - if (index <= reader) return false; - for (std::uint32_t later = index; later < program.instructions.size(); ++later) { - const auto& instruction = program.instructions[later]; - if ((IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) && instruction.branchTarget <= program.instructions[reader].programCounter) return false; - } + return labels; +} + +bool unchangedUntil(const RdnaProgram& program, std::uint32_t reg, std::uint32_t reader, std::uint32_t waveSize) { + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + if (!writesScalar(program.instructions[index], reg, waveSize)) continue; + if (index <= reader) return false; + for (std::uint32_t later = index; later < program.instructions.size(); ++later) { + const auto& instruction = program.instructions[later]; + if ((IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) && instruction.branchTarget <= program.instructions[reader].programCounter) return false; } - return true; - }; + } + return true; +} + +bool isScalarBufferLoad(RdnaOpcode op) { + switch (op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: return true; + default: return false; + } +} + +bool loadedDword(const RdnaProgram& program, std::uint32_t reg, std::uint32_t before, std::uint32_t waveSize, std::uint32_t& load, std::uint32_t& dword) { + for (std::uint32_t copies = 0; copies < 4u; ++copies) { + std::uint32_t writer = 0; + if (!findLastWriter(program, before, {reg}, writer, waveSize)) return false; + const auto& instruction = program.instructions[writer]; + const auto destination = scalarIndex(instruction.destination); + const auto source = scalarIndex(instruction.source0); + if (instruction.op == RdnaOpcode::SMovB32 && destination == reg && source != NoScalarRegister) { + reg = source; + } else if (instruction.op == RdnaOpcode::SMovB64 && destination != NoScalarRegister && reg - destination < 2u && source != NoScalarRegister) { + reg = source + (reg - destination); + } else if (isScalarBufferLoad(instruction.op) && destination != NoScalarRegister && reg >= destination && reg - destination < instruction.dataDwordCount) { + load = writer; + dword = reg - destination; + return true; + } else { + return false; + } + before = writer; + } + return false; +} + +std::uint32_t branchWord(std::uint32_t opcode, std::size_t from, std::size_t to, const char* what) { + const auto offset = static_cast(to) - static_cast(from) - 1; + if (offset < -0x8000 || offset > 0x7fff) throw std::invalid_argument(std::string(what) + " lands out of the 16-bit branch range"); + return opcode | (static_cast(offset) & 0xffffu); +} + +} + + +std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + const auto labels = branchLabels(program); + const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; + const auto unchanged = [&](std::uint32_t reg, std::uint32_t reader) { return unchangedUntil(program, reg, reader, swappc.waveSize); }; std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { const auto& instruction = program.instructions[index]; @@ -985,7 +1332,7 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw const auto source = scalarIndex(instruction.source0); if (link == NoScalarRegister || source == NoScalarRegister || source % 2u != 0u) continue; std::uint32_t pair = NoScalarRegister; - if (source != link && userData(source) && userData(source + 1u) && unchangedUntil(source, index) && unchangedUntil(source + 1u, index)) { + if (source != link && userData(source) && userData(source + 1u) && unchanged(source, index) && unchanged(source + 1u, index)) { pair = source; } else { std::uint32_t low = 0, high = 0; @@ -1003,7 +1350,7 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw from = scalarIndex(lowWriter.source0); if (from != NoScalarRegister && !isScalar(highWriter.source0, from + 1u)) from = NoScalarRegister; } - if (!sameBlock || from == NoScalarRegister || from % 2u != 0u || !userData(from) || !userData(from + 1u) || !unchangedUntil(from, low) || !unchangedUntil(from + 1u, high)) continue; + if (!sameBlock || from == NoScalarRegister || from % 2u != 0u || !userData(from) || !userData(from + 1u) || !unchanged(from, low) || !unchanged(from + 1u, high)) continue; pair = from; } calls.push_back({instruction.programCounter, link, pair}); @@ -1077,32 +1424,132 @@ std::optional UserDataCalleeLength(std::span } std::vector LinkUserDataCalls(const RdnaProgram& program, std::span calls, std::span> callees) { - if (calls.size() != callees.size()) throw std::logic_error("user-data calls and callees differ in number"); - for (const auto& instruction : program.instructions) { - if (instruction.op == RdnaOpcode::SGetpcB64) { - throw std::invalid_argument("a program that reads its program counter (s_getpc_b64 at program counter " + toHexString(instruction.programCounter) + ") cannot have user-data callees linked after it"); + return LinkCalls(program, calls, callees, {}, {}).code; +} + +std::vector FindTableCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + const auto labels = branchLabels(program); + const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; + const auto sameBlock = [&](std::uint32_t first, std::uint32_t last) { + return std::none_of(labels.begin(), labels.end(), [&](std::uint32_t label) { + return label > program.instructions[first].programCounter && label <= program.instructions[last].programCounter; + }); + }; + std::vector calls; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (instruction.op != RdnaOpcode::SSwappcB64) continue; + std::uint32_t ignored = 0; + if (resolveSetpcTarget(program, index, ignored)) continue; + const auto link = scalarIndex(instruction.destination); + const auto target = scalarIndex(instruction.source0); + if (link == NoScalarRegister || link % 2u != 0u || link > 104u || target == NoScalarRegister || target % 2u != 0u || target > 104u) continue; + std::uint32_t lowLoad = 0, lowDword = 0, highLoad = 0, highDword = 0; + if (!loadedDword(program, target, index, swappc.waveSize, lowLoad, lowDword) || !loadedDword(program, target + 1u, index, swappc.waveSize, highLoad, highDword)) continue; + if (lowLoad != highLoad || highDword != lowDword + 1u || !sameBlock(lowLoad, index)) continue; + const auto& load = program.instructions[lowLoad]; + const auto descriptor = scalarIndex(load.source0); + if (load.source0.kind != RdnaOperandKind::ScalarRegister || descriptor % 4u != 0u || load.memoryOffset >= 0x100000u || (load.memoryOffset & 3u) != 0u) continue; + std::uint32_t descriptorLoad = 0; + if (!findLastWriter(program, lowLoad, {descriptor, descriptor + 1u, descriptor + 2u, descriptor + 3u}, descriptorLoad, swappc.waveSize) || !sameBlock(descriptorLoad, index)) continue; + const auto& table = program.instructions[descriptorLoad]; + const auto pointer = scalarIndex(table.source0); + if (table.op != RdnaOpcode::SLoadDwordx4 || !isScalar(table.destination, descriptor) || table.source0.kind != RdnaOperandKind::ScalarRegister || table.source1.kind != RdnaOperandKind::Null || + table.memoryOffset >= 0x100000u || (table.memoryOffset & 3u) != 0u || !userData(pointer) || !userData(pointer + 1u) || + !unchangedUntil(program, pointer, descriptorLoad, swappc.waveSize) || !unchangedUntil(program, pointer + 1u, descriptorLoad, swappc.waveSize)) continue; + std::uint32_t stride = 0; + if (load.source1.kind != RdnaOperandKind::Null) { + const auto offset = scalarIndex(load.source1); + std::uint32_t shift = 0, shiftIndex = 0; + if (offset == NoScalarRegister || !findLastWriter(program, lowLoad, {offset}, shiftIndex, swappc.waveSize) || !sameBlock(shiftIndex, index)) continue; + const auto& shifted = program.instructions[shiftIndex]; + if (shifted.op != RdnaOpcode::SLshlB32 || !isScalar(shifted.destination, offset) || !isImmediate(shifted.source1, shift) || shift < 2u || shift > 24u) continue; + stride = 1u << shift; + } + calls.push_back({instruction.programCounter, link, target, pointer, table.memoryOffset, load.memoryOffset + lowDword * 4u, stride}); + } + return calls; +} + +LinkedCalls LinkCalls(const RdnaProgram& program, std::span userDataCalls, std::span> userDataCallees, + std::span tableCalls, std::span> tableCallees) { + if (userDataCalls.size() != userDataCallees.size()) throw std::logic_error("user-data calls and callees differ in number"); + if (tableCalls.size() != tableCallees.size()) throw std::logic_error("table calls and callees differ in number"); + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + BoundedJumpTable table; + if (program.instructions[index].op == RdnaOpcode::SSetpcB64 && resolveBoundedJumpTable(program, index, table)) { + throw std::invalid_argument("a program that jumps through a table in its code (s_setpc_b64 at program counter " + toHexString(program.instructions[index].programCounter) + ") cannot have callees linked over the words after its last instruction"); } } const auto end = instructionEndProgramCounter(program.instructions.back()) / 4u; std::vector linked(program.code.begin(), program.code.begin() + end); - for (std::size_t call = 0; call < calls.size(); ++call) { - const auto index = instructionIndexOfProgramCounter(program, calls[call].programCounter); + std::vector fallbacks; + const auto callWord = [&](std::uint32_t programCounter, const char* kind) { + const auto index = instructionIndexOfProgramCounter(program, programCounter); if (index == InvalidControlFlowId || program.instructions[index].op != RdnaOpcode::SSwappcB64 || program.instructions[index].wordCount != 1u) { - throw std::logic_error("user-data call does not name an s_swappc_b64"); + throw std::logic_error(std::string(kind) + " call does not name an s_swappc_b64"); } - const auto word = calls[call].programCounter / 4u; + return programCounter / 4u; + }; + for (std::size_t call = 0; call < userDataCalls.size(); ++call) { + const auto word = callWord(userDataCalls[call].programCounter, "user-data"); const auto offset = static_cast(linked.size()) - static_cast(word) - 1; - if (offset > 0x7fff) throw std::invalid_argument("user-data callee of the call at program counter " + toHexString(calls[call].programCounter) + " lands out of s_call_b64 range"); - linked[word] = 0xbb000000u | (calls[call].linkRegister << 16u) | static_cast(offset); - linked.insert(linked.end(), callees[call].begin(), callees[call].end()); + if (offset > 0x7fff) throw std::invalid_argument("user-data callee of the call at program counter " + toHexString(userDataCalls[call].programCounter) + " lands out of s_call_b64 range"); + linked[word] = 0xbb000000u | (userDataCalls[call].linkRegister << 16u) | static_cast(offset); + linked.insert(linked.end(), userDataCallees[call].begin(), userDataCallees[call].end()); + } + for (std::size_t call = 0; call < tableCalls.size(); ++call) { + const auto& table = tableCalls[call]; + const auto& callees = tableCallees[call]; + const auto word = callWord(table.programCounter, "table"); + const auto at = "the table callees of the call at program counter " + toHexString(table.programCounter); + const auto original = linked[word]; + const auto count = callees.size(); + const auto dispatch = linked.size(); + const auto zero = dispatch + 1u; + const auto one = zero + count * 7u + 2u; + const auto calls = one + count * 8u + 2u; + const auto fault = calls + count * 2u; + const auto branch = [&](std::uint32_t opcode, std::size_t target) { linked.push_back(branchWord(opcode, linked.size(), target, at.c_str())); }; + linked[word] = branchWord(0xbf820000u, word, dispatch, at.c_str()); + branch(0xbf850000u, one); + for (const bool sccOne : {false, true}) { + for (std::size_t callee = 0; callee < count; ++callee) { + const auto next = linked.size() + (sccOne ? 8u : 7u); + linked.insert(linked.end(), {0xbf07ff00u | table.targetRegister, static_cast(callees[callee].target)}); + branch(0xbf850000u, next); + linked.insert(linked.end(), {0xbf07ff00u | (table.targetRegister + 1u), static_cast(callees[callee].target >> 32u)}); + branch(0xbf850000u, next); + if (sccOne) linked.push_back(0xbf068080u); + branch(0xbf820000u, calls + callee * 2u); + } + linked.push_back(sccOne ? 0xbf068080u : 0xbf078080u); + branch(0xbf820000u, fault); + } + for (std::size_t callee = 0; callee < count; ++callee) { + linked.push_back(0u); + branch(0xbf820000u, word + 1u); + } + fallbacks.push_back(static_cast(linked.size() * 4u)); + linked.push_back(original); + branch(0xbf820000u, word + 1u); + for (std::size_t callee = 0; callee < count; ++callee) { + const auto site = calls + callee * 2u; + const auto offset = static_cast(linked.size()) - static_cast(site) - 1; + if (offset > 0x7fff) throw std::invalid_argument(at + " land out of s_call_b64 range"); + linked[site] = 0xbb000000u | (table.linkRegister << 16u) | static_cast(offset); + linked.insert(linked.end(), callees[callee].code.begin(), callees[callee].code.end()); + } } linked.push_back(0xbf9f0000u); - return linked; + return {std::move(linked), std::move(fallbacks)}; } ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program, const SwappcInfo* swappc) const { - const std::vector calls = analyzeSwappcCalls(program, swappc); + std::vector unlinked; + const std::vector calls = analyzeSwappcCalls(program, swappc, unlinked); ControlFlowGraph graph; + for (const auto index : unlinked) graph.unlinkedCallProgramCounters.push_back(program.instructions[index].programCounter); graph.blocks = splitIntoBlocks(program, calls); linkBlocks(graph.blocks, program, calls); for (const auto& call : calls) { diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index 93b102405197..bbff5df7b42a 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -762,7 +762,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(25u); + writer.WriteU32(26u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -786,6 +786,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteU32(request.target.registrationWordLimit); writer.WriteBool(request.target.vertexSubgroups); writer.WriteBool(request.target.float16DenormPreserve); + writer.WriteU32Span(request.context.linkedCallFallbacks); return base64Encode(buffer); } @@ -794,7 +795,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 25u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 26u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -820,14 +821,17 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const if (version >= 18u) result.request.target.vertexSubgroups = reader.ReadBool(); if (version >= 21u) { if (version >= 22u) reader.Require(1u); - if (reader.Remaining() == 1u) { + if (reader.Remaining() != 0u) { const auto float16DenormPreserve = reader.ReadU8(); if (float16DenormPreserve > 1u) throw std::runtime_error("RequestSerializer: invalid float16DenormPreserve value"); result.request.target.float16DenormPreserve = float16DenormPreserve != 0u; - } else if (reader.Remaining() != 0u) { - throw std::runtime_error("RequestSerializer: unexpected trailing data"); } } + if (version >= 26u) { + result.linkedCallFallbacks = reader.ReadU32Vector(); + result.request.context.linkedCallFallbacks = result.linkedCallFallbacks; + } + if (version >= 21u && reader.Remaining() != 0u) throw std::runtime_error("RequestSerializer: unexpected trailing data"); return result; } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 6ed27287abcc..fd43cc3a09a3 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -73,6 +73,7 @@ enum class IrOpcode : std::uint16_t { Void, Reference, ReferenceU32, + UnlinkedCall, GetUserData, GetShaderBase, ShaderClock, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index b61b111ffd9a..877646670945 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -114,6 +114,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("Void", Void), makeMeta("Reference", Void, U1), makeMeta("ReferenceU32", Void, U32), + makeMeta("UnlinkedCall", Void, U32, U32, U32), makeMeta("GetUserData", U32, ScalarReg), makeMeta("GetShaderBase", U64), makeMeta("ShaderClock", U64), @@ -888,6 +889,7 @@ bool IrOpcodeHasSideEffects(IrOpcode opcode) { case IrOpcode::Discard: case IrOpcode::Reference: case IrOpcode::ReferenceU32: + case IrOpcode::UnlinkedCall: case IrOpcode::SetTessellationAttribute: case IrOpcode::InterpolateHostP2: case IrOpcode::SetThreadBitScalarRegister: diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index d7574e7d4165..3530da409591 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -124,7 +124,7 @@ std::atomic frontEndBuilds{0}; struct PreparedControlFlow { PreparedControlFlow(const RecompileRequest& request, const SwappcInfo& swappcInfo) - : stage(request.shader.stage), swappc(swappcInfo), code(request.shader.code.begin(), request.shader.code.end()), + : stage(request.shader.stage), fallbacks(swappcInfo.linkedCallFallbacks.begin(), swappcInfo.linkedCallFallbacks.end()), swappc(WithFallbacks(swappcInfo, fallbacks)), code(request.shader.code.begin(), request.shader.code.end()), decoded(DecodeStageProgram(toShaderStageKind(request.shader.stage), code)), cfg(GraphBuilder{}.Build(decoded, &swappc)) { Structurizer{}.Structurize(cfg); } @@ -135,10 +135,16 @@ struct PreparedControlFlow { bool Matches(const RecompileRequest& request, const SwappcInfo& swappcInfo) const { return stage == request.shader.stage && swappc.fetchCallAllowed == swappcInfo.fetchCallAllowed && swappc.userDataBaseRegister == swappcInfo.userDataBaseRegister && swappc.userDataCount == swappcInfo.userDataCount && - std::ranges::equal(code, request.shader.code); + std::ranges::equal(fallbacks, swappcInfo.linkedCallFallbacks) && std::ranges::equal(code, request.shader.code); + } + + static SwappcInfo WithFallbacks(SwappcInfo info, const std::vector& fallbacks) { + info.linkedCallFallbacks = fallbacks; + return info; } ShaderStage stage; + std::vector fallbacks; SwappcInfo swappc; std::vector code; RdnaProgram decoded; @@ -147,7 +153,7 @@ struct PreparedControlFlow { std::shared_ptr ShaderPreparationContext::AcquireFrontend(const RecompileRequest& request) { const auto inputInfo = RequestInputInfo(request); - const SwappcInfo swappc{inputInfo.vertex != nullptr, request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + const SwappcInfo swappc{inputInfo.vertex != nullptr, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), 0u, request.context.linkedCallFallbacks}; static const bool reuse = std::getenv("APS5_NO_PERF_FRONTEND_PAIR") == nullptr; if (reuse && frontend != nullptr && frontend->Matches(request, swappc)) { return frontend; @@ -310,6 +316,7 @@ struct SourceEntry : std::enable_shared_from_this { // accepted only when its code matches word for word. Owned here because the request's span // points into a registration the driver may replace while the entry lives on. std::vector code; + std::vector linkedCallFallbacks; std::shared_ptr plan; // A plan build that threw (an unsupported resource chain or control flow) is remembered and // rethrown: the front end ran every pass before failing, ~13 ms per dispatch of a shader the @@ -408,6 +415,7 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr if (source == nullptr) { source = std::make_shared(); source->code.assign(request.shader.code.begin(), request.shader.code.end()); + source->linkedCallFallbacks.assign(request.context.linkedCallFallbacks.begin(), request.context.linkedCallFallbacks.end()); auto& bucket = sources[key]; if (!bucket.empty()) { // A second entry under one key is a code hash collision (or the unhashed key with @@ -1363,7 +1371,9 @@ std::span GetPreparedCode(const SourceHandle& handle) { return handle.source->code; } -PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) {} +PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) { + this->request.context.linkedCallFallbacks = handle->source->linkedCallFallbacks; +} std::optional PreparedShaderInvocation::TryCreate(const RecompileRequest& request, const std::shared_ptr& handle) { struct PreparedKeyStorage {}; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index bfb6796e11b5..99d67957f30c 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -200,6 +200,7 @@ struct GuestContext { std::optional vertex; std::span memory; std::optional floatMode; + std::span linkedCallFallbacks; }; struct MeshTargetLimits { diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index 0ad1aefdb9b2..51a49b08bbfd 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -220,6 +220,7 @@ std::uint32_t EmitFPNeg32(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFPCmpClass32(SpirvEmitterState& state, std::uint32_t value, std::uint32_t mask); void EmitReference(SpirvValueEmitContext& context); void EmitReferenceU32(SpirvValueEmitContext& context); +void EmitUnlinkedCall(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t low, std::uint32_t high, std::uint32_t programCounter); void EmitControlNop(SpirvValueEmitContext& context); void EmitWaitcnt(SpirvValueEmitContext& context); void EmitSendmsg(SpirvValueEmitContext& context); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 769ec4c3291a..456a79b76122 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -293,6 +293,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::Void: return Invoke(EmitVoid, ctx, inst); case IrOpcode::Reference: return Invoke(EmitReference, ctx, inst); case IrOpcode::ReferenceU32: return Invoke(EmitReferenceU32, ctx, inst); + case IrOpcode::UnlinkedCall: return Invoke(EmitUnlinkedCall, ctx, inst); case IrOpcode::GetUserData: return Invoke(EmitGetUserData, ctx, inst); case IrOpcode::GetShaderBase: return Invoke(EmitGetShaderBase, ctx, inst); case IrOpcode::ShaderClock: return Invoke(EmitShaderClock, ctx, inst); @@ -874,6 +875,13 @@ void EmitReferenceU32(SpirvValueEmitContext& context) { EmitVoid(context); } +void EmitUnlinkedCall(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t low, std::uint32_t high, std::uint32_t programCounter) { + auto& state = ctx.state; + if (state.faultBufferVariable == 0) ctx.Fail(inst, "calls an unlinked target without a fault buffer"); + RecordBdaFaultWords(state, low, high, ConstantU32(state, 0u), programCounter, BdaAbi::FaultReason::UnlinkedCall); + StopBdaInvocationIf(state, ConstantBool(state, true)); +} + void EmitControlNop(SpirvValueEmitContext& context) { EmitVoid(context); } diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 711f2190571d..f5c21e95cae8 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -27,6 +27,7 @@ class TranslationContext { void SetNativeFloat16(bool enabled) { nativeFloat16 = enabled; } void AddBranchCondition(const BasicBlock& source, BlockInfo& info); void TranslateCodeTableLoad(const RdnaInstruction& instruction, const ControlFlowGraph::CodeTableLoad& table); + void TranslateUnlinkedCall(const RdnaInstruction& instruction); private: struct AddressOperands { diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 18bb250332f7..9c67377bf07a 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -283,6 +283,13 @@ void TranslationContext::sSwappcB64(const RdnaInstruction& inst) { writeU32Pair(inst.destination, extractU64(link)); } +void TranslationContext::TranslateUnlinkedCall(const RdnaInstruction& inst) { + const std::array target = readU32Pair(inst.source0); + (void)ir.Emit(IrOpcode::UnlinkedCall, IrType::Void, {&target[0].Value(), &target[1].Value(), &ir.Constant(inst.programCounter)}); + program.Info().usesFaultBuffer = true; + sSwappcB64(inst); +} + void TranslationContext::sCselectB32(const RdnaInstruction& inst) { const IrU32 trueValue = readU32(sourceAt(inst, 0u)); const IrU32 falseValue = readU32(sourceAt(inst, 1u)); diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index 8bd53ac8db52..a138ba9fa378 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -86,6 +86,10 @@ const ShaderWorkgroupInputInfo* shaderWorkgroupInput(ShaderStageKind stage, cons } } +bool isUnlinkedCall(const ControlFlowGraph& cfg, std::uint32_t programCounter) { + return std::find(cfg.unlinkedCallProgramCounters.begin(), cfg.unlinkedCallProgramCounters.end(), programCounter) != cfg.unlinkedCallProgramCounters.end(); +} + bool isCodeTableLoad(const ControlFlowGraph& cfg, std::uint32_t programCounter) { return std::find(cfg.codeTableLoadProgramCounters.begin(), cfg.codeTableLoadProgramCounters.end(), programCounter) != cfg.codeTableLoadProgramCounters.end(); } @@ -534,6 +538,10 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con context.TranslateCodeTableLoad(instruction, *table); continue; } + if (isUnlinkedCall(cfg, instruction.programCounter)) { + context.TranslateUnlinkedCall(instruction); + continue; + } if (options.embeddedFetch != nullptr) { const auto load = std::ranges::find(options.embeddedFetch->loads, instruction.programCounter, &EmbeddedFetchLoad::programCounter); if (load != options.embeddedFetch->loads.end()) { diff --git a/core/shader/recompiler/tests/RequestSerializerTests.cpp b/core/shader/recompiler/tests/RequestSerializerTests.cpp index c60d90b112be..b7c52f62cf4c 100644 --- a/core/shader/recompiler/tests/RequestSerializerTests.cpp +++ b/core/shader/recompiler/tests/RequestSerializerTests.cpp @@ -202,9 +202,10 @@ int main() { Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u); Require(request.target.float16DenormPreserve == preserve); auto upgraded = bytes; - upgraded[4] = 25u; + upgraded[4] = 26u; if (fixture.pixel) upgraded.insert(upgraded.begin() + 258u, {0u, 0u}); if (fixture.tessellation) upgraded.insert(upgraded.end() - 35u, 5u, 0u); + upgraded.insert(upgraded.end(), sizeof(std::uint64_t), 0u); Require(serializer.Serialize(request) == EncodeBase64(upgraded)); auto truncated = bytes; truncated.pop_back(); @@ -239,9 +240,10 @@ int main() { layerRejected = std::string_view(error.what()) == "RequestSerializer: truncated data"; } Require(layerRejected); - layered[4] = 25u; + layered[4] = 26u; layered.insert(layered.begin() + 259u, 0u); if (fixture.tessellation) layered.insert(layered.end() - 35u, 5u, 0u); + layered.insert(layered.end(), sizeof(std::uint64_t), 0u); Require(serializer.Serialize(layerReplay.request) == EncodeBase64(layered)); } } @@ -262,8 +264,9 @@ int main() { Require(replay.request.context.pixel->rasterPrimitive == ShaderRecompiler::RasterPrimitive::Rectangle); Require(replay.request.context.vertex->paClVsOutCntl == 0x00050a05u); Require(replay.request.target.float16DenormPreserve == preserve); - bytes[4] = 25u; + bytes[4] = 26u; bytes.insert(bytes.begin() + 259u, 0u); + bytes.insert(bytes.end(), sizeof(std::uint64_t), 0u); Require(serializer.Serialize(replay.request) == EncodeBase64(bytes)); auto truncated23 = DecodeBase64(encoded23); @@ -384,8 +387,9 @@ int main() { Require(request.graphics->draw.indexAddress == 0xabcdef3400u && request.graphics->draw.indexCount == 57u && request.graphics->draw.indexElementBytes == 4u && request.graphics->draw.instanceCount == 9u); Require(request.target.float16DenormPreserve == preserve); auto upgraded = bytes; - upgraded[4] = 25u; + upgraded[4] = 26u; if (fixture.pixel) upgraded.insert(upgraded.begin() + 259u, 0u); + upgraded.insert(upgraded.end(), sizeof(std::uint64_t), 0u); Require(serializer.Serialize(request) == EncodeBase64(upgraded)); } } @@ -398,9 +402,9 @@ int main() { const auto replay = serializer.Deserialize(encoded); Require(replay.request.target.float16DenormPreserve == preserve); auto bytes = DecodeBase64(encoded); - Require(bytes.size() >= 8u && bytes[4] == 25u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); + Require(bytes.size() >= 8u && bytes[4] == 26u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); auto future = bytes; - future[4] = 26u; + future[4] = 27u; bool futureRejected = false; try { static_cast(serializer.Deserialize(EncodeBase64(future))); diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 979a7bfa7e72..dd1f856efb23 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -124,7 +124,8 @@ - [Table S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) DISABLE_CUBE_WRAP faults on selection without `VK_EXT_non_seamless_cube_map`: no runtime dimension check, including valid non-cube direct/table T#s. Unselected entries do not fault; with the feature this bit alone is accepted. Table T#/direct S# faults only for cubes needing the missing feature. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) whose record range overlaps a buffer or storage image written by its own dispatch faults (`table written by its dispatch`) when any entry is selected, even if the write misses the selected record. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) T# whose surface has no mapped byte faults when selected instead of becoming a zero texture as a direct T# does; what the console returns for it is not measured. -- [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped; in the shaders seen they are metadata, and a caller that reads its program counter, which could address them, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. +- [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped from the recompiled code; in the shaders seen they are metadata or constants the caller reads relative to its program counter, and those reads go to the captured guest code, which linking does not change. A caller that jumps through a table among those words, which the control flow analysis reads from the recompiled code, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. +- [Table callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is two consecutive dwords of an `s_buffer_load_dword*`, or a copy of them made with `s_mov_b32` or `s_mov_b64` in the same block, calls one of the targets that buffer holds when the load's buffer descriptor comes from an `s_load_dwordx4` without SOFFSET from a user-data pointer and the load's SOFFSET from an `s_lshl_b32` by 2 to 24 bits. At dispatch the descriptor and the buffer are read from guest memory, and every non-zero 64-bit value at the loaded offset plus a multiple of the shift's stride, below the descriptor's size (stride times num_records, or num_records bytes with stride 0), is read as a user-data callee and appended once per call. The call becomes a branch to a chain that compares the source pair with each target (`s_cmp_lg_u32` with a literal and `s_cbranch_scc1` per dword) and `s_call_b64`s the copy that matches; no match runs the original `s_swappc_b64` as an unlinked call. There is one chain for each value SCC has at the call, and the chain sets SCC back to that value (`s_cmp_eq_u32 0, 0` or `s_cmp_lg_u32 0, 0`) before the callee or the unlinked call runs. The searched range is the descriptor's whole size, which covers the range the load can reach whether it is checked in bytes or in strides; which one the PS5 uses for `s_buffer_load` was not measured. More than 64 distinct targets, a buffer above 64 MiB or a copy out of `s_call_b64` range throws. As with user-data callees, the buffer is read again only when a dispatch misses the dispatch cache, so a target a title adds to the buffer later is an unlinked call when a cached program runs. - [DB_RENDER_CONTROL resummarize draws](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - HTILE is not modeled, so a RESUMMARIZE_ENABLE draw without depth or stencil tests and without color writes changes no modeled state and draws nothing. A pure resummarize with pass-through D16/D32 depth, no depth writes or bounds test, no stencil plane, no color writes and no running pixel stage also draws nothing; other resummarize, copy or decompress draws throw. - [Comparison-sampled R32 float and R16 unorm textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) whose surface a storage image holds are sampled from a D32_SFLOAT or D16_UNORM image that the GPU refills from the storage image whenever its version moves, because Vulkan compares only depth formats. Only surfaces of at least 1 MiB (`APS5_DEPTH_COPY_MIN_KIB`) that are not 3D take the copy; smaller ones stay CPU snapshots compared against guest memory. With copies of small surfaces, one title hit an intermittent boot-time crash that was not explained. - [Null vertex buffer V#](../../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - a V# whose four dwords are all zero is bound to a zeroed vertex buffer of the attribute's size with stride 0, so every component of its attribute reads 0. This follows the V# fields in the [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) (`NUM_RECORDS` 0, `DST_SEL` 0 selects 0), not measured on hardware. diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 92d3cea3b1fc..30aed89f1c41 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -18,6 +18,7 @@ ### Functional +- [Unlinked calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) - the `s_swappc_b64` a table call's compare chain falls back to, when its target matches no linked copy, records the `UnlinkedCall` fault with the target and the program counter and stops the invocation, or continues after the call in programs whose faults do not stop invocations (see `bdaStopsInvocations`). The driver lists those calls in the request (`linkedCallFallbacks`), and it reports the fault like a BDA fault, naming the target: a dispatch that checks its faults when it completes throws, one whose check runs in a deferred write-back logs it and continues. The hardware would jump to the target, which is not modelled. Any other `s_swappc_b64` whose target is not resolved still throws when the shader is compiled. - [Cached source IR](../../../core/shader/recompiler/Recompiler.cpp) releases lazily allocated SSA construction tables after resource preparation and reclaims values unreachable from instructions, metadata and resource plans when retaining a pending source program. Immediate shader preparation skips reclamation because compilation consumes the program. Reclamation must account for every new IR pointer owner; a remaining external use throws. Live IDs are retained. Source and variant entries remain cached without an eviction bound. A pending source program is released after its first variant is obtained; memory beyond the measured title checkpoint and Linux performance remain unmeasured. - [Vertex user-data tail jumps](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) link a linear raw vertex prolog's terminal `s_setpc_b64` through an unchanged user-data pair to its continuation. The instruction semantics follow AMD RDNA2 ISA section 12.3; the prolog convention follows Mesa `aco_select_vs_prolog.cpp` at `0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8`. Console hardware has not been measured. Branching prologs, computed low pointers and continuations reading `s_getpc_b64` remain unsupported - [Raw fragment user-data tails](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) also link a terminal `s_setpc_b64` when the low half is unchanged user data and the last high-half writer is an unconditional immediate `s_mov_b32`. Other computed high halves remain unsupported. This follows AMD RDNA2 ISA section 12.3 and Mesa `aco_select_nir.cpp` (`create_fs_jump_to_epilog`), `aco_isel_helpers.cpp` (`convert_pointer_to_64_bit`) and `aco_lower_to_hw_instr.cpp` (`p_jump_to_epilog`) at `0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8`; console hardware has not been measured @@ -122,13 +123,13 @@ - [Raw `buffer_load_dwordx2`](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`. - [Buffer loads without GLC or DLC](../../../core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp) (shader recompiler) are not coherent in a program that has GLC or DLC buffer accesses and no `buffer_gl0_inv`, `buffer_gl1_inv`, `s_gl1_inv` or `s_dcache_inv`: only its GLC and DLC loads, stores and atomics use the `Coherent` buffer view, so a plain load can return a value cached before another work-group's write. This follows the [LLVM AMDGPU memory model for GFX10](https://llvm.org/docs/AMDGPUUsage.html#memory-model-gfx10-gfx11), in which a load-acquire is a GLC DLC load followed by `buffer_gl1_inv` and `buffer_gl0_inv`; it is not measured on the console or on RDNA hardware. - [DS src2 operations](../../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) (`ds_*_src2_*`, GFX10.1 only) read `DS[B]`, with `B = A + 4 * sext(offset1[6:0]:offset0)`, and then update `DS[A]` with the matching 32-bit atomic, so the read of `B` is not atomic with the update of `A`. The form that takes the offset from the address (`offset1[7]` set), the 64-bit forms and GDS throw: whether the LDS address keeps `A[31:17]` in that form, and whether the 64-bit forms scale the offset by 4 or 8, is not documented. `ds_min_src2_f32`, `ds_max_src2_f32` and `ds_add_src2_f32` throw: on RDNA2 (gfx1035, the oracle's hardware, which has no src2 opcodes) `ds_min_f32`, `ds_max_f32` and `ds_add_f32` ignore `IEEE_MODE` but depend on whether f32 denormals are flushed (28, 28 and 12 of 196 special rows differ), and that is not modelled for LDS float atomics yet (#981). -- [s_swappc_b64](../../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store) throws, apart from the overwrites and shared links that [static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) allow; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. +- [s_swappc_b64](../../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, hash, store) throws, apart from the overwrites, SOPC compares and shared links that [static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) allow; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. - The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. - The scalar aperture sources [src_shared_base, src_shared_limit, src_private_base, src_private_limit](../../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler, scalar source codes 235 to 238) read as the 64-bit apertures the flat routing uses: a base is the aperture's high dword (`SharedApertureHigh`, `PrivateApertureHigh`) over a low dword of 0, a limit is the same high dword over 0xffffffff. A 64-bit read (`s_mov_b64`, a 64-bit VALU source, a wave64 lane mask) gets the whole value and a 32-bit read gets the low dword. This follows LLVM's `SIRegisterInfo.td` (aperture registers are 64-bit, their low 32 bits are always 0 for a base and -1 for a limit) and `getSegmentAperture` (`AMDGPULegalizerInfo.cpp`, `SIISelLowering.cpp`: a 64-bit read whose high half is the aperture), the KFD's `MAKE_LDS_APP_LIMIT` (limit = base | 0xffffffff) and the gfx1030 and gfx1035 readouts on [#1190](https://github.com/boykopovar/AnyPS5/pull/1190) and [#3803](https://github.com/boykopovar/AnyPS5/issues/3803). Not measured on a console: the PS5's high dwords are the inferred values of the flat aperture bases entry, and the wave is taken to be in 64-bit address mode (PTR32 clear) - [EmitSetAttribute](../../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - a primitive with a vertex whose clip position is (0,0,0,0) is clipped away through an extra `gl_ClipDistance` element on vertex and tessellation evaluation shaders, matching what a Radeon 610M (RDNA2, RADV) and llvmpipe rasterise; NVIDIA 615.78.08 without it draws the vertex at the viewport centre. Measured for triangles (strip and list) and for a point at (0,0,0,0), which the Radeon 610M does not draw, NVIDIA draws at the viewport centre and llvmpipe at the top-left corner; lines with such a vertex are clipped by the same element and were not measured. Mesh, geometry and the generated rect-list and barycentric stages do not write it, a device without `shaderClipDistance` cannot compile vertex shaders (the compile throws), and guest clip distances are still refused by the pipeline validator. The PS5 rasteriser was not measured. - [PERSP_SAMPLE and LINEAR_SAMPLE](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) hold the pixel-centre barycentrics, like the centroid pairs. With one sample this is the sample position ([Vulkan standard sample locations](https://docs.vulkan.org/spec/latest/chapters/primsrast.html#primsrast-multisampling): (0.5, 0.5)); with more samples the console's per-sample weights are not computed and were not measured. - [LINE_STIPPLE_TEX](../../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) reads as 0: line stipple is not emulated. RADV enables the input only as the required pixel input of a shader that reads no interpolants, because it initializes the fewest VGPRs ([Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_shader.c) `radv_compute_spi_ps_input`), and never reads it -- [Static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair may only be overwritten whole, by `s_mov_b32`, `s_mov_b64`, an SOP2 result or a scalar load, as the caller of a user-data callee does before each call. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. +- [Static scalar calls](../../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair is free to use wherever no return address can reach: a dataflow over the decoded instructions follows each dword of the pair from the calls that write it until a scalar load, SALU or VALU result is known to overwrite it (a VALU mask counts as one SGPR unless the request says wave64), and an instruction there that may read a dword still holding a return address throws; SOPC compares are exempt, as they only set SCC. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. - [Relative vector addressing](../../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) (`v_movrels_b32`, `v_movreld_b32`, `v_movrelsd_b32`, `v_movrelsd_2_b32`, `v_swaprel_b32`) keeps every vector register from the lowest base register the program addresses relatively up to the last register the program names in a zero-initialised function-scope array per invocation (`LoadVectorWindow`/`StoreVectorWindow`, one element per register), so an indexed access is one load or store instead of a compare-and-select chain over the window; the host compiler decides whether that array stays in registers or spills to scratch, which is not measured. A register in the window the program never wrote reads 0, as an unwritten SSA register does; the console's value is undefined. M0 is unsigned: an index at or past the window reads `v0` and drops its write, and `v_swaprel_b32` writes neither side when either is out of range, as measured on RDNA2 (gfx1035, `agc_driver_relative_indexing`). The window ends at the last register the program names rather than at the shader's VGPR allocation, which the recompiler does not receive, so a relative write past the named registers but inside the allocation is dropped where the hardware keeps it for a later relative read. Killzone: Liberation's PSP vertex program (PPSA11142, 3176 ISA words, 104 relative accesses over 154 registers) went from 649,515 to 392,333 SPIR-V words with this. Pixel programs whose relative span contains the packed ancillary input instead keep that span in SSA and implement indexed operations with compare/select chains. This keeps ancillary definitions visible to field lowering and masked-write proofs; storing the unmodelled whole word in a private array would require composing unknown bits. A fold after masked-select elimination lowers fields whose consumers become visible only after redundant copies are removed. A whole-word use that the masked-select pass keeps gets the composed word described under the packed pixel ancillary word. Both representations include implicit entry inputs in the inferred register limit and retain unsigned M0, inactive-write and DPP update rules; wave32/wave64 native Vulkan readbacks cover relative reads, writes, split-offset swaps, inactive writes, DPP masks and out-of-range accesses, but console comparison and the performance cost of selecting over this span are unverified. - [s_call_b64](../../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) saves `PC + 4` over `GetShaderBase` and branches to `PC + 4 + signext(SIMM16) * 4`, following [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) sections 3.6.3, 12.2 and 12.3. Forward callees after `s_endpgm` are decoded. Measured on RDNA2 (gfx1036) in wave32 and wave64 with SIMM16 = 3: the link pair receives `PC + 4`, the branch goes to `PC + 4 + signext(SIMM16) * 4`, and `s_setpc_b64` of the link returns to the instruction after the call. Execution on the PS5 (gfx1013) and title compatibility for these bounded scalar calls remain unverified; `s_rfe_b64` still throws in translation. - [BDA span reads](../../../core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp) (shader recompiler) - `read_bda_span` is one selection: one arm calls `load_bda_span`, a `DontInline` function that holds the in-place dword loads and calls nothing, and the other arm is the per-dword byte fallback. Two other shapes of the same reads each hang a group of NVIDIA drivers, where a dispatch never finishes and the device is lost. With the loads emitted in the arm itself, a dispatch whose lanes took both arms hung on an RTX 3060 and an RTX 4060 Laptop GPU (Windows, driver 591.86) and an RTX 5060 (Linux, driver 590.48.01) ([#3291](https://github.com/boykopovar/AnyPS5/issues/3291)). With the loads and the fallback as two selections, the first under the span's condition and the second under its negation, it hung on an RTX 4070 Laptop GPU (Windows, drivers 595.97 and 596.36), an RTX 5080 (Windows, driver 610.62) and an RTX 4090 (Linux, driver 610.57.04) ([#4975](https://github.com/boykopovar/AnyPS5/issues/4975)). The current shape is measured on the RTX 4090 with driver 610.57.04 only, where it does not hang with or without SPIRV-Tools; a shape with the loads in their own `DontInline` function was reported passing on driver 591.86 in #3291. The cause inside the driver is not known.