diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp index bb0fcc4fda0c..dbdf2f167957 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp @@ -8,6 +8,9 @@ #define NOMINMAX #endif #include +#include +#else +#include #endif #include #include @@ -34,9 +37,9 @@ constexpr std::uint32_t OutputBase = 0x8000; alignas(256) std::array, 4> Programs{}; -std::span Code(bool coherent, bool barrier) { +std::span Code(bool coherent, bool barrier, std::uint32_t loadBase = LoadBase) { auto& program = Programs[(coherent ? 2u : 0u) + (barrier ? 1u : 0u)]; - std::vector code{0x34040084, 0x4a0404ff, LoadBase, 0x34080081, 0x36080882, 0x4a040504, 0x34060083, 0x4a0606ff, OutputBase, 0xdc388ff8 | (coherent ? 0x10000u : 0u), 0x04000002, 0xbf8c3f70}; + std::vector code{0x34040084, 0x4a0404ff, loadBase, 0x34080081, 0x36080882, 0x4a040504, 0x34060083, 0x4a0606ff, OutputBase, 0xdc388ff8 | (coherent ? 0x10000u : 0u), 0x04000002, 0xbf8c3f70}; if (barrier) code.push_back(0xbf8a0000); code.insert(code.end(), {0xdc708000, 0x00000503, 0xdc708004, 0x00000703, 0xbf810000}); std::copy(code.begin(), code.end(), program.begin()); @@ -118,6 +121,86 @@ ShaderRecompiler::RecompileResult Recompile(std::span code, return ShaderRecompiler::Recompile(request); } +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "BDA span reads: cannot capture the fault report"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + std::array buffer{}; + for (std::size_t read = 0; (read = std::fread(buffer.data(), 1, buffer.size(), file)) != 0;) text.append(buffer.data(), read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +bool RunFault(AgcDriver::VulkanDevice& device, GuestBlock& guest, bool coherent, bool barrier, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, const std::string& name, bool mixed = false) { + const auto loadBase = static_cast(mixed ? BlockBytes - 512u : BlockBytes + LoadBase); + const auto code = Code(coherent, barrier, loadBase); + auto memory = Initial(); + if (mixed) { + for (std::size_t dword = memory.size() - 128u; dword < memory.size(); ++dword) memory[dword] = 0x51000000u + static_cast(dword) * 0x00010203u; + } + std::memcpy(guest.Data(), memory.data(), BlockBytes); + auto expected = memory; + if (mixed) { + for (std::uint32_t tid = 0u; tid < Threads; ++tid) { + const auto load = loadBase + tid * 16u + (tid & 1u) * 2u - 8u; + if (load + 16u > BlockBytes) continue; + expected[OutputBase / 4u + tid * 2u] = Bytes(memory, load + 4u); + expected[OutputBase / 4u + tid * 2u + 1u] = Bytes(memory, load + 12u); + } + Require(expected != memory, "BDA span reads: the mixed control has no distinguishable valid lane writes"); + } + const auto address = static_cast(reinterpret_cast(guest.Data())); + Require(!GuestAllocations::Mutation().Overlaps(reinterpret_cast(address + BlockBytes), Threads * 16u + LoadBase + 16u), "BDA span reads: the fault input overlaps a registered allocation"); + const std::array userData{static_cast(address), static_cast(address >> 32u)}; + const auto result = Recompile(code, waveSize, userData, target); + Require(std::ranges::any_of(result.bindings, [](const auto& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable; }), "BDA span reads: the fault control has no BDA table"); + std::string failure; + StderrCapture log; + try { + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + } catch (const std::exception& error) { + failure = error.what(); + } + const auto deferred = log.Finish(); + if (failure.empty()) failure = deferred; + Require(failure.find("BDA access failed:") != std::string::npos && failure.find("reason=1") != std::string::npos, "BDA span reads: no unmapped fault report: " + failure); + std::uint32_t changed = 0u; + for (std::size_t dword = 0; dword < memory.size(); ++dword) { + std::uint32_t actual = 0u; + std::memcpy(&actual, guest.Data() + dword * 4u, sizeof(actual)); + if (actual != expected[dword]) ++changed; + } + std::printf("BDA fault span %s%s coherent=%u barrier=%u changed=%u refused=1\n", name.c_str(), mixed ? "-mixed" : "", coherent ? 1u : 0u, barrier ? 1u : 0u, changed); + return changed != 0u; +} + void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, bool coherent, bool barrier, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, const std::string& name) { const auto code = Code(coherent, barrier); auto memory = Initial(); @@ -143,13 +226,24 @@ int main() { GuestBlock guest; const auto native = device->Target(); const auto split = device->ComputeTarget(32); + std::uint32_t faultingWrites = 0u; for (const bool coherent : {false, true}) { for (const bool barrier : {false, true}) { Run(*device, guest, coherent, barrier, 32, native, "wave32"); Run(*device, guest, coherent, barrier, 64, native, "wave64"); if (split.subgroupSize != native.subgroupSize) Run(*device, guest, coherent, barrier, 64, split, "wave64 split"); + faultingWrites += RunFault(*device, guest, coherent, barrier, 32, native, "wave32") ? 1u : 0u; + faultingWrites += RunFault(*device, guest, coherent, barrier, 64, native, "wave64") ? 1u : 0u; + if (split.subgroupSize != native.subgroupSize) faultingWrites += RunFault(*device, guest, coherent, barrier, 64, split, "wave64-split") ? 1u : 0u; + if (barrier && native.subgroupSize == 32u) { + faultingWrites += RunFault(*device, guest, coherent, barrier, 32, native, "wave32", true) ? 1u : 0u; + faultingWrites += RunFault(*device, guest, coherent, barrier, 64, native, "wave64", true) ? 1u : 0u; + if (split.subgroupSize != native.subgroupSize) faultingWrites += RunFault(*device, guest, coherent, barrier, 64, split, "wave64-split", true) ? 1u : 0u; + } } } + if (native.subgroupSize != 32u) std::printf("skipped, mixed-lane barrier BDA fault controls need subgroup size 32, device reports %u\n", native.subgroupSize); + Require(faultingWrites == 0u, "BDA span reads: " + std::to_string(faultingWrites) + " faulting programs still wrote guest memory"); std::puts("BDA span read tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FaultingWaveInvocation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FaultingWaveInvocation.cpp new file mode 100644 index 000000000000..a5d18d34591b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FaultingWaveInvocation.cpp @@ -0,0 +1,123 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t FaultingLane = 5; +constexpr std::uint32_t PassLimit = 100; +constexpr std::uint32_t CountOffset = 0x200; +constexpr std::uint32_t ValueOffset = 0x300; +constexpr std::uint32_t Untouched = 0xdeadbeefu; +constexpr std::size_t BlockBytes = 65536; + +alignas(256) constexpr std::array Code{ + 0x34020082u, 0x7d8a0085u, 0x020202ffu, 0x00010000u, 0xdc308000u, 0x02000001u, 0xbf8c3f70u, 0xbe8a0380u, + 0x7da2040au, 0xbf880004u, 0x800a810au, 0xbf0aff0au, PassLimit, 0xbf85fffau, 0xbefe04c1u, 0x7e06020au, + 0x34080082u, 0xdc708200u, 0x00000304u, 0xdc708300u, 0x00000204u, 0xbf810000u, +}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "faulting wave invocation: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint32_t& At(std::uint32_t offset) { return block[offset / 4u]; } + std::uint64_t Address() const { return reinterpret_cast(block); } + +private: + std::uint32_t* block = nullptr; +}; + +std::uint32_t Passes(std::uint32_t lane) { + return lane % 4u + 1u; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestBlock guest; + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + guest.At(lane * 4u) = Passes(lane); + guest.At(CountOffset + lane * 4u) = Untouched; + guest.At(ValueOffset + lane * 4u) = Untouched; + } + const std::vector userData{static_cast(guest.Address()), static_cast(guest.Address() >> 32u)}; + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {64, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device->Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device->Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device->WaitIdle(); + std::uint32_t passes = 0; + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + if (lane != FaultingLane) passes = std::max(passes, Passes(lane)); + } + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + const auto what = "faulting wave invocation: lane " + std::to_string(lane); + const auto count = guest.At(CountOffset + lane * 4u); + const auto value = guest.At(ValueOffset + lane * 4u); + if (lane == FaultingLane) { + Require(count == Untouched && value == Untouched, what + " stored after its read faulted"); + continue; + } + Require(value == Passes(lane), what + " stored " + std::to_string(value) + ", expected its loaded " + std::to_string(Passes(lane))); + Require(count == passes, what + " left the EXEC loop after " + std::to_string(count) + " passes, expected " + std::to_string(passes) + " (the loop stops at " + std::to_string(PassLimit) + ")"); + } + std::puts("faulting wave invocation tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index bce467f22cdb..70c76b0abc2c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -25,6 +25,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe // every invocation running, so their faulting reads return zero instead. bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); +std::uint32_t BdaInvocationFaulted(SpirvEmitterState& state); std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); std::uint32_t EmitBdaReadBytes(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 3adafd732ebe..e6ad5665dae0 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -136,6 +136,7 @@ struct SpirvEmitterState { bool ldsLockFences = false; std::unordered_set sharedLaneValues; std::uint32_t laneHalf = 0; + bool laneValueShared = false; std::uint32_t spirvVersion = 0x00010300u; std::span supportedCapabilities; std::span supportedExtensions; @@ -199,6 +200,8 @@ struct SpirvEmitterState { std::uint32_t bdaPcOverride = 0; // False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop). bool bdaStopsInvocations = true; + bool bdaFaultMasksWrites = false; + std::array bdaFaultedVariables {}; bool continueTarget = false; // Execution scope of the barriers that keep one guest wave's LDS accesses in program order across // host invocations (see WaveLdsScope); 0 when none are emitted. diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index cb6d2a7cdacd..c416fe0e2e31 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -107,7 +107,34 @@ void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std:: EmitLabel(state, next); } +namespace { + +std::uint32_t BdaFaultedVariable(SpirvEmitterState& state, std::uint32_t half) { + auto& variable = state.bdaFaultedVariables.at(half); + if (variable == 0) variable = state.module.DefineInitializedGlobalVariable(TypePointer(state, spv::StorageClassPrivate, TypeBool(state)), spv::StorageClassPrivate, ConstantBool(state, false)); + return variable; +} + +std::uint32_t LoadBdaFaulted(SpirvEmitterState& state, std::uint32_t half) { + const auto faulted = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeBool(state), faulted, BdaFaultedVariable(state, half)); + return faulted; +} + +} + +std::uint32_t BdaInvocationFaulted(SpirvEmitterState& state) { + return LoadBdaFaulted(state, state.laneHalf); +} + void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition) { + if (state.bdaFaultMasksWrites && state.bdaStopValue == 0) { + for (std::uint32_t half = 0; half < state.laneCount; half++) { + if (half != state.laneHalf && !state.laneValueShared) continue; + state.module.AddFunction(spv::OpStore, BdaFaultedVariable(state, half), Binary(state, spv::OpLogicalOr, TypeBool(state), LoadBdaFaulted(state, half), condition)); + } + return; + } if (!state.bdaStopsInvocations) return; const auto failed = state.module.AllocateId(); const auto next = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index 9730b91a5c46..5c3246aa65c9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -41,8 +41,8 @@ std::uint32_t EmitBdaByte(SpirvEmitterState& state, std::uint32_t guest, std::ui const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType); const auto physical = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaPointerFunction, guest, ConstantU32(state, 1u), instruction); - if (state.bdaStopsInvocations) { - StopBdaInvocationIf(state, Binary(state, spv::OpIEqual, TypeBool(state), physical, BdaConstant(state, 0u))); + StopBdaInvocationIf(state, Binary(state, spv::OpIEqual, TypeBool(state), physical, BdaConstant(state, 0u))); + if (state.bdaStopsInvocations && !state.bdaFaultMasksWrites) { const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpConvertUToPtr, bytePointer, pointer, physical); const auto loaded = state.module.AllocateId(); @@ -134,9 +134,9 @@ std::uint32_t AddBdaSignedOffset(SpirvEmitterState& state, std::uint32_t address return result; } -std::uint32_t BdaSpanReadResultType(SpirvEmitterState& state) { +std::uint32_t BdaSpanReadResultType(SpirvEmitterState& state, bool reportsFault) { const auto values = TypeU32Vector(state, 4); - return state.bdaStopsInvocations ? state.module.Type(spv::OpTypeStruct, values, TypeU32(state)) : values; + return reportsFault ? state.module.Type(spv::OpTypeStruct, values, TypeU32(state)) : values; } std::uint32_t DefineBdaSpanLoadFunction(SpirvEmitterState& state, bool coherent) { @@ -180,7 +180,7 @@ std::uint32_t DefineBdaSpanReadFunction(SpirvEmitterState& state, bool coherent, const auto values = TypeU32Vector(state, 4); const auto zero = ConstantU32(state, 0u); const bool stops = state.bdaStopsInvocations; - const auto result = BdaSpanReadResultType(state); + const auto result = BdaSpanReadResultType(state, stops); const auto reader = state.bdaDwordReadFunctions[stops][coherent]; const auto function = state.module.AllocateId(); state.module.AddName(function, std::string("read_bda_span") + (stops ? "_stop" : "") + (coherent ? "_coherent" : "")); @@ -279,10 +279,13 @@ void DefineBdaDwordReadFunctions(SpirvEmitterState& state) { const auto& memory = state.program.Resources().memoryInfo; const bool coherentAccesses = std::any_of(memory.begin(), memory.end(), [](const MemoryInfo& info) { return info.coherent; }); const bool stops = state.bdaStopsInvocations; - if (stops) state.bdaStopValue = state.module.Constant(spv::OpConstantComposite, pair, ConstantU32(state, 0u), ConstantU32(state, 1u)); + const bool masksWrites = state.bdaFaultMasksWrites; + const bool reportsFault = stops || masksWrites; + state.bdaFaultMasksWrites = false; + if (reportsFault) state.bdaStopValue = state.module.Constant(spv::OpConstantComposite, pair, ConstantU32(state, 0u), ConstantU32(state, 1u)); for (const bool stop : {false, true}) { for (const bool coherent : {false, true}) { - if ((stop && !stops) || (coherent && !coherentAccesses)) continue; + if ((stop && !reportsFault) || (coherent && !coherentAccesses)) continue; const auto function = state.module.AllocateId(); state.module.AddName(function, std::string("read_bda_dword_bytes") + (stop ? "_stop" : "") + (coherent ? "_coherent" : "")); state.module.AddFunction(spv::OpFunction, pair, function, spv::FunctionControlDontInlineMask, type); @@ -330,6 +333,7 @@ void DefineBdaDwordReadFunctions(SpirvEmitterState& state) { } } state.bdaStopsInvocations = stops; + state.bdaFaultMasksWrites = masksWrites; state.bdaStopValue = 0; } @@ -338,21 +342,25 @@ void DefineBdaSpanReadFunctions(SpirvEmitterState& state) { const auto& memory = state.program.Resources().memoryInfo; const bool coherentAccesses = std::any_of(memory.begin(), memory.end(), [](const MemoryInfo& info) { return info.coherent; }); const bool stops = state.bdaStopsInvocations; + const bool masksWrites = state.bdaFaultMasksWrites; + const bool reportsFault = stops || masksWrites; + state.bdaFaultMasksWrites = false; const auto zero = ConstantU32(state, 0u); std::array loaders{}; for (const bool coherent : {false, true}) { if (!coherent || coherentAccesses) loaders[coherent] = DefineBdaSpanLoadFunction(state, coherent); } for (const bool stop : {false, true}) { - if (stop && !stops) continue; + if (stop && !reportsFault) continue; state.bdaStopsInvocations = stop; - state.bdaStopValue = stop ? state.module.Constant(spv::OpConstantComposite, BdaSpanReadResultType(state), state.module.Constant(spv::OpConstantComposite, TypeU32Vector(state, 4), zero, zero, zero, zero), ConstantU32(state, 1u)) : 0u; + state.bdaStopValue = stop ? state.module.Constant(spv::OpConstantComposite, BdaSpanReadResultType(state, stop), state.module.Constant(spv::OpConstantComposite, TypeU32Vector(state, 4), zero, zero, zero, zero), ConstantU32(state, 1u)) : 0u; for (const bool coherent : {false, true}) { if (coherent && !coherentAccesses) continue; state.bdaSpanReadFunctions[stop][coherent] = DefineBdaSpanReadFunction(state, coherent, loaders[coherent]); } } state.bdaStopsInvocations = stops; + state.bdaFaultMasksWrites = masksWrites; state.bdaStopValue = 0; } @@ -857,6 +865,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); const auto instruction = BdaInstructionPc(state, inst); const auto used = usedDwords != 0u ? usedDwords : everyDword ? (1u << dwords) - 1u : UsedBdaDwords(inst, dwords); + const bool reportsFault = state.bdaStopsInvocations || state.bdaFaultMasksWrites; const auto isUsed = [&](std::uint32_t dword) { return (used & (1u << dword)) != 0u; }; std::array values{}; std::uint32_t first = dwords; @@ -873,7 +882,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const std::array addresses{}; const auto dwordAddress = [&](std::uint32_t dword) { return AddBdaImmediate(ctx, inst, address, static_cast(offset + dword * 4u)); }; const auto readBytes = [&](std::array& into) { - const auto function = state.bdaDwordReadFunctions[state.bdaStopsInvocations][BdaCoherent(state, inst)]; + const auto function = state.bdaDwordReadFunctions[reportsFault][BdaCoherent(state, inst)]; if (function == 0) ctx.Fail(inst, "BDA dword read function is missing"); for (std::uint32_t dword = first; dword <= last; ++dword) { if (!isUsed(dword)) continue; @@ -882,7 +891,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 2), read, function, addresses[dword], instruction); into[dword] = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), into[dword], read, 0u); - if (!state.bdaStopsInvocations) continue; + if (!reportsFault) continue; const auto stopped = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), stopped, read, 1u); StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, TypeBool(state), stopped, ConstantU32(state, 0u))); @@ -893,11 +902,11 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const readBytes(values); return values; } - const auto function = state.bdaSpanReadFunctions[state.bdaStopsInvocations][BdaCoherent(state, inst)]; + const auto function = state.bdaSpanReadFunctions[reportsFault][BdaCoherent(state, inst)]; if (function == 0) ctx.Fail(inst, "BDA span read function is missing"); const auto read = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, BdaSpanReadResultType(state), read, function, address, ConstantU32(state, offset), ConstantU32(state, used), instruction); - if (state.bdaStopsInvocations) { + state.module.AddFunction(spv::OpFunctionCall, BdaSpanReadResultType(state, reportsFault), read, function, address, ConstantU32(state, offset), ConstantU32(state, used), instruction); + if (reportsFault) { const auto stopped = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), stopped, read, 1u); StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, TypeBool(state), stopped, ConstantU32(state, 0u))); @@ -905,7 +914,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const for (std::uint32_t dword = first; dword <= last; ++dword) { if (!isUsed(dword)) continue; values[dword] = state.module.AllocateId(); - if (state.bdaStopsInvocations) state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), values[dword], read, 0u, dword); + if (reportsFault) state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), values[dword], read, 0u, dword); else state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), values[dword], read, dword); } return values; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 591fac5fd98f..6242f7d1c589 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -331,6 +331,10 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh // Stopped invocations would leave the wave LDS barriers incomplete. state.bdaStopsInvocations = state.waveLdsScope == 0 && !state.spannedWave && BdaInvocationsMayStop(program) && program.Resources().stage != IrShaderStage::Mesh; state.meshWindowBuffer = target.meshWindowBuffer && program.Resources().stage == IrShaderStage::Mesh && program.Metadata().vectorWindowSize != 0u && state.laneCount * MeshWindowInvocations(state) * program.Metadata().vectorWindowSize <= MeshWindowSlotDwords; + const bool waveOperations = state.laneCount != 1u || state.requirements.subgroupBallot || state.requirements.subgroupShuffle; + const bool faults = program.Info().usesDma || program.Info().usesFaultBuffer; + const bool barrierCompute = !state.bdaStopsInvocations && program.Resources().stage == IrShaderStage::Compute; + state.bdaFaultMasksWrites = faults && (barrierCompute || (state.bdaStopsInvocations && waveOperations && program.Resources().stage != IrShaderStage::Pixel)); EmitModuleHeader(state, bindings); EmitProgram(state); state.module.EmitEntryPoint(ExecutionModelForStage(state.program.Resources().stage), state.mainFunc, "main", state.interfaceVariables); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 892a2ae2a37c..49b8204fb827 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -736,6 +736,41 @@ void EmitStructuredInstruction(SpirvValueEmitContext& ctx, StructuredFunctionSta EmitDirectInstruction(ctx, inst); } +bool WritesGuestState(IrOpcode opcode) { + if (opcode == IrOpcode::SetAttribute || opcode == IrOpcode::SetTessellationAttribute || opcode == IrOpcode::EmitVertex) return true; + const auto shared = SharedAccessOf(opcode); + const auto buffer = BufferAccessOf(opcode); + const auto address = AddressOpcodeInfoOf(opcode).access; + const auto image = ImageOpcodeInfoOf(opcode).access; + return (shared != SharedAccess::None && shared != SharedAccess::Read) || buffer == BufferAccess::Write || buffer == BufferAccess::Atomic || address == AddressAccess::Write || address == AddressAccess::Atomic || image == ImageAccess::Write || image == ImageAccess::Atomic; +} + +void EmitUnlessInvocationFaulted(SpirvValueEmitContext& ctx, StructuredFunctionState& functionState, const IrValue& inst) { + auto& state = ctx.state; + const bool returnsValue = inst.Type() != IrType::Void; + const auto result = returnsValue ? ctx.Result(inst) : 0u; + ctx.definitions.erase(&inst); + const auto faulted = BdaInvocationFaulted(state); + const auto body = state.module.AllocateId(); + const auto skip = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, faulted, skip, body); + EmitLabel(state, body); + EmitStructuredInstruction(ctx, functionState, inst); + const auto value = returnsValue ? ctx.Def(&inst) : 0u; + const auto exit = state.currentLabel; + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, skip); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); + if (!returnsValue) return; + ctx.definitions.erase(&inst); + const auto type = TypeId(state, inst.Type()); + state.module.AddFunction(spv::OpPhi, type, result, value, exit, state.module.Constant(spv::OpConstantNull, type), skip); + ctx.definitions.emplace(&inst, result); +} + void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& functionState, const IrBlock* block) { auto& state = ctx.state; state.currentBlock = block; @@ -784,11 +819,14 @@ void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& fu } SpirvValueEmitContext& lane = half == 0 ? ctx : *ctx.otherHalf; state.laneHalf = half; + state.laneValueShared = shared; if (half == 0 || (inst->Opcode() != IrOpcode::Barrier && inst->Opcode() != IrOpcode::MeshAllocate)) { - EmitStructuredInstruction(lane, functionState, *inst); + if (state.bdaFaultMasksWrites && WritesGuestState(inst->Opcode())) EmitUnlessInvocationFaulted(lane, functionState, *inst); + else EmitStructuredInstruction(lane, functionState, *inst); } } state.laneHalf = 0; + state.laneValueShared = false; } } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 03f6b6d33e2f..7d5954d7c6b4 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -1,5 +1,6 @@ # shader technical debt +- [BDA read faults](../../../core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp) keep compute invocations at their barriers and mask later writes by faulted lanes. Hull and mesh programs with barriers still use the non-stopping fault protocol without that write mask. Console fault behavior is not modeled. ### Build - [GuestContext memory](../../../core/shader/recompiler/Recompiler.hpp) rejects direct construction from temporary owning ranges. The view does not own the region array or its referenced bytes. That storage must outlive the request and its consumers, including when supplied through named containers, existing spans or pointers.